diff options
-rw-r--r-- | .gitignore | 6 | ||||
-rw-r--r-- | .travis.yml | 9 | ||||
-rw-r--r-- | CONTRIBUTING.md | 41 | ||||
-rw-r--r-- | Makefile.am | 79 | ||||
-rw-r--r-- | README.md | 4 | ||||
-rw-r--r-- | build-aux/m4/bitcoin_qt.m4 | 2 | ||||
-rw-r--r-- | configure.ac | 83 | ||||
-rw-r--r-- | contrib/debian/changelog | 125 | ||||
-rw-r--r-- | contrib/debian/control | 55 | ||||
-rw-r--r-- | contrib/debian/examples/bitcoin.conf | 14 | ||||
-rwxr-xr-x | contrib/debian/rules | 4 | ||||
-rwxr-xr-x | contrib/devtools/check-doc.py | 2 | ||||
-rwxr-xr-x | contrib/devtools/commit-script-check.sh | 39 | ||||
-rwxr-xr-x | contrib/devtools/copyright_header.py | 2 | ||||
-rwxr-xr-x | contrib/devtools/github-merge.py | 120 | ||||
-rw-r--r-- | contrib/gitian-descriptors/gitian-linux.yml | 1 | ||||
-rw-r--r-- | contrib/gitian-descriptors/gitian-osx.yml | 1 | ||||
-rw-r--r-- | contrib/gitian-descriptors/gitian-win.yml | 10 | ||||
-rw-r--r-- | contrib/init/README.md | 4 | ||||
-rw-r--r-- | contrib/init/bitcoind.openrcconf | 2 | ||||
-rw-r--r-- | contrib/linearize/README.md | 3 | ||||
-rw-r--r-- | contrib/linearize/example-linearize.cfg | 1 | ||||
-rwxr-xr-x | contrib/linearize/linearize-hashes.py | 23 | ||||
-rwxr-xr-x | contrib/macdeploy/detached-sig-create.sh | 2 | ||||
-rwxr-xr-x | contrib/macdeploy/macdeployqtplus | 6 | ||||
-rw-r--r-- | contrib/rpm/bitcoin.spec | 6 | ||||
-rw-r--r-- | contrib/seeds/README.md | 2 | ||||
-rwxr-xr-x | contrib/testgen/gen_base58_test_vectors.py | 2 | ||||
-rwxr-xr-x | contrib/verify-commits/gpg.sh | 40 | ||||
-rw-r--r-- | contrib/verify-commits/trusted-keys | 4 | ||||
-rw-r--r-- | contrib/verify-commits/trusted-sha512-root-commit | 1 | ||||
-rwxr-xr-x | contrib/verify-commits/verify-commits.sh | 149 | ||||
-rwxr-xr-x | contrib/windeploy/detached-sig-create.sh | 34 | ||||
-rw-r--r-- | contrib/windeploy/win-codesign.cert | 99 | ||||
-rwxr-xr-x | contrib/zmq/zmq_sub3.4.py | 2 | ||||
-rwxr-xr-x | depends/config.guess | 12 | ||||
-rwxr-xr-x | depends/config.sub | 14 | ||||
-rw-r--r-- | depends/packages/boost.mk | 6 | ||||
-rw-r--r-- | depends/packages/dbus.mk | 6 | ||||
-rw-r--r-- | depends/packages/libevent.mk | 6 | ||||
-rw-r--r-- | depends/packages/miniupnpc.mk | 4 | ||||
-rw-r--r-- | depends/packages/native_ccache.mk | 4 | ||||
-rw-r--r-- | depends/packages/zlib.mk | 4 | ||||
-rw-r--r-- | doc/.gitignore | 1 | ||||
-rw-r--r-- | doc/Doxyfile.in (renamed from doc/Doxyfile) | 2 | ||||
-rw-r--r-- | doc/README.md | 6 | ||||
-rw-r--r-- | doc/README_windows.txt | 4 | ||||
-rw-r--r-- | doc/build-osx.md | 6 | ||||
-rw-r--r-- | doc/build-unix.md | 6 | ||||
-rw-r--r-- | doc/developer-notes.md | 129 | ||||
-rw-r--r-- | doc/fuzzing.md | 8 | ||||
-rw-r--r-- | doc/gitian-building.md | 2 | ||||
-rw-r--r-- | doc/man/Makefile.am | 14 | ||||
-rw-r--r-- | doc/release-notes.md | 64 | ||||
-rw-r--r-- | doc/release-notes/release-notes-0.14.0.md | 873 | ||||
-rw-r--r-- | doc/release-notes/release-notes-0.14.1.md | 143 | ||||
-rw-r--r-- | doc/release-process.md | 55 | ||||
-rw-r--r-- | qa/README.md | 87 | ||||
-rwxr-xr-x | qa/rpc-tests/blockchain.py | 85 | ||||
-rwxr-xr-x | qa/rpc-tests/maxblocksinflight.py | 97 | ||||
-rwxr-xr-x | qa/rpc-tests/nodehandling.py | 82 | ||||
-rwxr-xr-x | qa/rpc-tests/p2p-mempool.py | 106 | ||||
-rwxr-xr-x | qa/rpc-tests/test_framework/test_framework.py | 218 | ||||
-rwxr-xr-x | share/genbuild.sh | 6 | ||||
-rw-r--r-- | src/Makefile.am | 26 | ||||
-rw-r--r-- | src/Makefile.bench.include | 3 | ||||
-rw-r--r-- | src/Makefile.qt.include | 3 | ||||
-rw-r--r-- | src/Makefile.qttest.include | 23 | ||||
-rw-r--r-- | src/Makefile.test.include | 67 | ||||
-rw-r--r-- | src/addrdb.cpp | 18 | ||||
-rw-r--r-- | src/addrdb.h | 6 | ||||
-rw-r--r-- | src/addrman.cpp | 10 | ||||
-rw-r--r-- | src/addrman.h | 23 | ||||
-rw-r--r-- | src/amount.h | 45 | ||||
-rw-r--r-- | src/arith_uint256.cpp | 6 | ||||
-rw-r--r-- | src/base58.cpp | 2 | ||||
-rw-r--r-- | src/bench/bench.cpp | 3 | ||||
-rw-r--r-- | src/bench/bench.h | 7 | ||||
-rw-r--r-- | src/bench/ccoins_caching.cpp | 4 | ||||
-rw-r--r-- | src/bench/checkblock.cpp | 8 | ||||
-rw-r--r-- | src/bench/checkqueue.cpp | 2 | ||||
-rw-r--r-- | src/bench/coin_selection.cpp | 4 | ||||
-rw-r--r-- | src/bench/crypto_hash.cpp | 25 | ||||
-rw-r--r-- | src/bench/lockedpool.cpp | 4 | ||||
-rw-r--r-- | src/bench/mempool_eviction.cpp | 7 | ||||
-rw-r--r-- | src/bench/prevector_destructor.cpp | 36 | ||||
-rw-r--r-- | src/bitcoin-cli.cpp | 10 | ||||
-rw-r--r-- | src/bitcoin-tx.cpp | 56 | ||||
-rw-r--r-- | src/bitcoind.cpp | 21 | ||||
-rw-r--r-- | src/blockencodings.cpp | 11 | ||||
-rw-r--r-- | src/bloom.cpp | 12 | ||||
-rw-r--r-- | src/bloom.h | 8 | ||||
-rw-r--r-- | src/chain.h | 18 | ||||
-rw-r--r-- | src/chainparams.cpp | 42 | ||||
-rw-r--r-- | src/chainparams.h | 21 | ||||
-rw-r--r-- | src/chainparamsbase.cpp | 24 | ||||
-rw-r--r-- | src/chainparamsbase.h | 16 | ||||
-rw-r--r-- | src/checkqueue.h | 24 | ||||
-rw-r--r-- | src/clientversion.h | 26 | ||||
-rw-r--r-- | src/coins.cpp | 267 | ||||
-rw-r--r-- | src/coins.h | 360 | ||||
-rw-r--r-- | src/compat.h | 11 | ||||
-rw-r--r-- | src/consensus/tx_verify.cpp | 246 | ||||
-rw-r--r-- | src/consensus/tx_verify.h | 78 | ||||
-rw-r--r-- | src/core_memusage.h | 5 | ||||
-rw-r--r-- | src/core_write.cpp | 6 | ||||
-rw-r--r-- | src/crypto/chacha20.cpp | 180 | ||||
-rw-r--r-- | src/crypto/chacha20.h | 26 | ||||
-rw-r--r-- | src/crypto/common.h | 21 | ||||
-rw-r--r-- | src/cuckoocache.h | 29 | ||||
-rw-r--r-- | src/dbwrapper.cpp | 67 | ||||
-rw-r--r-- | src/dbwrapper.h | 52 | ||||
-rw-r--r-- | src/fs.cpp | 17 | ||||
-rw-r--r-- | src/fs.h | 24 | ||||
-rw-r--r-- | src/hash.cpp | 41 | ||||
-rw-r--r-- | src/hash.h | 36 | ||||
-rw-r--r-- | src/httprpc.cpp | 14 | ||||
-rw-r--r-- | src/httpserver.cpp | 81 | ||||
-rw-r--r-- | src/httpserver.h | 4 | ||||
-rw-r--r-- | src/init.cpp | 296 | ||||
-rw-r--r-- | src/key.cpp | 8 | ||||
-rw-r--r-- | src/key.h | 5 | ||||
-rw-r--r-- | src/memusage.h | 20 | ||||
-rw-r--r-- | src/merkleblock.cpp | 2 | ||||
-rw-r--r-- | src/merkleblock.h | 2 | ||||
-rw-r--r-- | src/miner.cpp | 269 | ||||
-rw-r--r-- | src/miner.h | 36 | ||||
-rw-r--r-- | src/net.cpp | 277 | ||||
-rw-r--r-- | src/net.h | 35 | ||||
-rw-r--r-- | src/net_processing.cpp | 519 | ||||
-rw-r--r-- | src/net_processing.h | 12 | ||||
-rw-r--r-- | src/netaddress.cpp | 6 | ||||
-rw-r--r-- | src/netaddress.h | 1 | ||||
-rw-r--r-- | src/netbase.cpp | 28 | ||||
-rw-r--r-- | src/netbase.h | 2 | ||||
-rw-r--r-- | src/policy/feerate.cpp (renamed from src/amount.cpp) | 2 | ||||
-rw-r--r-- | src/policy/feerate.h | 54 | ||||
-rw-r--r-- | src/policy/fees.cpp | 800 | ||||
-rw-r--r-- | src/policy/fees.h | 322 | ||||
-rw-r--r-- | src/policy/policy.cpp | 44 | ||||
-rw-r--r-- | src/policy/policy.h | 8 | ||||
-rw-r--r-- | src/pow.cpp | 5 | ||||
-rw-r--r-- | src/prevector.h | 17 | ||||
-rw-r--r-- | src/primitives/transaction.cpp | 29 | ||||
-rw-r--r-- | src/primitives/transaction.h | 47 | ||||
-rw-r--r-- | src/qt/bantablemodel.cpp | 4 | ||||
-rw-r--r-- | src/qt/bitcoin.cpp | 47 | ||||
-rw-r--r-- | src/qt/bitcoingui.cpp | 3 | ||||
-rw-r--r-- | src/qt/callback.h | 30 | ||||
-rw-r--r-- | src/qt/clientmodel.cpp | 34 | ||||
-rw-r--r-- | src/qt/clientmodel.h | 6 | ||||
-rw-r--r-- | src/qt/coincontroldialog.cpp | 36 | ||||
-rw-r--r-- | src/qt/forms/debugwindow.ui | 2 | ||||
-rw-r--r-- | src/qt/forms/optionsdialog.ui | 102 | ||||
-rw-r--r-- | src/qt/forms/sendcoinsdialog.ui | 51 | ||||
-rw-r--r-- | src/qt/guiutil.cpp | 71 | ||||
-rw-r--r-- | src/qt/guiutil.h | 10 | ||||
-rw-r--r-- | src/qt/intro.cpp | 5 | ||||
-rw-r--r-- | src/qt/optionsdialog.cpp | 12 | ||||
-rw-r--r-- | src/qt/optionsdialog.h | 1 | ||||
-rw-r--r-- | src/qt/paymentserver.cpp | 12 | ||||
-rw-r--r-- | src/qt/peertablemodel.cpp | 2 | ||||
-rw-r--r-- | src/qt/rpcconsole.cpp | 27 | ||||
-rw-r--r-- | src/qt/sendcoinsdialog.cpp | 48 | ||||
-rw-r--r-- | src/qt/sendcoinsdialog.h | 3 | ||||
-rw-r--r-- | src/qt/test/paymentservertests.cpp | 2 | ||||
-rw-r--r-- | src/qt/test/rpcnestedtests.cpp | 9 | ||||
-rw-r--r-- | src/qt/test/test_main.cpp | 48 | ||||
-rw-r--r-- | src/qt/test/wallettests.cpp | 196 | ||||
-rw-r--r-- | src/qt/test/wallettests.h | 15 | ||||
-rw-r--r-- | src/qt/transactiondesc.cpp | 9 | ||||
-rw-r--r-- | src/qt/transactionrecord.cpp | 14 | ||||
-rw-r--r-- | src/qt/transactionrecord.h | 2 | ||||
-rw-r--r-- | src/qt/transactiontablemodel.cpp | 4 | ||||
-rw-r--r-- | src/qt/transactionview.cpp | 30 | ||||
-rw-r--r-- | src/qt/transactionview.h | 2 | ||||
-rw-r--r-- | src/qt/walletmodel.cpp | 147 | ||||
-rw-r--r-- | src/qt/walletmodel.h | 5 | ||||
-rw-r--r-- | src/random.cpp | 236 | ||||
-rw-r--r-- | src/random.h | 102 | ||||
-rw-r--r-- | src/rest.cpp | 40 | ||||
-rw-r--r-- | src/rpc/blockchain.cpp | 322 | ||||
-rw-r--r-- | src/rpc/blockchain.h | 40 | ||||
-rw-r--r-- | src/rpc/client.cpp | 19 | ||||
-rw-r--r-- | src/rpc/mining.cpp | 238 | ||||
-rw-r--r-- | src/rpc/misc.cpp | 253 | ||||
-rw-r--r-- | src/rpc/net.cpp | 85 | ||||
-rw-r--r-- | src/rpc/protocol.cpp | 20 | ||||
-rw-r--r-- | src/rpc/protocol.h | 11 | ||||
-rw-r--r-- | src/rpc/rawtransaction.cpp | 223 | ||||
-rw-r--r-- | src/rpc/server.cpp | 113 | ||||
-rw-r--r-- | src/rpc/server.h | 23 | ||||
-rw-r--r-- | src/scheduler.cpp | 22 | ||||
-rw-r--r-- | src/scheduler.h | 13 | ||||
-rw-r--r-- | src/script/interpreter.cpp | 46 | ||||
-rw-r--r-- | src/script/interpreter.h | 2 | ||||
-rw-r--r-- | src/script/ismine.cpp | 10 | ||||
-rw-r--r-- | src/script/script.cpp | 8 | ||||
-rw-r--r-- | src/script/script.h | 15 | ||||
-rw-r--r-- | src/script/sigcache.cpp | 22 | ||||
-rw-r--r-- | src/script/sigcache.h | 23 | ||||
-rw-r--r-- | src/script/sign.cpp | 28 | ||||
-rw-r--r-- | src/script/sign.h | 2 | ||||
-rw-r--r-- | src/script/standard.cpp | 24 | ||||
-rw-r--r-- | src/secp256k1/Makefile.am | 10 | ||||
-rw-r--r-- | src/secp256k1/configure.ac | 17 | ||||
-rw-r--r-- | src/secp256k1/include/secp256k1.h | 39 | ||||
-rw-r--r-- | src/secp256k1/src/bench.h | 2 | ||||
-rw-r--r-- | src/secp256k1/src/bench_schnorr_verify.c | 73 | ||||
-rw-r--r-- | src/secp256k1/src/ecdsa_impl.h | 16 | ||||
-rw-r--r-- | src/secp256k1/src/field_10x26_impl.h | 69 | ||||
-rw-r--r-- | src/secp256k1/src/field_5x52_impl.h | 91 | ||||
-rw-r--r-- | src/secp256k1/src/group_impl.h | 6 | ||||
-rw-r--r-- | src/secp256k1/src/modules/ecdh/main_impl.h | 2 | ||||
-rw-r--r-- | src/secp256k1/src/modules/ecdh/tests_impl.h | 30 | ||||
-rwxr-xr-x | src/secp256k1/src/modules/recovery/main_impl.h | 2 | ||||
-rw-r--r-- | src/secp256k1/src/modules/recovery/tests_impl.h | 143 | ||||
-rw-r--r-- | src/secp256k1/src/scalar_impl.h | 171 | ||||
-rw-r--r--[-rwxr-xr-x] | src/secp256k1/src/secp256k1.c | 31 | ||||
-rw-r--r-- | src/secp256k1/src/tests.c | 51 | ||||
-rw-r--r-- | src/secp256k1/src/tests_exhaustive.c | 143 | ||||
-rw-r--r-- | src/secp256k1/src/util.h | 5 | ||||
-rw-r--r-- | src/serialize.h | 11 | ||||
-rw-r--r-- | src/streams.h | 18 | ||||
-rw-r--r-- | src/support/cleanse.h | 1 | ||||
-rw-r--r-- | src/support/lockedpool.cpp | 4 | ||||
-rw-r--r-- | src/sync.cpp | 2 | ||||
-rw-r--r-- | src/sync.h | 1 | ||||
-rw-r--r-- | src/test/DoS_tests.cpp | 8 | ||||
-rw-r--r-- | src/test/README.md | 9 | ||||
-rw-r--r-- | src/test/addrman_tests.cpp | 212 | ||||
-rw-r--r-- | src/test/amount_tests.cpp | 39 | ||||
-rwxr-xr-x | src/test/bitcoin-util-test.py | 45 | ||||
-rw-r--r-- | src/test/blockencodings_tests.cpp | 8 | ||||
-rw-r--r-- | src/test/buildenv.py.in | 2 | ||||
-rw-r--r-- | src/test/checkqueue_tests.cpp | 442 | ||||
-rw-r--r-- | src/test/coins_tests.cpp | 563 | ||||
-rw-r--r-- | src/test/crypto_tests.cpp | 68 | ||||
-rw-r--r-- | src/test/cuckoocache_tests.cpp | 40 | ||||
-rw-r--r-- | src/test/data/script_tests.json | 2 | ||||
-rw-r--r-- | src/test/dbwrapper_tests.cpp | 20 | ||||
-rw-r--r-- | src/test/hash_tests.cpp | 17 | ||||
-rw-r--r-- | src/test/key_tests.cpp | 28 | ||||
-rw-r--r-- | src/test/main_tests.cpp | 7 | ||||
-rw-r--r-- | src/test/mempool_tests.cpp | 51 | ||||
-rw-r--r-- | src/test/merkle_tests.cpp | 2 | ||||
-rw-r--r-- | src/test/miner_tests.cpp | 55 | ||||
-rw-r--r-- | src/test/net_tests.cpp | 17 | ||||
-rw-r--r-- | src/test/policyestimator_tests.cpp | 80 | ||||
-rw-r--r-- | src/test/pow_tests.cpp | 32 | ||||
-rw-r--r-- | src/test/prevector_tests.cpp | 8 | ||||
-rw-r--r-- | src/test/random_tests.cpp | 53 | ||||
-rw-r--r-- | src/test/rpc_tests.cpp | 2 | ||||
-rw-r--r-- | src/test/script_P2SH_tests.cpp | 6 | ||||
-rw-r--r-- | src/test/script_tests.cpp | 33 | ||||
-rw-r--r-- | src/test/sighash_tests.cpp | 2 | ||||
-rw-r--r-- | src/test/sigopcount_tests.cpp | 4 | ||||
-rw-r--r-- | src/test/test_bitcoin.cpp | 50 | ||||
-rw-r--r-- | src/test/test_bitcoin.h | 13 | ||||
-rw-r--r-- | src/test/test_bitcoin_fuzzy.cpp | 27 | ||||
-rw-r--r-- | src/test/test_bitcoin_main.cpp | 26 | ||||
-rw-r--r-- | src/test/test_random.h | 8 | ||||
-rw-r--r-- | src/test/testutil.cpp | 6 | ||||
-rw-r--r-- | src/test/testutil.h | 4 | ||||
-rw-r--r-- | src/test/torcontrol_tests.cpp | 199 | ||||
-rw-r--r-- | src/test/transaction_tests.cpp | 28 | ||||
-rw-r--r-- | src/test/util_tests.cpp | 81 | ||||
-rw-r--r-- | src/test/versionbits_tests.cpp | 6 | ||||
-rw-r--r-- | src/timedata.cpp | 17 | ||||
-rw-r--r-- | src/timedata.h | 14 | ||||
-rw-r--r-- | src/tinyformat.h | 9 | ||||
-rw-r--r-- | src/torcontrol.cpp | 168 | ||||
-rw-r--r-- | src/txdb.cpp | 177 | ||||
-rw-r--r-- | src/txdb.h | 30 | ||||
-rw-r--r-- | src/txmempool.cpp | 192 | ||||
-rw-r--r-- | src/txmempool.h | 222 | ||||
-rw-r--r-- | src/undo.h | 84 | ||||
-rw-r--r-- | src/util.cpp | 277 | ||||
-rw-r--r-- | src/util.h | 155 | ||||
-rw-r--r-- | src/utilmoneystr.cpp | 8 | ||||
-rw-r--r-- | src/utilstrencodings.cpp | 53 | ||||
-rw-r--r-- | src/utilstrencodings.h | 3 | ||||
-rw-r--r-- | src/utiltime.cpp | 24 | ||||
-rw-r--r-- | src/utiltime.h | 2 | ||||
-rw-r--r-- | src/validation.cpp | 986 | ||||
-rw-r--r-- | src/validation.h | 91 | ||||
-rw-r--r-- | src/validationinterface.cpp | 18 | ||||
-rw-r--r-- | src/validationinterface.h | 45 | ||||
-rw-r--r-- | src/versionbits.cpp | 40 | ||||
-rw-r--r-- | src/versionbits.h | 14 | ||||
-rw-r--r-- | src/wallet/coincontrol.h | 8 | ||||
-rw-r--r-- | src/wallet/db.cpp | 339 | ||||
-rw-r--r-- | src/wallet/db.h | 119 | ||||
-rw-r--r-- | src/wallet/feebumper.cpp | 288 | ||||
-rw-r--r-- | src/wallet/feebumper.h | 59 | ||||
-rw-r--r-- | src/wallet/rpcdump.cpp | 391 | ||||
-rw-r--r-- | src/wallet/rpcwallet.cpp | 1209 | ||||
-rw-r--r-- | src/wallet/rpcwallet.h | 13 | ||||
-rw-r--r-- | src/wallet/test/wallet_test_fixture.cpp | 3 | ||||
-rw-r--r-- | src/wallet/test/wallet_tests.cpp | 350 | ||||
-rw-r--r-- | src/wallet/wallet.cpp | 1367 | ||||
-rw-r--r-- | src/wallet/wallet.h | 250 | ||||
-rw-r--r-- | src/wallet/walletdb.cpp | 360 | ||||
-rw-r--r-- | src/wallet/walletdb.h | 70 | ||||
-rw-r--r-- | src/zmq/zmqnotificationinterface.cpp | 34 | ||||
-rw-r--r-- | src/zmq/zmqnotificationinterface.h | 7 | ||||
-rw-r--r-- | src/zmq/zmqpublishnotifier.cpp | 16 | ||||
-rw-r--r-- | test/README.md | 99 | ||||
-rw-r--r-- | test/functional/.gitignore (renamed from qa/rpc-tests/.gitignore) | 0 | ||||
-rw-r--r-- | test/functional/README.md (renamed from qa/rpc-tests/README.md) | 2 | ||||
-rwxr-xr-x | test/functional/abandonconflict.py (renamed from qa/rpc-tests/abandonconflict.py) | 35 | ||||
-rwxr-xr-x | test/functional/assumevalid.py (renamed from qa/rpc-tests/assumevalid.py) | 114 | ||||
-rwxr-xr-x | test/functional/bip65-cltv-p2p.py (renamed from qa/rpc-tests/bip65-cltv-p2p.py) | 7 | ||||
-rwxr-xr-x | test/functional/bip65-cltv.py (renamed from qa/rpc-tests/bip65-cltv.py) | 15 | ||||
-rwxr-xr-x | test/functional/bip68-112-113-p2p.py (renamed from qa/rpc-tests/bip68-112-113-p2p.py) | 7 | ||||
-rwxr-xr-x | test/functional/bip68-sequence.py (renamed from qa/rpc-tests/bip68-sequence.py) | 94 | ||||
-rwxr-xr-x | test/functional/bip9-softforks.py (renamed from qa/rpc-tests/bip9-softforks.py) | 70 | ||||
-rwxr-xr-x | test/functional/bipdersig-p2p.py (renamed from qa/rpc-tests/bipdersig-p2p.py) | 6 | ||||
-rwxr-xr-x | test/functional/bipdersig.py (renamed from qa/rpc-tests/bipdersig.py) | 15 | ||||
-rwxr-xr-x | test/functional/blockchain.py | 133 | ||||
-rwxr-xr-x | test/functional/bumpfee.py (renamed from qa/rpc-tests/bumpfee.py) | 141 | ||||
-rwxr-xr-x | test/functional/combine_logs.py | 114 | ||||
-rw-r--r-- | test/functional/combined_log_template.html | 40 | ||||
-rw-r--r-- | test/functional/config.ini.in (renamed from qa/pull-tester/tests_config.ini.in) | 2 | ||||
-rwxr-xr-x | test/functional/create_cache.py (renamed from qa/rpc-tests/create_cache.py) | 2 | ||||
-rwxr-xr-x | test/functional/decodescript.py (renamed from qa/rpc-tests/decodescript.py) | 4 | ||||
-rwxr-xr-x | test/functional/disablewallet.py (renamed from qa/rpc-tests/disablewallet.py) | 25 | ||||
-rwxr-xr-x | test/functional/disconnect_ban.py | 108 | ||||
-rwxr-xr-x | test/functional/forknotify.py (renamed from qa/rpc-tests/forknotify.py) | 22 | ||||
-rwxr-xr-x | test/functional/fundrawtransaction.py (renamed from qa/rpc-tests/fundrawtransaction.py) | 110 | ||||
-rwxr-xr-x | test/functional/getblocktemplate_longpoll.py (renamed from qa/rpc-tests/getblocktemplate_longpoll.py) | 6 | ||||
-rwxr-xr-x | test/functional/getblocktemplate_proposals.py (renamed from qa/rpc-tests/getblocktemplate_proposals.py) | 8 | ||||
-rwxr-xr-x | test/functional/getchaintips.py (renamed from qa/rpc-tests/getchaintips.py) | 2 | ||||
-rwxr-xr-x | test/functional/httpbasics.py (renamed from qa/rpc-tests/httpbasics.py) | 2 | ||||
-rwxr-xr-x | test/functional/import-rescan.py (renamed from qa/rpc-tests/import-rescan.py) | 23 | ||||
-rwxr-xr-x | test/functional/importmulti.py (renamed from qa/rpc-tests/importmulti.py) | 72 | ||||
-rwxr-xr-x | test/functional/importprunedfunds.py (renamed from qa/rpc-tests/importprunedfunds.py) | 22 | ||||
-rwxr-xr-x | test/functional/invalidateblock.py (renamed from qa/rpc-tests/invalidateblock.py) | 39 | ||||
-rwxr-xr-x | test/functional/invalidblockrequest.py (renamed from qa/rpc-tests/invalidblockrequest.py) | 0 | ||||
-rwxr-xr-x | test/functional/invalidtxrequest.py (renamed from qa/rpc-tests/invalidtxrequest.py) | 0 | ||||
-rwxr-xr-x | test/functional/keypool.py (renamed from qa/rpc-tests/keypool.py) | 64 | ||||
-rwxr-xr-x | test/functional/listsinceblock.py (renamed from qa/rpc-tests/listsinceblock.py) | 6 | ||||
-rwxr-xr-x | test/functional/listtransactions.py (renamed from qa/rpc-tests/listtransactions.py) | 6 | ||||
-rwxr-xr-x | test/functional/maxuploadtarget.py (renamed from qa/rpc-tests/maxuploadtarget.py) | 109 | ||||
-rwxr-xr-x | test/functional/mempool_limit.py (renamed from qa/rpc-tests/mempool_limit.py) | 19 | ||||
-rwxr-xr-x | test/functional/mempool_packages.py (renamed from qa/rpc-tests/mempool_packages.py) | 54 | ||||
-rwxr-xr-x | test/functional/mempool_persist.py | 90 | ||||
-rwxr-xr-x | test/functional/mempool_reorg.py (renamed from qa/rpc-tests/mempool_reorg.py) | 26 | ||||
-rwxr-xr-x | test/functional/mempool_resurrect_test.py (renamed from qa/rpc-tests/mempool_resurrect_test.py) | 7 | ||||
-rwxr-xr-x | test/functional/mempool_spendcoinbase.py (renamed from qa/rpc-tests/mempool_spendcoinbase.py) | 10 | ||||
-rwxr-xr-x | test/functional/merkle_blocks.py (renamed from qa/rpc-tests/merkle_blocks.py) | 13 | ||||
-rwxr-xr-x | test/functional/multi_rpc.py (renamed from qa/rpc-tests/multi_rpc.py) | 5 | ||||
-rwxr-xr-x | test/functional/net.py | 100 | ||||
-rwxr-xr-x | test/functional/nulldummy.py (renamed from qa/rpc-tests/nulldummy.py) | 47 | ||||
-rwxr-xr-x | test/functional/p2p-acceptblock.py (renamed from qa/rpc-tests/p2p-acceptblock.py) | 107 | ||||
-rwxr-xr-x | test/functional/p2p-compactblocks.py (renamed from qa/rpc-tests/p2p-compactblocks.py) | 233 | ||||
-rwxr-xr-x | test/functional/p2p-feefilter.py (renamed from qa/rpc-tests/p2p-feefilter.py) | 22 | ||||
-rwxr-xr-x | test/functional/p2p-fullblocktest.py (renamed from qa/rpc-tests/p2p-fullblocktest.py) | 4 | ||||
-rwxr-xr-x | test/functional/p2p-leaktests.py (renamed from qa/rpc-tests/p2p-leaktests.py) | 28 | ||||
-rwxr-xr-x | test/functional/p2p-mempool.py | 39 | ||||
-rwxr-xr-x | test/functional/p2p-segwit.py (renamed from qa/rpc-tests/p2p-segwit.py) | 206 | ||||
-rwxr-xr-x | test/functional/p2p-timeouts.py (renamed from qa/rpc-tests/p2p-timeouts.py) | 24 | ||||
-rwxr-xr-x | test/functional/p2p-versionbits-warning.py (renamed from qa/rpc-tests/p2p-versionbits-warning.py) | 46 | ||||
-rwxr-xr-x | test/functional/preciousblock.py (renamed from qa/rpc-tests/preciousblock.py) | 35 | ||||
-rwxr-xr-x | test/functional/prioritise_transaction.py (renamed from qa/rpc-tests/prioritise_transaction.py) | 70 | ||||
-rwxr-xr-x | test/functional/proxy_test.py (renamed from qa/rpc-tests/proxy_test.py) | 15 | ||||
-rwxr-xr-x | test/functional/pruning.py (renamed from qa/rpc-tests/pruning.py) | 160 | ||||
-rwxr-xr-x | test/functional/rawtransactions.py (renamed from qa/rpc-tests/rawtransactions.py) | 36 | ||||
-rwxr-xr-x | test/functional/receivedby.py (renamed from qa/rpc-tests/receivedby.py) | 2 | ||||
-rwxr-xr-x | test/functional/reindex.py (renamed from qa/rpc-tests/reindex.py) | 17 | ||||
-rwxr-xr-x | test/functional/replace-by-fee.py (renamed from qa/rpc-tests/replace-by-fee.py) | 150 | ||||
-rwxr-xr-x | test/functional/rest.py (renamed from qa/rpc-tests/rest.py) | 10 | ||||
-rwxr-xr-x | test/functional/rpcbind_test.py (renamed from qa/rpc-tests/rpcbind_test.py) | 33 | ||||
-rwxr-xr-x | test/functional/rpcnamedargs.py (renamed from qa/rpc-tests/rpcnamedargs.py) | 6 | ||||
-rwxr-xr-x | test/functional/segwit.py (renamed from qa/rpc-tests/segwit.py) | 238 | ||||
-rwxr-xr-x | test/functional/sendheaders.py (renamed from qa/rpc-tests/sendheaders.py) | 142 | ||||
-rwxr-xr-x | test/functional/signmessages.py (renamed from qa/rpc-tests/signmessages.py) | 5 | ||||
-rwxr-xr-x | test/functional/signrawtransactions.py (renamed from qa/rpc-tests/signrawtransactions.py) | 31 | ||||
-rwxr-xr-x | test/functional/smartfees.py (renamed from qa/rpc-tests/smartfees.py) | 108 | ||||
-rw-r--r-- | test/functional/test_framework/__init__.py (renamed from qa/rpc-tests/test_framework/__init__.py) | 0 | ||||
-rw-r--r-- | test/functional/test_framework/address.py (renamed from qa/rpc-tests/test_framework/address.py) | 0 | ||||
-rw-r--r-- | test/functional/test_framework/authproxy.py (renamed from qa/rpc-tests/test_framework/authproxy.py) | 7 | ||||
-rw-r--r-- | test/functional/test_framework/bignum.py (renamed from qa/rpc-tests/test_framework/bignum.py) | 0 | ||||
-rw-r--r-- | test/functional/test_framework/blockstore.py (renamed from qa/rpc-tests/test_framework/blockstore.py) | 6 | ||||
-rw-r--r-- | test/functional/test_framework/blocktools.py (renamed from qa/rpc-tests/test_framework/blocktools.py) | 11 | ||||
-rwxr-xr-x | test/functional/test_framework/comptool.py (renamed from qa/rpc-tests/test_framework/comptool.py) | 25 | ||||
-rw-r--r-- | test/functional/test_framework/coverage.py (renamed from qa/rpc-tests/test_framework/coverage.py) | 0 | ||||
-rw-r--r-- | test/functional/test_framework/key.py (renamed from qa/rpc-tests/test_framework/key.py) | 0 | ||||
-rwxr-xr-x | test/functional/test_framework/mininode.py (renamed from qa/rpc-tests/test_framework/mininode.py) | 249 | ||||
-rw-r--r-- | test/functional/test_framework/netutil.py (renamed from qa/rpc-tests/test_framework/netutil.py) | 0 | ||||
-rw-r--r-- | test/functional/test_framework/script.py (renamed from qa/rpc-tests/test_framework/script.py) | 0 | ||||
-rw-r--r-- | test/functional/test_framework/siphash.py (renamed from qa/rpc-tests/test_framework/siphash.py) | 0 | ||||
-rw-r--r-- | test/functional/test_framework/socks5.py (renamed from qa/rpc-tests/test_framework/socks5.py) | 8 | ||||
-rwxr-xr-x | test/functional/test_framework/test_framework.py | 387 | ||||
-rw-r--r-- | test/functional/test_framework/util.py (renamed from qa/rpc-tests/test_framework/util.py) | 224 | ||||
-rwxr-xr-x | test/functional/test_runner.py (renamed from qa/pull-tester/rpc-tests.py) | 287 | ||||
-rwxr-xr-x | test/functional/txn_clone.py (renamed from qa/rpc-tests/txn_clone.py) | 4 | ||||
-rwxr-xr-x | test/functional/txn_doublespend.py (renamed from qa/rpc-tests/txn_doublespend.py) | 4 | ||||
-rwxr-xr-x | test/functional/wallet-accounts.py (renamed from qa/rpc-tests/wallet-accounts.py) | 71 | ||||
-rwxr-xr-x | test/functional/wallet-dump.py (renamed from qa/rpc-tests/wallet-dump.py) | 17 | ||||
-rwxr-xr-x | test/functional/wallet-hd.py (renamed from qa/rpc-tests/wallet-hd.py) | 41 | ||||
-rwxr-xr-x | test/functional/wallet.py (renamed from qa/rpc-tests/wallet.py) | 81 | ||||
-rwxr-xr-x | test/functional/walletbackup.py (renamed from qa/rpc-tests/walletbackup.py) | 32 | ||||
-rwxr-xr-x | test/functional/zapwallettxes.py (renamed from qa/rpc-tests/zapwallettxes.py) | 20 | ||||
-rwxr-xr-x | test/functional/zmq_test.py (renamed from qa/rpc-tests/zmq_test.py) | 27 | ||||
-rw-r--r-- | test/util/bctest.py (renamed from src/test/bctest.py) | 30 | ||||
-rwxr-xr-x | test/util/bitcoin-util-test.py | 36 | ||||
-rw-r--r-- | test/util/buildenv.py.in | 4 | ||||
-rw-r--r-- | test/util/data/bitcoin-util-test.json (renamed from src/test/data/bitcoin-util-test.json) | 36 | ||||
-rw-r--r-- | test/util/data/blanktxv1.hex (renamed from src/test/data/blanktxv1.hex) | 0 | ||||
-rw-r--r-- | test/util/data/blanktxv1.json (renamed from src/test/data/blanktxv1.json) | 2 | ||||
-rw-r--r-- | test/util/data/blanktxv2.hex (renamed from src/test/data/blanktxv2.hex) | 0 | ||||
-rw-r--r-- | test/util/data/blanktxv2.json (renamed from src/test/data/blanktxv2.json) | 2 | ||||
-rw-r--r-- | test/util/data/tt-delin1-out.hex (renamed from src/test/data/tt-delin1-out.hex) | 0 | ||||
-rw-r--r-- | test/util/data/tt-delin1-out.json (renamed from src/test/data/tt-delin1-out.json) | 2 | ||||
-rw-r--r-- | test/util/data/tt-delout1-out.hex (renamed from src/test/data/tt-delout1-out.hex) | 0 | ||||
-rw-r--r-- | test/util/data/tt-delout1-out.json (renamed from src/test/data/tt-delout1-out.json) | 2 | ||||
-rw-r--r-- | test/util/data/tt-locktime317000-out.hex (renamed from src/test/data/tt-locktime317000-out.hex) | 0 | ||||
-rw-r--r-- | test/util/data/tt-locktime317000-out.json (renamed from src/test/data/tt-locktime317000-out.json) | 2 | ||||
-rw-r--r-- | test/util/data/tx394b54bb.hex (renamed from src/test/data/tx394b54bb.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreate1.hex (renamed from src/test/data/txcreate1.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreate1.json (renamed from src/test/data/txcreate1.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreate2.hex (renamed from src/test/data/txcreate2.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreate2.json (renamed from src/test/data/txcreate2.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatedata1.hex (renamed from src/test/data/txcreatedata1.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatedata1.json (renamed from src/test/data/txcreatedata1.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatedata2.hex (renamed from src/test/data/txcreatedata2.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatedata2.json (renamed from src/test/data/txcreatedata2.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatedata_seq0.hex (renamed from src/test/data/txcreatedata_seq0.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatedata_seq0.json (renamed from src/test/data/txcreatedata_seq0.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatedata_seq1.hex (renamed from src/test/data/txcreatedata_seq1.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatedata_seq1.json (renamed from src/test/data/txcreatedata_seq1.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatemultisig1.hex (renamed from src/test/data/txcreatemultisig1.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatemultisig1.json (renamed from src/test/data/txcreatemultisig1.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatemultisig2.hex (renamed from src/test/data/txcreatemultisig2.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatemultisig2.json (renamed from src/test/data/txcreatemultisig2.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatemultisig3.hex (renamed from src/test/data/txcreatemultisig3.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatemultisig3.json (renamed from src/test/data/txcreatemultisig3.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatemultisig4.hex (renamed from src/test/data/txcreatemultisig4.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatemultisig4.json (renamed from src/test/data/txcreatemultisig4.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreateoutpubkey1.hex (renamed from src/test/data/txcreateoutpubkey1.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreateoutpubkey1.json (renamed from src/test/data/txcreateoutpubkey1.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreateoutpubkey2.hex (renamed from src/test/data/txcreateoutpubkey2.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreateoutpubkey2.json (renamed from src/test/data/txcreateoutpubkey2.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreateoutpubkey3.hex (renamed from src/test/data/txcreateoutpubkey3.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreateoutpubkey3.json (renamed from src/test/data/txcreateoutpubkey3.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatescript1.hex (renamed from src/test/data/txcreatescript1.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatescript1.json (renamed from src/test/data/txcreatescript1.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatescript2.hex (renamed from src/test/data/txcreatescript2.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatescript2.json (renamed from src/test/data/txcreatescript2.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatescript3.hex (renamed from src/test/data/txcreatescript3.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatescript3.json (renamed from src/test/data/txcreatescript3.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatescript4.hex (renamed from src/test/data/txcreatescript4.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatescript4.json (renamed from src/test/data/txcreatescript4.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatesignv1.hex (renamed from src/test/data/txcreatesignv1.hex) | 0 | ||||
-rw-r--r-- | test/util/data/txcreatesignv1.json (renamed from src/test/data/txcreatesignv1.json) | 2 | ||||
-rw-r--r-- | test/util/data/txcreatesignv2.hex (renamed from src/test/data/txcreatesignv2.hex) | 0 |
457 files changed, 16775 insertions, 10281 deletions
diff --git a/.gitignore b/.gitignore index 479889cb78..f1e9ca20c1 100644 --- a/.gitignore +++ b/.gitignore @@ -101,9 +101,9 @@ coverage_percent.txt linux-coverage-build linux-build win32-build -qa/pull-tester/tests_config.py -qa/pull-tester/tests_config.ini -qa/cache/* +test/functional/config.ini +test/util/buildenv.py +test/cache/* !src/leveldb*/Makefile diff --git a/.travis.yml b/.travis.yml index ce6cdc2db0..fcf4b082f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,11 +39,11 @@ env: before_install: - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") install: - - if [ -n "$PPA" ]; then travis_retry sudo add-apt-repository "$PPA" -y; fi - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi before_script: + - if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then contrib/devtools/commit-script-check.sh $TRAVIS_COMMIT_RANGE; fi - unset CC; unset CXX - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi - mkdir -p depends/SDKs depends/sdk-sources @@ -51,7 +51,7 @@ before_script: - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS script: - - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi + - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then git fetch --unshallow; fi - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then contrib/verify-commits/verify-commits.sh; fi - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` @@ -67,8 +67,9 @@ script: - ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS check VERBOSE=1; fi - - if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage; fi + - if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi + - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude pruning"; fi + - if [ "$RUN_TESTS" = "true" ]; then test/functional/test_runner.py --coverage --quiet ${extended}; fi after_script: - echo $TRAVIS_COMMIT_RANGE - echo $TRAVIS_COMMIT_LOG diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc8d58d97d..f5d63517b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,8 +37,8 @@ fixes or code moves with actual code changes. Commit messages should be verbose by default consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate -paragraph(s); unless the title alone is self-explanatory (like "Corrected typo -in init.cpp") then a single title line is sufficient. Commit messages should be +paragraph(s), unless the title alone is self-explanatory (like "Corrected typo +in init.cpp") in which case a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/). @@ -130,7 +130,7 @@ pull requests which attempt to do too much, are overly large, or overly complex as this makes review difficult. -###Features +### Features When adding a new feature, thought must be given to the long term technical debt and maintenance that feature may require after inclusion. Before proposing a new @@ -139,7 +139,7 @@ maintain it (including bug fixing). If features get orphaned with no maintainer in the future, they may be removed by the Repository Maintainer. -###Refactoring +### Refactoring Refactoring is a necessary part of any software project's evolution. The following guidelines cover refactoring pull requests for the project. @@ -187,7 +187,7 @@ other kinds of patches because of increased peer review and consensus building requirements. -###Peer Review +### Peer Review Anyone may participate in peer review which is expressed by comments in the pull request. Typically reviewers will review the code for obvious errors, as well as @@ -225,6 +225,37 @@ discussed extensively on the mailing list and IRC, be accompanied by a widely discussed BIP and have a generally widely perceived technical consensus of being a worthwhile change based on the judgement of the maintainers. +### Finding Reviewers + +As most reviewers are themselves developers with their own projects, the review +process can be quite lengthy, and some amount of patience is required. If you find +that you've been waiting for a pull request to be given attention for several +months, there may be a number of reasons for this, some of which you can do something +about: + + - It may be because of a feature freeze due to an upcoming release. During this time, + only bug fixes are taken into consideration. If your pull request is a new feature, + it will not be prioritized until the release is over. Wait for release. + - It may be because the changes you are suggesting do not appeal to people. Rather than + nits and critique, which require effort and means they care enough to spend time on your + contribution, thundering silence is a good sign of widespread (mild) dislike of a given change + (because people don't assume *others* won't actually like the proposal). Don't take + that personally, though! Instead, take another critical look at what you are suggesting + and see if it: changes too much, is too broad, doesn't adhere to the + [developer notes](doc/developer-notes.md), is dangerous or insecure, is messily written, etc. + Identify and address any of the issues you find. Then ask e.g. on IRC if someone could give + their opinion on the concept itself. + - It may be because your code is too complex for all but a few people. And those people + may not have realized your pull request even exists. A great way to find people who + are qualified and care about the code you are touching is the + [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply + find the person touching the code you are touching before you and see if you can find + them and give them a nudge. Don't be incessant about the nudging though. + - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request + a look. If you think you've been waiting an unreasonably long amount of time (month+) for + no particular reason (few lines changed, etc), this is totally fine. Try to return the favor + when someone else is asking for feedback on their code, and universe balances out. + Release Policy -------------- diff --git a/Makefile.am b/Makefile.am index 1ee2dfb734..3a56eea0c0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -61,7 +61,7 @@ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \ COVERAGE_INFO = baseline_filtered_combined.info baseline.info \ leveldb_baseline.info test_bitcoin_filtered.info total_coverage.info \ - baseline_filtered.info rpc_test.info rpc_test_filtered.info \ + baseline_filtered.info functional_test.info functional_test_filtered.info \ leveldb_baseline_filtered.info test_bitcoin_coverage.info test_bitcoin.info dist-hook: @@ -76,9 +76,6 @@ $(BITCOIN_WIN_INSTALLER): all-recursive echo error: could not build $@ @echo built $@ -$(if $(findstring src/,$(MAKECMDGOALS)),$(MAKECMDGOALS), none): FORCE - $(MAKE) -C src $(patsubst src/%,%,$@) - $(OSX_APP)/Contents/PkgInfo: $(MKDIR_P) $(@D) @echo "APPL????" > $@ @@ -194,20 +191,20 @@ test_bitcoin.info: baseline_filtered_combined.info test_bitcoin_filtered.info: test_bitcoin.info $(LCOV) -r $< "/usr/include/*" -o $@ -rpc_test.info: test_bitcoin_filtered.info - -@TIMEOUT=15 python qa/pull-tester/rpc-tests.py $(EXTENDED_RPC_TESTS) - $(LCOV) -c -d $(abs_builddir)/src --t rpc-tests -o $@ +functional_test.info: test_bitcoin_filtered.info + -@TIMEOUT=15 python test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS) + $(LCOV) -c -d $(abs_builddir)/src --t functional-tests -o $@ $(LCOV) -z -d $(abs_builddir)/src $(LCOV) -z -d $(abs_builddir)/src/leveldb -rpc_test_filtered.info: rpc_test.info +functional_test_filtered.info: functional_test.info $(LCOV) -r $< "/usr/include/*" -o $@ test_bitcoin_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -o $@ -total_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info rpc_test_filtered.info - $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a rpc_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt +total_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info functional_test_filtered.info + $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt test_bitcoin.coverage/.dirstamp: test_bitcoin_coverage.info $(GENHTML) -s $< -o $(@D) @@ -223,14 +220,70 @@ endif dist_noinst_SCRIPTS = autogen.sh -EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc-tests $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) +EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) + +EXTRA_DIST += \ + test/util/bctest.py \ + test/util/bitcoin-util-test.py \ + test/util/data/bitcoin-util-test.json \ + test/util/data/blanktxv1.hex \ + test/util/data/blanktxv1.json \ + test/util/data/blanktxv2.hex \ + test/util/data/blanktxv2.json \ + test/util/data/tt-delin1-out.hex \ + test/util/data/tt-delin1-out.json \ + test/util/data/tt-delout1-out.hex \ + test/util/data/tt-delout1-out.json \ + test/util/data/tt-locktime317000-out.hex \ + test/util/data/tt-locktime317000-out.json \ + test/util/data/tx394b54bb.hex \ + test/util/data/txcreate1.hex \ + test/util/data/txcreate1.json \ + test/util/data/txcreate2.hex \ + test/util/data/txcreate2.json \ + test/util/data/txcreatedata1.hex \ + test/util/data/txcreatedata1.json \ + test/util/data/txcreatedata2.hex \ + test/util/data/txcreatedata2.json \ + test/util/data/txcreatedata_seq0.hex \ + test/util/data/txcreatedata_seq0.json \ + test/util/data/txcreatedata_seq1.hex \ + test/util/data/txcreatedata_seq1.json \ + test/util/data/txcreatemultisig1.hex \ + test/util/data/txcreatemultisig1.json \ + test/util/data/txcreatemultisig2.hex \ + test/util/data/txcreatemultisig2.json \ + test/util/data/txcreatemultisig3.hex \ + test/util/data/txcreatemultisig3.json \ + test/util/data/txcreatemultisig4.hex \ + test/util/data/txcreatemultisig4.json \ + test/util/data/txcreateoutpubkey1.hex \ + test/util/data/txcreateoutpubkey1.json \ + test/util/data/txcreateoutpubkey2.hex \ + test/util/data/txcreateoutpubkey2.json \ + test/util/data/txcreateoutpubkey3.hex \ + test/util/data/txcreateoutpubkey3.json \ + test/util/data/txcreatescript1.hex \ + test/util/data/txcreatescript1.json \ + test/util/data/txcreatescript2.hex \ + test/util/data/txcreatescript2.json \ + test/util/data/txcreatescript3.hex \ + test/util/data/txcreatescript3.json \ + test/util/data/txcreatescript4.hex \ + test/util/data/txcreatescript4.json \ + test/util/data/txcreatesignv1.hex \ + test/util/data/txcreatesignv1.json \ + test/util/data/txcreatesignv2.hex CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) +# This file is problematic for out-of-tree builds if it exists. +DISTCLEANFILES = test/util/buildenv.pyc + .INTERMEDIATE: $(COVERAGE_INFO) DISTCHECK_CONFIGURE_FLAGS = --enable-man clean-local: - rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ qa/tmp/ cache/ $(OSX_APP) - rm -rf qa/pull-tester/__pycache__ + rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP) + rm -rf test/functional/__pycache__ @@ -54,9 +54,9 @@ submit new unit tests for old code. Unit tests can be compiled and run (assuming they weren't disabled in configure) with: `make check`. Further details on running and extending unit tests can be found in [/src/test/README.md](/src/test/README.md). -There are also [regression and integration tests](/qa) of the RPC interface, written +There are also [regression and integration tests](/test), written in Python, that are run automatically on the build server. -These tests can be run (if the [test dependencies](/qa) are installed) with: `qa/pull-tester/rpc-tests.py` +These tests can be run (if the [test dependencies](/test) are installed) with: `test/functional/test_runner.py` The Travis CI system makes sure that every pull request is built for Windows, Linux, and OS X, and that unit/sanity tests are run automatically. diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index f43fc3037f..1b339559fc 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -130,6 +130,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ if test "x$bitcoin_cv_need_acc_widget" = "xyes"; then _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(AccessibleFactory)], [-lqtaccessiblewidgets]) fi + _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) + AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) if test x$TARGET_OS = xwindows; then _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QWindowsIntegrationPlugin)],[-lqwindows]) AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows]) diff --git a/configure.ac b/configure.ac index 5a09befea2..ca66216554 100644 --- a/configure.ac +++ b/configure.ac @@ -124,10 +124,10 @@ AC_ARG_ENABLE(bench, [use_bench=$enableval], [use_bench=yes]) -AC_ARG_ENABLE([extended-rpc-tests], - AS_HELP_STRING([--enable-extended-rpc-tests],[enable expensive RPC tests when using lcov (default no)]), - [use_extended_rpc_tests=$enableval], - [use_extended_rpc_tests=no]) +AC_ARG_ENABLE([extended-functional-tests], + AS_HELP_STRING([--enable-extended-functional-tests],[enable expensive functional tests when using lcov (default no)]), + [use_extended_functional_tests=$enableval], + [use_extended_functional_tests=no]) AC_ARG_WITH([qrencode], [AS_HELP_STRING([--with-qrencode], @@ -235,6 +235,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wself-assign],[CXXFLAGS="$CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[CXXFLAGS="$CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdeprecated-register],[CXXFLAGS="$CXXFLAGS -Wno-deprecated-register"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[CXXFLAGS="$CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) fi CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" @@ -413,8 +414,8 @@ if test x$use_pkgconfig = xyes; then ]) fi -if test x$use_extended_rpc_tests != xno; then - AC_SUBST(EXTENDED_RPC_TESTS, -extended) +if test x$use_extended_functional_tests != xno; then + AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi if test x$use_lcov = xyes; then @@ -549,6 +550,8 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, #include <byteswap.h> #endif]) +AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll]) + dnl Check for MSG_NOSIGNAL AC_MSG_CHECKING(for MSG_NOSIGNAL) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/socket.h>]], @@ -557,6 +560,30 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/socket.h>]], [ AC_MSG_RESULT(no)] ) +dnl Check for MSG_DONTWAIT +AC_MSG_CHECKING(for MSG_DONTWAIT) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/socket.h>]], + [[ int f = MSG_DONTWAIT; ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MSG_DONTWAIT, 1,[Define this symbol if you have MSG_DONTWAIT]) ], + [ AC_MSG_RESULT(no)] +) + +dnl Check for malloc_info (for memory statistics information in getmemoryinfo) +AC_MSG_CHECKING(for getmemoryinfo) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], + [[ int f = malloc_info(0, NULL); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOC_INFO, 1,[Define this symbol if you have malloc_info]) ], + [ AC_MSG_RESULT(no)] +) + +dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas) +AC_MSG_CHECKING(for mallopt M_ARENA_MAX) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], + [[ mallopt(M_ARENA_MAX, 1); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOPT_ARENA_MAX, 1,[Define this symbol if you have mallopt with M_ARENA_MAX]) ], + [ AC_MSG_RESULT(no)] +) + AC_MSG_CHECKING([for visibility attribute]) AC_LINK_IFELSE([AC_LANG_SOURCE([ int foo_def( void ) __attribute__((visibility("default"))); @@ -574,6 +601,33 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ ] ) +# Check for different ways of gathering OS randomness +AC_MSG_CHECKING(for Linux getrandom syscall) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> + #include <sys/syscall.h> + #include <linux/random.h>]], + [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYS_GETRANDOM, 1,[Define this symbol if the Linux getrandom system call is available]) ], + [ AC_MSG_RESULT(no)] +) + +AC_MSG_CHECKING(for getentropy) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h>]], + [[ getentropy(nullptr, 32) ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY, 1,[Define this symbol if the BSD getentropy system call is available]) ], + [ AC_MSG_RESULT(no)] +) + +AC_MSG_CHECKING(for sysctl KERN_ARND) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> + #include <sys/sysctl.h>]], + [[ static const int name[2] = {CTL_KERN, KERN_ARND}; + sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL_ARND, 1,[Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], + [ AC_MSG_RESULT(no)] +) + +# Check for reduced exports if test x$use_reduce_exports = xyes; then AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[RE_CXXFLAGS="-fvisibility=hidden"], [AC_MSG_ERROR([Cannot set default symbol visibility. Use --disable-reduce-exports.])]) @@ -625,6 +679,10 @@ AX_BOOST_PROGRAM_OPTIONS AX_BOOST_THREAD AX_BOOST_CHRONO +dnl Boost 1.56 through 1.62 allow using std::atomic instead of its own atomic +dnl counter implementations. In 1.63 and later the std::atomic approach is default. +m4_pattern_allow(DBOOST_AC_USE_STD_ATOMIC) dnl otherwise it's treated like a macro +BOOST_CPPFLAGS="-DBOOST_SP_USE_STD_ATOMIC -DBOOST_AC_USE_STD_ATOMIC $BOOST_CPPFLAGS" if test x$use_reduce_exports = xyes; then AC_MSG_CHECKING([for working boost reduced exports]) @@ -1103,10 +1161,13 @@ AC_SUBST(EVENT_PTHREADS_LIBS) AC_SUBST(ZMQ_LIBS) AC_SUBST(PROTOBUF_LIBS) AC_SUBST(QR_LIBS) -AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py]) -AC_CONFIG_FILES([qa/pull-tester/tests_config.ini],[chmod +x qa/pull-tester/tests_config.ini]) +AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/functional/config.ini]) +AC_CONFIG_FILES([test/util/buildenv.py],[chmod +x test/util/buildenv.py]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) -AC_CONFIG_LINKS([qa/pull-tester/rpc-tests.py:qa/pull-tester/rpc-tests.py]) +AC_CONFIG_FILES([doc/Doxyfile]) +AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) +AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py]) +AC_CONFIG_LINKS([test/util/bctest.py:test/util/bctest.py]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. @@ -1154,8 +1215,8 @@ esac dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows case ${OS} in *Windows*) - sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' qa/pull-tester/tests_config.py > qa/pull-tester/tests_config-2.py - mv qa/pull-tester/tests_config-2.py qa/pull-tester/tests_config.py + sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/functional/config.ini > test/functional/config-2.ini + mv test/functional/config-2.ini test/functional/config.ini ;; esac diff --git a/contrib/debian/changelog b/contrib/debian/changelog index 110bfe03ef..33dab9b638 100644 --- a/contrib/debian/changelog +++ b/contrib/debian/changelog @@ -1,3 +1,122 @@ +bitcoin (0.14.1-trusty4) trusty; urgency=medium + + * Re-enable UPnP support. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Fri, 05 May 2017 13:28:00 -0400 + +bitcoin (0.14.1-trusty3) trusty; urgency=medium + + * Build with qt5 if we are on a non-Ubuntu (ie non-Unity) distro. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Thu, 04 May 2017 17:13:00 -0400 + +bitcoin (0.14.1-trusty2) trusty; urgency=medium + + * Bump minimum boost version in deps. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Thu, 04 May 2017 17:12:00 -0400 + +bitcoin (0.14.1-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Sat, 22 Apr 2017 17:10:00 -0400 + +bitcoin (0.14.0-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Wed, 08 Mar 2017 10:30:00 -0500 + +bitcoin (0.13.2-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Thu, 05 Jan 2017 09:59:00 -0500 + +bitcoin (0.13.1-trusty2) trusty; urgency=medium + + * Revert to Qt4, due to https://github.com/bitcoin/bitcoin/issues/9038 + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Mon, 31 Oct 2016 11:16:00 -0400 + +bitcoin (0.13.1-trusty1) trusty; urgency=medium + + * New upstream release. + * Backport updated bitcoin-qt.desktop from upstream master + * Add zmq dependency + * Switch to Qt5 (breaks precise, but that was already broken by C++11) + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Thu, 27 Oct 2016 17:32:00 -0400 + +bitcoin (0.13.0-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Sun, 04 Sep 2016 22:09:00 -0400 + +bitcoin (0.12.1-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Mon, 18 Apr 2016 14:26:00 -0700 + +bitcoin (0.12.0-trusty6) trusty; urgency=medium + + * Fix program-options dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Fri, 25 Mar 2016 21:41:00 -0700 + +bitcoin (0.12.0-trusty5) trusty; urgency=medium + + * Test explicit --with-gui + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 23:25:00 -0800 + +bitcoin (0.12.0-trusty4) trusty; urgency=medium + + * Fix libevent-dev dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 23:25:00 -0800 + +bitcoin (0.12.0-trusty3) trusty; urgency=medium + + * Fix precise boost dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 19:55:00 -0800 + +bitcoin (0.12.0-trusty2) trusty; urgency=medium + + * Fix libevent dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 19:53:00 -0800 + +bitcoin (0.12.0-trusty1) trusty; urgency=medium + + * New upstream release + * Various updates to contrib/debian were merged, a few were not + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 19:29:00 -0800 + +bitcoin (0.11.2-trusty1) trusty; urgency=low + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Fri, 13 Nov 2015 18:39:00 -0800 + +bitcoin (0.11.1-trusty2) trusty; urgency=low + + * Remove minupnpc builddep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Wed, 14 Oct 2015 23:06:00 -1000 + +bitcoin (0.11.1-trusty1) trusty; urgency=high + + * New upstream release. + * Disable all UPnP support. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Wed, 14 Oct 2015 13:57:00 -1000 + bitcoin (0.11.0-precise1) precise; urgency=medium * New upstream release. @@ -179,7 +298,7 @@ bitcoin (0.5.3-natty0) natty; urgency=low bitcoin (0.5.2-natty1) natty; urgency=low * Remove mentions on anonymity in package descriptions and manpage. - These should never have been there, bitcoin isn't anonymous without + These should never have been there, bitcoin isnt anonymous without a ton of work that virtually no users will ever be willing and capable of doing @@ -220,7 +339,7 @@ bitcoin (0.5.0~rc1-natty1) natty; urgency=low * Add test_bitcoin to build test * Fix clean - * Remove unnecessary build-dependancies + * Remove uneccessary build-dependancies -- Matt Corallo <matt@bluematt.me> Wed, 26 Oct 2011 14:37:18 -0400 @@ -380,7 +499,7 @@ bitcoin (0.3.20.01~dfsg-1) unstable; urgency=low bitcoin (0.3.19~dfsg-6) unstable; urgency=low - * Fix override aggressive optimizations. + * Fix override agressive optimizations. * Fix tighten build-dependencies to really fit backporting to Lenny: + Add fallback build-dependency on libdb4.6++-dev. + Tighten unversioned Boost build-dependencies to recent versions, diff --git a/contrib/debian/control b/contrib/debian/control index fce6bc0118..0d6ad25e24 100644 --- a/contrib/debian/control +++ b/contrib/debian/control @@ -1,27 +1,30 @@ Source: bitcoin Section: utils Priority: optional -Maintainer: Jonas Smedegaard <dr@jones.dk> -Uploaders: Micah Anderson <micah@debian.org> +Maintainer: Matt Corallo <matt@mattcorallo.com> +Uploaders: Matt Corallo <matt@mattcorallo.com> Build-Depends: debhelper, devscripts, automake, libtool, bash-completion, - libboost-system-dev (>> 1.35) | libboost-system1.35-dev, libdb4.8++-dev, libssl-dev, pkg-config, - libminiupnpc8-dev | libminiupnpc-dev (>> 1.6), - libboost-filesystem-dev (>> 1.35) | libboost-filesystem1.35-dev, - libboost-program-options-dev (>> 1.35) | libboost-program-options1.35-dev, - libboost-thread-dev (>> 1.35) | libboost-thread1.35-dev, - libboost-test-dev (>> 1.35) | libboost-test1.35-dev, - qt4-qmake, - libqt4-dev, + libevent-dev, + libboost-system1.48-dev | libboost-system-dev (>> 1.47), + libboost-filesystem1.48-dev | libboost-filesystem-dev (>> 1.47), + libboost-program-options1.48-dev | libboost-program-options-dev (>> 1.47), + libboost-thread1.48-dev | libboost-thread-dev (>> 1.47), + libboost-test1.48-dev | libboost-test-dev (>> 1.47), + libboost-chrono1.48-dev | libboost-chrono-dev (>> 1.47), + libminiupnpc8-dev | libminiupnpc-dev, + qt4-qmake, libqt4-dev, + qttools5-dev-tools, qttools5-dev, libqrencode-dev, libprotobuf-dev, protobuf-compiler, - python + python, + libzmq3-dev Standards-Version: 3.9.2 Homepage: https://bitcoincore.org/ Vcs-Git: git://github.com/bitcoin/bitcoin.git @@ -31,11 +34,11 @@ Package: bitcoind Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Description: peer-to-peer network based digital currency - daemon - Bitcoin is an experimental new digital currency that enables instant - payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer - technology to operate with no central authority: managing transactions - and issuing money are carried out collectively by the network. Bitcoin Core - is the name of the open source software which enables the use of this currency. + Bitcoin is a free open source peer-to-peer electronic cash system that + is completely decentralized, without the need for a central server or + trusted parties. Users hold the crypto keys to their own money and + transact directly with each other, with the help of a P2P network to + check for double-spending. . This package provides the daemon, bitcoind, and the CLI tool bitcoin-cli to interact with the daemon. @@ -44,11 +47,11 @@ Package: bitcoin-qt Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Description: peer-to-peer network based digital currency - Qt GUI - Bitcoin is an experimental new digital currency that enables instant - payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer - technology to operate with no central authority: managing transactions - and issuing money are carried out collectively by the network. Bitcoin Core - is the name of the open source software which enables the use of this currency. + Bitcoin is a free open source peer-to-peer electronic cash system that + is completely decentralized, without the need for a central server or + trusted parties. Users hold the crypto keys to their own money and + transact directly with each other, with the help of a P2P network to + check for double-spending. . This package provides Bitcoin-Qt, a GUI for Bitcoin based on Qt. @@ -56,11 +59,11 @@ Package: bitcoin-tx Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Description: peer-to-peer digital currency - standalone transaction tool - Bitcoin is an experimental new digital currency that enables instant - payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer - technology to operate with no central authority: managing transactions - and issuing money are carried out collectively by the network. Bitcoin Core - is the name of the open source software which enables the use of this currency. + Bitcoin is a free open source peer-to-peer electronic cash system that + is completely decentralized, without the need for a central server or + trusted parties. Users hold the crypto keys to their own money and + transact directly with each other, with the help of a P2P network to + check for double-spending. . This package provides bitcoin-tx, a command-line transaction creation tool which can be used without a bitcoin daemon. Some means of diff --git a/contrib/debian/examples/bitcoin.conf b/contrib/debian/examples/bitcoin.conf index afbc7882e0..1029a51073 100644 --- a/contrib/debian/examples/bitcoin.conf +++ b/contrib/debian/examples/bitcoin.conf @@ -116,12 +116,7 @@ # running on another host using this option: #rpcconnect=127.0.0.1 -# Transaction Fee Changes in 0.10.0 - -# Send transactions as zero-fee transactions if possible (default: 0) -#sendfreetransactions=0 - -# Create transactions that have enough fees (or priority) so they are likely to begin confirmation within n blocks (default: 1). +# Create transactions that have enough fees so they are likely to begin confirmation within n blocks (default: 6). # This setting is over-ridden by the -paytxfee option. #txconfirmtarget=n @@ -136,6 +131,13 @@ # be validated sooner. #paytxfee=0.00 +# Enable pruning to reduce storage requirements by deleting old blocks. +# This mode is incompatible with -txindex and -rescan. +# 0 = default (no pruning). +# 1 = allows manual pruning via RPC. +# >=550 = target to stay under in MiB. +#prune=550 + # User interface options # Start Bitcoin minimized diff --git a/contrib/debian/rules b/contrib/debian/rules index 3896d2caa3..6885e38521 100755 --- a/contrib/debian/rules +++ b/contrib/debian/rules @@ -12,10 +12,12 @@ override_dh_auto_clean: if [ -f Makefile ]; then $(MAKE) distclean; fi rm -rf Makefile.in aclocal.m4 configure src/Makefile.in src/bitcoin-config.h.in src/build-aux src/qt/Makefile.in src/qt/test/Makefile.in src/test/Makefile.in +QT=$(shell dpkg-vendor --derives-from Ubuntu && echo qt4 || echo qt5) + # Yea, autogen should be run on the source archive, but I like doing git archive override_dh_auto_configure: ./autogen.sh - ./configure + ./configure --with-gui=$(QT) override_dh_auto_test: make check diff --git a/contrib/devtools/check-doc.py b/contrib/devtools/check-doc.py index 249214e931..445175ec2b 100755 --- a/contrib/devtools/check-doc.py +++ b/contrib/devtools/check-doc.py @@ -21,7 +21,7 @@ CMD_GREP_DOCS = r"egrep -r -I 'HelpMessageOpt\(\"\-[^\"=]+?(=|\")' %s" % (CMD_RO REGEX_ARG = re.compile(r'(?:map(?:Multi)?Args(?:\.count\(|\[)|Get(?:Bool)?Arg\()\"(\-[^\"]+?)\"') REGEX_DOC = re.compile(r'HelpMessageOpt\(\"(\-[^\"=]+?)(?:=|\")') # list unsupported, deprecated and duplicate args as they need no documentation -SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize']) +SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize', '-sendfreetransactions']) def main(): used = check_output(CMD_GREP_ARGS, shell=True) diff --git a/contrib/devtools/commit-script-check.sh b/contrib/devtools/commit-script-check.sh new file mode 100755 index 0000000000..add4bb4883 --- /dev/null +++ b/contrib/devtools/commit-script-check.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# This simple script checks for commits beginning with: scripted-diff: +# If found, looks for a script between the lines -BEGIN VERIFY SCRIPT- and +# -END VERIFY SCRIPT-. If no ending is found, it reads until the end of the +# commit message. + +# The resulting script should exactly transform the previous commit into the current +# one. Any remaining diff signals an error. + +if test "x$1" = "x"; then + echo "Usage: $0 <commit>..." + exit 1 +fi + +RET=0 +PREV_BRANCH=`git name-rev --name-only HEAD` +PREV_HEAD=`git rev-parse HEAD` +for i in `git rev-list --reverse $1`; do + git rev-list -n 1 --pretty="%s" $i | grep -q "^scripted-diff:" || continue + git checkout --quiet $i^ || exit + SCRIPT="`git rev-list --format=%b -n1 $i | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d'`" + if test "x$SCRIPT" = "x"; then + echo "Error: missing script for: $i" + echo "Failed" + RET=1 + else + echo "Running script for: $i" + echo "$SCRIPT" + eval "$SCRIPT" + git --no-pager diff --exit-code $i && echo "OK" || (echo "Failed"; false) || RET=1 + fi + git reset --quiet --hard HEAD +done +git checkout --quiet $PREV_BRANCH 2>/dev/null || git checkout --quiet $PREV_HEAD +exit $RET diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 9f35c378bf..6d801d3ac7 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -32,7 +32,7 @@ EXCLUDE = [ 'src/tinyformat.h', 'src/leveldb/util/env_win.cc', 'src/crypto/ctaes/bench.c', - 'qa/rpc-tests/test_framework/bignum.py', + 'test/functional/test_framework/bignum.py', # python init: '*__init__.py', ] diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index bb6ffb0253..e9816f7d19 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 Bitcoin Core Developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -70,26 +70,66 @@ def ask_prompt(text): print("",file=stderr) return reply -def tree_sha512sum(): - files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', '--name-only', 'HEAD']).splitlines()) +def get_symlink_files(): + files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', 'HEAD']).splitlines()) + ret = [] + for f in files: + if (int(f.decode('utf-8').split(" ")[0], 8) & 0o170000) == 0o120000: + ret.append(f.decode('utf-8').split("\t")[1]) + return ret + +def tree_sha512sum(commit='HEAD'): + # request metadata for entire tree, recursively + files = [] + blob_by_name = {} + for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines(): + name_sep = line.index(b'\t') + metadata = line[:name_sep].split() # perms, 'blob', blobid + assert(metadata[1] == b'blob') + name = line[name_sep+1:] + files.append(name) + blob_by_name[name] = metadata[2] + + files.sort() + # open connection to git-cat-file in batch mode to request data for all blobs + # this is much faster than launching it per file + p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) overall = hashlib.sha512() for f in files: + blob = blob_by_name[f] + # request blob + p.stdin.write(blob + b'\n') + p.stdin.flush() + # read header: blob, "blob", size + reply = p.stdout.readline().split() + assert(reply[0] == blob and reply[1] == b'blob') + size = int(reply[2]) + # hash the blob data intern = hashlib.sha512() - fi = open(f, 'rb') - while True: - piece = fi.read(65536) - if piece: + ptr = 0 + while ptr < size: + bs = min(65536, size - ptr) + piece = p.stdout.read(bs) + if len(piece) == bs: intern.update(piece) else: - break - fi.close() + raise IOError('Premature EOF reading git cat-file output') + ptr += bs dig = intern.hexdigest() + assert(p.stdout.read(1) == b'\n') # ignore LF that follows blob data + # update overall hash with file hash overall.update(dig.encode("utf-8")) overall.update(" ".encode("utf-8")) overall.update(f) overall.update("\n".encode("utf-8")) + p.stdin.close() + if p.wait(): + raise IOError('Non-zero return value executing git cat-file') return overall.hexdigest() +def print_merge_details(pull, title, branch, base_branch, head_branch): + print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) + subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) def parse_arguments(): epilog = ''' @@ -134,7 +174,7 @@ def main(): info = retrieve_pr_info(repo,pull) if info is None: exit(1) - title = info['title'] + title = info['title'].strip() # precedence order for destination branch argument: # - command line argument # - githubmerge.branch setting @@ -200,6 +240,12 @@ def main(): print("ERROR: Creating merge failed (already merged?).",file=stderr) exit(4) + symlink_files = get_symlink_files() + for f in symlink_files: + print("ERROR: File %s was a symlink" % f) + if len(symlink_files) > 0: + exit(4) + # Put tree SHA512 into the message try: first_sha512 = tree_sha512sum() @@ -212,13 +258,8 @@ def main(): except subprocess.CalledProcessError as e: printf("ERROR: Cannot update message.",file=stderr) exit(4) - second_sha512 = tree_sha512sum() - if first_sha512 != second_sha512: - print("ERROR: Tree hash changed unexpectedly",file=stderr) - exit(4) - print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) - subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) + print_merge_details(pull, title, branch, base_branch, head_branch) print() # Run test command if configured. @@ -237,12 +278,6 @@ def main(): print("Difference with github ignored.",file=stderr) else: exit(6) - reply = ask_prompt("Press 'd' to accept the diff.") - if reply.lower() == 'd': - print("Diff accepted.",file=stderr) - else: - print("ERROR: Diff rejected.",file=stderr) - exit(6) else: # Verify the result manually. print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr) @@ -251,24 +286,25 @@ def main(): if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt os.putenv('debian_chroot',pull) subprocess.call([BASH,'-i']) - reply = ask_prompt("Type 'm' to accept the merge.") - if reply.lower() == 'm': - print("Merge accepted.",file=stderr) - else: - print("ERROR: Merge rejected.",file=stderr) - exit(7) + + second_sha512 = tree_sha512sum() + if first_sha512 != second_sha512: + print("ERROR: Tree hash changed unexpectedly",file=stderr) + exit(8) # Sign the merge commit. - reply = ask_prompt("Type 's' to sign off on the merge.") - if reply == 's': - try: - subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) - except subprocess.CalledProcessError as e: - print("Error signing, exiting.",file=stderr) + print_merge_details(pull, title, branch, base_branch, head_branch) + while True: + reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower() + if reply == 's': + try: + subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) + break + except subprocess.CalledProcessError as e: + print("Error while signing, asking again.",file=stderr) + elif reply == 'x': + print("Not signing off on merge, exiting.",file=stderr) exit(1) - else: - print("Not signing off on merge, exiting.",file=stderr) - exit(1) # Put the result in branch. subprocess.check_call([GIT,'checkout','-q',branch]) @@ -282,9 +318,13 @@ def main(): subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull) # Push the result. - reply = ask_prompt("Type 'push' to push the result to %s, branch %s." % (host_repo,branch)) - if reply.lower() == 'push': - subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + while True: + reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower() + if reply == 'push': + subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + break + elif reply == 'x': + exit(1) if __name__ == '__main__': main() diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 00af4bdc6f..3da8510cfb 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -132,6 +132,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host + export GIT_DIR="$PWD/.git" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 05cc65414f..206db7c19e 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -101,6 +101,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host + export GIT_DIR="$PWD/.git" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 6fead7c208..1d4d70494b 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -116,6 +116,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host + export GIT_DIR="$PWD/.git" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist @@ -146,6 +147,7 @@ script: | make ${MAKEOPTS} -C src check-security make deploy make install DESTDIR=${INSTALLPATH} + rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe cp -f bitcoin-*setup*.exe $OUTDIR/ cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ @@ -159,9 +161,11 @@ script: | cd ../../ rm -rf distsrc-${i} done - cd $OUTDIR - rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe - find . -name "*-setup-unsigned.exe" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz + cp -rf contrib/windeploy $BUILD_DIR + cd $BUILD_DIR/windeploy + mkdir unsigned + cp $OUTDIR/bitcoin-*setup-unsigned.exe unsigned/ + find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip mv ${OUTDIR}/${DISTNAME}-i686-*-debug.zip ${OUTDIR}/${DISTNAME}-win32-debug.zip mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip diff --git a/contrib/init/README.md b/contrib/init/README.md index eb5d30acce..1a949f3c07 100644 --- a/contrib/init/README.md +++ b/contrib/init/README.md @@ -1,12 +1,12 @@ Sample configuration files for: - +``` SystemD: bitcoind.service Upstart: bitcoind.conf OpenRC: bitcoind.openrc bitcoind.openrcconf CentOS: bitcoind.init OS X: org.bitcoin.bitcoind.plist - +``` have been made available to assist packagers in creating node packages here. See doc/init.md for more information. diff --git a/contrib/init/bitcoind.openrcconf b/contrib/init/bitcoind.openrcconf index 0cbff6d30d..f70e25cb5f 100644 --- a/contrib/init/bitcoind.openrcconf +++ b/contrib/init/bitcoind.openrcconf @@ -23,7 +23,7 @@ #BITCOIND_NICE=0 # Additional options (avoid -conf and -datadir, use flags above) -BITCOIND_OPTS="-disablewallet" +#BITCOIND_OPTS="" # The timeout in seconds OpenRC will wait for bitcoind to terminate # after a SIGTERM has been raised. diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 0971e7816b..f2a2ab2768 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -7,7 +7,8 @@ run using Python 3 but are compatible with Python 2. $ ./linearize-hashes.py linearize.cfg > hashlist.txt Required configuration file settings for linearize-hashes: -* RPC: `rpcuser`, `rpcpassword` +* RPC: `datadir` (Required if `rpcuser` and `rpcpassword` are not specified) +* RPC: `rpcuser`, `rpcpassword` (Required if `datadir` is not specified) Optional config file setting for linearize-hashes: * RPC: `host` (Default: `127.0.0.1`) diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 2cc910edfe..d019b06b6c 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -1,6 +1,7 @@ # bitcoind RPC settings (linearize-hashes) rpcuser=someuser rpcpassword=somepassword +#datadir=~/.bitcoin host=127.0.0.1 port=8332 #port=18332 diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 00a54d0820..db8eb7021e 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -16,6 +16,8 @@ import json import re import base64 import sys +import os +import os.path settings = {} @@ -93,6 +95,14 @@ def get_block_hashes(settings, max_blocks_per_call=10000): height += num_blocks +def get_rpc_cookie(): + # Open the cookie file + with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: + combined = f.readline() + combined_split = combined.split(":") + settings['rpcuser'] = combined_split[0] + settings['rpcpassword'] = combined_split[1] + if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: linearize-hashes.py CONFIG-FILE") @@ -122,8 +132,15 @@ if __name__ == '__main__': settings['max_height'] = 313000 if 'rev_hash_bytes' not in settings: settings['rev_hash_bytes'] = 'false' + + use_userpass = True + use_datadir = False if 'rpcuser' not in settings or 'rpcpassword' not in settings: - print("Missing username and/or password in cfg file", file=stderr) + use_userpass = False + if 'datadir' in settings and not use_userpass: + use_datadir = True + if not use_userpass and not use_datadir: + print("Missing datadir or username and/or password in cfg file", file=stderr) sys.exit(1) settings['port'] = int(settings['port']) @@ -133,4 +150,8 @@ if __name__ == '__main__': # Force hash byte format setting to be lowercase to make comparisons easier. settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() + # Get the rpc user and pass from the cookie if the datadir is set + if use_datadir: + get_rpc_cookie() + get_block_hashes(settings) diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 5022ea88bc..7f017bb4f1 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -10,7 +10,7 @@ BUNDLE="${ROOTDIR}/Bitcoin-Qt.app" CODESIGN=codesign TEMPDIR=sign.temp TEMPLIST=${TEMPDIR}/signatures.txt -OUT=signature.tar.gz +OUT=signature-osx.tar.gz OUTROOT=osx if [ ! -n "$1" ]; then diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 5995f9f438..23a568ad13 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -302,7 +302,6 @@ def copyFramework(framework, path, verbose): if os.path.exists(fromContentsDir): toContentsDir = os.path.join(path, framework.destinationVersionContentsDirectory) shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) - contentslinkfrom = os.path.join(path, framework.destinationContentsDirectory) if verbose >= 3: print("Copied Contents:", fromContentsDir) print(" to:", toContentsDir) @@ -675,9 +674,8 @@ else: if verbose >= 2: print("+ Installing qt.conf +") -f = open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") -f.write(qt_conf.encode()) -f.close() +with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: + f.write(qt_conf.encode()) # ------------------------------------------------ diff --git a/contrib/rpm/bitcoin.spec b/contrib/rpm/bitcoin.spec index 516f42334e..cc54fcaf3d 100644 --- a/contrib/rpm/bitcoin.spec +++ b/contrib/rpm/bitcoin.spec @@ -311,10 +311,8 @@ rm -f %{buildroot}%{_bindir}/test_* %check make check -pushd src -srcdir=. test/bitcoin-util-test.py -popd -qa/pull-tester/rpc-tests.py -extended +srcdir=src test/bitcoin-util-test.py +test/functional/test_runner.py --extended %post libs -p /sbin/ldconfig diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index afe902fd7f..139c03181f 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -8,7 +8,7 @@ and remove old versions as necessary. The seeds compiled into the release are created from sipa's DNS seed data, like this: - curl -s http://bitcoin.sipa.be/seeds.txt > seeds_main.txt + curl -s http://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt python3 makeseeds.py < seeds_main.txt > nodes_main.txt python3 generate-seeds.py . > ../../src/chainparamsseeds.h diff --git a/contrib/testgen/gen_base58_test_vectors.py b/contrib/testgen/gen_base58_test_vectors.py index ac0701e6be..86366eb8e3 100755 --- a/contrib/testgen/gen_base58_test_vectors.py +++ b/contrib/testgen/gen_base58_test_vectors.py @@ -13,7 +13,7 @@ Usage: # Released under MIT License import os from itertools import islice -from base58 import b58encode, b58decode, b58encode_chk, b58decode_chk, b58chars +from base58 import b58encode_chk, b58decode_chk, b58chars import random from binascii import b2a_hex diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 09ff237544..b01e2a6d39 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -8,21 +8,43 @@ VALID=false REVSIG=false IFS=' ' -for LINE in $(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null); do +if [ "$BITCOIN_VERIFY_COMMITS_ALLOW_SHA1" = 1 ]; then + GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" +else + # Note how we've disabled SHA1 with the --weak-digest option, disabling + # signatures - including selfsigs - that use SHA1. While you might think that + # collision attacks shouldn't be an issue as they'd be an attack on yourself, + # in fact because what's being signed is a commit object that's + # semi-deterministically generated by untrusted input (the pull-req) in theory + # an attacker could construct a pull-req that results in a commit object that + # they've created a collision for. Not the most likely attack, but preventing + # it is pretty easy so we do so as a "belt-and-suspenders" measure. + GPG_RES="" + for LINE in "$(gpg --version)"; do + case "$LINE" in + "gpg (GnuPG) 1.4.1"*|"gpg (GnuPG) 2.0."*) + echo "Please upgrade to at least gpg 2.1.10 to check for weak signatures" > /dev/stderr + GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + ;; + # We assume if you're running 2.1+, you're probably running 2.1.10+ + # gpg will fail otherwise + # We assume if you're running 1.X, it is either 1.4.1X or 1.4.20+ + # gpg will fail otherwise + esac + done + [ "$GPG_RES" = "" ] && GPG_RES="$(echo "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" +fi +for LINE in $(echo "$GPG_RES"); do case "$LINE" in "[GNUPG:] VALIDSIG "*) while read KEY; do - case "$LINE" in "[GNUPG:] VALIDSIG $KEY "*) VALID=true;; esac + [ "${LINE#?GNUPG:? VALIDSIG * * * * * * * * * }" = "$KEY" ] && VALID=true done < ./contrib/verify-commits/trusted-keys ;; "[GNUPG:] REVKEYSIG "*) [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 - while read KEY; do - case "$LINE" in "[GNUPG:] REVKEYSIG ${KEY#????????????????????????} "*) - REVSIG=true - GOODREVSIG="[GNUPG:] GOODSIG ${KEY#????????????????????????} " - esac - done < ./contrib/verify-commits/trusted-keys + REVSIG=true + GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" ;; esac done @@ -30,7 +52,7 @@ if ! $VALID; then exit 1 fi if $VALID && $REVSIG; then - echo "$INPUT" | gpg --trust-model always "$@" | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" 2>/dev/null + echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" echo "$GOODREVSIG" else echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index 75242c2a97..5610692616 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -1,4 +1,4 @@ 71A3B16735405025D447E8F274810B012346C9A6 -3F1888C6DCA92A6499C4911FDBA1A67379A1A931 +133EAC179436F14A5CF1B794860FEB804E669320 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC -FE09B823E6D83A3BC7983EAA2D7F2372E50FE137 +B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B diff --git a/contrib/verify-commits/trusted-sha512-root-commit b/contrib/verify-commits/trusted-sha512-root-commit new file mode 100644 index 0000000000..7d41f90ad7 --- /dev/null +++ b/contrib/verify-commits/trusted-sha512-root-commit @@ -0,0 +1 @@ +309bf16257b2395ce502017be627186b749ee749 diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh index b2cebdf1a0..74b7f38375 100755 --- a/contrib/verify-commits/verify-commits.sh +++ b/contrib/verify-commits/verify-commits.sh @@ -3,61 +3,130 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# Not technically POSIX-compliant due to use of "local", but almost every -# shell anyone uses today supports it, so its probably fine - DIR=$(dirname "$0") [ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0") +echo "Using verify-commits data from ${DIR}" + VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root") +VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit") REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits") HAVE_FAILED=false -IS_SIGNED () { - if [ $1 = $VERIFIED_ROOT ]; then - return 0; + +HAVE_GNU_SHA512=1 +[ ! -x "$(which sha512sum)" ] && HAVE_GNU_SHA512=0 + +if [ x"$1" = "x" ]; then + CURRENT_COMMIT="HEAD" +else + CURRENT_COMMIT="$1" +fi + +if [ "${CURRENT_COMMIT#* }" != "$CURRENT_COMMIT" ]; then + echo "Commit must not contain spaces?" > /dev/stderr + exit 1 +fi + +VERIFY_TREE=0 +if [ x"$2" = "x--tree-checks" ]; then + VERIFY_TREE=1 +fi + +NO_SHA1=1 +PREV_COMMIT="" + +while true; do + if [ "$CURRENT_COMMIT" = $VERIFIED_ROOT ]; then + echo "There is a valid path from "$CURRENT_COMMIT" to $VERIFIED_ROOT where all commits are signed!" + exit 0; + fi + + if [ "$CURRENT_COMMIT" = $VERIFIED_SHA512_ROOT ]; then + if [ "$VERIFY_TREE" = "1" ]; then + echo "All Tree-SHA512s matched up to $VERIFIED_SHA512_ROOT" > /dev/stderr + fi + VERIFY_TREE=0 + NO_SHA1=0 fi - if [ "${REVSIG_ALLOWED#*$1}" != "$REVSIG_ALLOWED" ]; then + + if [ "$NO_SHA1" = "1" ]; then + export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=0 + else + export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=1 + fi + + if [ "${REVSIG_ALLOWED#*$CURRENT_COMMIT}" != "$REVSIG_ALLOWED" ]; then export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1 else export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0 fi - if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null 2>&1; then - return 1; - fi - local PARENTS - PARENTS=$(git show -s --format=format:%P $1) - for PARENT in $PARENTS; do - if IS_SIGNED $PARENT; then - return 0; + + if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit "$CURRENT_COMMIT" > /dev/null; then + if [ "$PREV_COMMIT" != "" ]; then + echo "No parent of $PREV_COMMIT was signed with a trusted key!" > /dev/stderr + echo "Parents are:" > /dev/stderr + PARENTS=$(git show -s --format=format:%P $PREV_COMMIT) + for PARENT in $PARENTS; do + git show -s $PARENT > /dev/stderr + done + else + echo "$CURRENT_COMMIT was not signed with a trusted key!" > /dev/stderr fi - break - done - if ! "$HAVE_FAILED"; then - echo "No parent of $1 was signed with a trusted key!" > /dev/stderr - echo "Parents are:" > /dev/stderr - for PARENT in $PARENTS; do - git show -s $PARENT > /dev/stderr - done - HAVE_FAILED=true + exit 1 fi - return 1; -} -if [ x"$1" = "x" ]; then - TEST_COMMIT="HEAD" -else - TEST_COMMIT="$1" -fi + # We always verify the top of the tree + if [ "$VERIFY_TREE" = 1 -o "$PREV_COMMIT" = "" ]; then + IFS_CACHE="$IFS" + IFS=' +' + for LINE in $(git ls-tree --full-tree -r "$CURRENT_COMMIT"); do + case "$LINE" in + "12"*) + echo "Repo contains symlinks" > /dev/stderr + IFS="$IFS_CACHE" + exit 1 + ;; + esac + done + IFS="$IFS_CACHE" + + FILE_HASHES="" + for FILE in $(git ls-tree --full-tree -r --name-only "$CURRENT_COMMIT" | LC_ALL=C sort); do + if [ "$HAVE_GNU_SHA512" = 1 ]; then + HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST OTHER; echo $FIRST; } ) + else + HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST OTHER; echo $FIRST; } ) + fi + [ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"' +' + FILE_HASHES="$FILE_HASHES$HASH $FILE" + done + + if [ "$HAVE_GNU_SHA512" = 1 ]; then + TREE_HASH="$(echo "$FILE_HASHES" | sha512sum)" + else + TREE_HASH="$(echo "$FILE_HASHES" | shasum -a 512)" + fi + HASH_MATCHES=0 + MSG="$(git show -s --format=format:%B "$CURRENT_COMMIT" | tail -n1)" -IS_SIGNED "$TEST_COMMIT" -RES=$? -if [ "$RES" = 1 ]; then - if ! "$HAVE_FAILED"; then - echo "$TEST_COMMIT was not signed with a trusted key!" + case "$MSG -" in + "Tree-SHA512: $TREE_HASH") + HASH_MATCHES=1;; + esac + + if [ "$HASH_MATCHES" = "0" ]; then + echo "Tree-SHA512 did not match for commit $CURRENT_COMMIT" > /dev/stderr + exit 1 + fi fi -else - echo "There is a valid path from $TEST_COMMIT to $VERIFIED_ROOT where all commits are signed!" -fi -exit $RES + PARENTS=$(git show -s --format=format:%P "$CURRENT_COMMIT") + for PARENT in $PARENTS; do + PREV_COMMIT="$CURRENT_COMMIT" + CURRENT_COMMIT="$PARENT" + break + done +done diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh new file mode 100755 index 0000000000..bf4978d143 --- /dev/null +++ b/contrib/windeploy/detached-sig-create.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +if [ -z "$OSSLSIGNCODE" ]; then + OSSLSIGNCODE=osslsigncode +fi + +if [ ! -n "$1" ]; then + echo "usage: $0 <osslcodesign args>" + echo "example: $0 -key codesign.key" + exit 1 +fi + +OUT=signature-win.tar.gz +SRCDIR=unsigned +WORKDIR=./.tmp +OUTDIR="${WORKDIR}/out" +OUTSUBDIR="${OUTDIR}/win" +TIMESERVER=http://timestamp.comodoca.com +CERTFILE="win-codesign.cert" + +mkdir -p "${OUTSUBDIR}" +basename -a `ls -1 "${SRCDIR}"/*-unsigned.exe` | while read UNSIGNED; do + echo Signing "${UNSIGNED}" + "${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@" + "${OSSLSIGNCODE}" extract-signature -pem -in "${WORKDIR}/${UNSIGNED}" -out "${OUTSUBDIR}/${UNSIGNED}.pem" && rm "${WORKDIR}/${UNSIGNED}" +done + +rm -f "${OUT}" +tar -C "${OUTDIR}" -czf "${OUT}" . +rm -rf "${WORKDIR}" +echo "Created ${OUT}" diff --git a/contrib/windeploy/win-codesign.cert b/contrib/windeploy/win-codesign.cert new file mode 100644 index 0000000000..200b30a3f0 --- /dev/null +++ b/contrib/windeploy/win-codesign.cert @@ -0,0 +1,99 @@ +-----BEGIN CERTIFICATE----- +MIIFTTCCBDWgAwIBAgIRALlW05RLwG2hMQMX5d/o5J8wDQYJKoZIhvcNAQELBQAw +fTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxIzAhBgNV +BAMTGkNPTU9ETyBSU0EgQ29kZSBTaWduaW5nIENBMB4XDTE2MDIwMzAwMDAwMFoX +DTE5MDMwNTIzNTk1OVowgbUxCzAJBgNVBAYTAlVTMQ4wDAYDVQQRDAU5ODEwNDEL +MAkGA1UECAwCV0ExEDAOBgNVBAcMB1NlYXR0bGUxEDAOBgNVBAkMB1N0ZSAzMDAx +FzAVBgNVBAkMDjcxIENvbHVtYmlhIFN0MSUwIwYDVQQKDBxUaGUgQml0Y29pbiBG +b3VuZGF0aW9uLCBJbmMuMSUwIwYDVQQDDBxUaGUgQml0Y29pbiBGb3VuZGF0aW9u +LCBJbmMuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw37Vrv9Gbku0 ++kuV0t89TuyxtAcmT7QE4GcwESKKjmkxfzD9a0qlhqk8GfQ+fw4DHNN+nLKNv7xB +bk6aS7J2v2DcXkOjrP99P9jqgTkp7MC04VtG3OqVRGB+gum0pptRovYZUQXIdkY7 +GJOok/NDagwKiiUe2V2meZ7UctsZNvYeilQdTgKIIhrMB9NowCOhT8ocVL4Ki55/ +l7hukJn3fueCM3fHTwY2/1gaGsOHoCkFRsD7vokjAVpiY+8rUgvHjb0gxgojiVGd +6a6/F5XJwKJacvUyN4Hfc2K5lRMQjTTmo4aWNWIa0iJ3TK9BHpdSLJBqerMPvmnM +kkapS+ZTNQIDAQABo4IBjTCCAYkwHwYDVR0jBBgwFoAUKZFg/4pN+uv5pmq4z/nm +S71JzhIwHQYDVR0OBBYEFONpQ+cV82URVe+V8G57377KxxexMA4GA1UdDwEB/wQE +AwIHgDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMBEGCWCGSAGG ++EIBAQQEAwIEEDBGBgNVHSAEPzA9MDsGDCsGAQQBsjEBAgEDAjArMCkGCCsGAQUF +BwIBFh1odHRwczovL3NlY3VyZS5jb21vZG8ubmV0L0NQUzBDBgNVHR8EPDA6MDig +NqA0hjJodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9DT01PRE9SU0FDb2RlU2lnbmlu +Z0NBLmNybDB0BggrBgEFBQcBAQRoMGYwPgYIKwYBBQUHMAKGMmh0dHA6Ly9jcnQu +Y29tb2RvY2EuY29tL0NPTU9ET1JTQUNvZGVTaWduaW5nQ0EuY3J0MCQGCCsGAQUF +BzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQELBQADggEB +AGnBSi9K/9rgTAyKFKrfGWSfNOwAghmsnsvpZSQ7QyoGWBFKSgCs/70kErl18oHA +g7Y8loQB1yukZmJaCa3OvGud7smn45TCh0TMf4EpP20Wxf4rMQTxwAatasHL3+vi +I+Nl5bsRZ09kWjvayqLII5upjS/yq0JfpmyGl5k2C/fIpztq0iOLvqWlXcL4+51r +cMUAfX6E6EaZQm//ikp+w2+7MEXTKguOuV3gwsrTy0DsvkZl4YDgx/FA4ImzXopv +d+3KJPLvO+OSBqUD3JPwXHnuJqGAbLBFyyCa/feGUjLlR8cxcNWLWdp4qxtoIUPG +3wTsC9YgrglS0F7FKMXlNRY= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIF4DCCA8igAwIBAgIQLnyHzA6TSlL+lP0ct800rzANBgkqhkiG9w0BAQwFADCB +hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV +BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTMwNTA5 +MDAwMDAwWhcNMjgwNTA4MjM1OTU5WjB9MQswCQYDVQQGEwJHQjEbMBkGA1UECBMS +R3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFD +T01PRE8gQ0EgTGltaXRlZDEjMCEGA1UEAxMaQ09NT0RPIFJTQSBDb2RlIFNpZ25p +bmcgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmmJBjd5E0f4rR +3elnMRHrzB79MR2zuWJXP5O8W+OfHiQyESdrvFGRp8+eniWzX4GoGA8dHiAwDvth +e4YJs+P9omidHCydv3Lj5HWg5TUjjsmK7hoMZMfYQqF7tVIDSzqwjiNLS2PgIpQ3 +e9V5kAoUGFEs5v7BEvAcP2FhCoyi3PbDMKrNKBh1SMF5WgjNu4xVjPfUdpA6M0ZQ +c5hc9IVKaw+A3V7Wvf2pL8Al9fl4141fEMJEVTyQPDFGy3CuB6kK46/BAW+QGiPi +XzjbxghdR7ODQfAuADcUuRKqeZJSzYcPe9hiKaR+ML0btYxytEjy4+gh+V5MYnmL +Agaff9ULAgMBAAGjggFRMIIBTTAfBgNVHSMEGDAWgBS7r34CPfqm8TyEjq3uOJjs +2TIy1DAdBgNVHQ4EFgQUKZFg/4pN+uv5pmq4z/nmS71JzhIwDgYDVR0PAQH/BAQD +AgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAwEwYDVR0lBAwwCgYIKwYBBQUHAwMwEQYD +VR0gBAowCDAGBgRVHSAAMEwGA1UdHwRFMEMwQaA/oD2GO2h0dHA6Ly9jcmwuY29t +b2RvY2EuY29tL0NPTU9ET1JTQUNlcnRpZmljYXRpb25BdXRob3JpdHkuY3JsMHEG +CCsGAQUFBwEBBGUwYzA7BggrBgEFBQcwAoYvaHR0cDovL2NydC5jb21vZG9jYS5j +b20vQ09NT0RPUlNBQWRkVHJ1c3RDQS5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9v +Y3NwLmNvbW9kb2NhLmNvbTANBgkqhkiG9w0BAQwFAAOCAgEAAj8COcPu+Mo7id4M +bU2x8U6ST6/COCwEzMVjEasJY6+rotcCP8xvGcM91hoIlP8l2KmIpysQGuCbsQci +GlEcOtTh6Qm/5iR0rx57FjFuI+9UUS1SAuJ1CAVM8bdR4VEAxof2bO4QRHZXavHf +WGshqknUfDdOvf+2dVRAGDZXZxHNTwLk/vPa/HUX2+y392UJI0kfQ1eD6n4gd2HI +TfK7ZU2o94VFB696aSdlkClAi997OlE5jKgfcHmtbUIgos8MbAOMTM1zB5TnWo46 +BLqioXwfy2M6FafUFRunUkcyqfS/ZEfRqh9TTjIwc8Jvt3iCnVz/RrtrIh2IC/gb +qjSm/Iz13X9ljIwxVzHQNuxHoc/Li6jvHBhYxQZ3ykubUa9MCEp6j+KjUuKOjswm +5LLY5TjCqO3GgZw1a6lYYUoKl7RLQrZVnb6Z53BtWfhtKgx/GWBfDJqIbDCsUgmQ +Fhv/K53b0CDKieoofjKOGd97SDMe12X4rsn4gxSTdn1k0I7OvjV9/3IxTZ+evR5s +L6iPDAZQ+4wns3bJ9ObXwzTijIchhmH+v1V04SF3AwpobLvkyanmz1kl63zsRQ55 +ZmjoIs2475iFTZYRPAmK0H+8KCgT+2rKVI2SXM3CZZgGns5IW9S1N5NGQXwH3c/6 +Q++6Z2H/fUnguzB9XIDj5hY5S6c= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB +hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV +BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 +MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR +Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR +6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X +pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC +9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV +/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf +Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z ++pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w +qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah +SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC +u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf +Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq +crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E +FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl +wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM +4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV +2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna +FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ +CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK +boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke +jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL +S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb +QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl +0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB +NVOFBkpdn627G190 +-----END CERTIFICATE----- diff --git a/contrib/zmq/zmq_sub3.4.py b/contrib/zmq/zmq_sub3.4.py index a2ff64b29b..1cb7eec0c0 100755 --- a/contrib/zmq/zmq_sub3.4.py +++ b/contrib/zmq/zmq_sub3.4.py @@ -57,7 +57,7 @@ class ZMQHandler(): msg = yield from self.zmqSubSocket.recv_multipart() topic = msg[0] body = msg[1] - sequence = "Unknown"; + sequence = "Unknown" if len(msg[-1]) == 4: msgSequence = struct.unpack('<I', msg[-1])[-1] sequence = str(msgSequence) diff --git a/depends/config.guess b/depends/config.guess index bbd48b60e8..69ed3e573b 100755 --- a/depends/config.guess +++ b/depends/config.guess @@ -2,7 +2,7 @@ # Attempt to guess a canonical system name. # Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2017-01-01' +timestamp='2017-03-05' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -837,10 +837,11 @@ EOF UNAME_PROCESSOR=`/usr/bin/uname -p` case ${UNAME_PROCESSOR} in amd64) - echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin @@ -1343,6 +1344,9 @@ EOF NSR-?:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} exit ;; + NSX-?:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk${UNAME_RELEASE} + exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux exit ;; diff --git a/depends/config.sub b/depends/config.sub index 7e792b4ae1..40ea5dfe11 100755 --- a/depends/config.sub +++ b/depends/config.sub @@ -2,7 +2,7 @@ # Configuration validation subroutine script. # Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2017-01-01' +timestamp='2017-04-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -263,7 +263,7 @@ case $basic_machine in | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ - | i370 | i860 | i960 | ia64 \ + | i370 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ | le32 | le64 \ @@ -315,6 +315,7 @@ case $basic_machine in | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ | visium \ + | wasm32 \ | we32k \ | x86 | xc16x | xstormy16 | xtensa \ | z8k | z80) @@ -388,7 +389,7 @@ case $basic_machine in | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ + | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ | ip2k-* | iq2000-* \ | k1om-* \ | le32-* | le64-* \ @@ -446,6 +447,7 @@ case $basic_machine in | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ | visium-* \ + | wasm32-* \ | we32k-* \ | x86-* | x86_64-* | xc16x-* | xps100-* \ | xstormy16-* | xtensa*-* \ @@ -948,6 +950,9 @@ case $basic_machine in nsr-tandem) basic_machine=nsr-tandem ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; op50n-* | op60c-*) basic_machine=hppa1.1-oki os=-proelf @@ -1243,6 +1248,9 @@ case $basic_machine in basic_machine=a29k-wrs os=-vxworks ;; + wasm32) + basic_machine=wasm32-unknown + ;; w65*) basic_machine=w65-wdc os=-none diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 57d96e4821..bf773ccd14 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,8 +1,8 @@ package=boost -$(package)_version=1_63_0 -$(package)_download_path=https://sourceforge.net/projects/boost/files/boost/1.63.0 +$(package)_version=1_64_0 +$(package)_download_path=https://dl.bintray.com/boostorg/release/1.64.0/source/ $(package)_file_name=$(package)_$($(package)_version).tar.bz2 -$(package)_sha256_hash=beae2529f759f6b3bf3f4969a19c2e9d6f0c503edcb2de4a61d1428519fcb3b0 +$(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 define $(package)_set_vars $(package)_config_opts_release=variant=release diff --git a/depends/packages/dbus.mk b/depends/packages/dbus.mk index 90ddcb923f..bbe0375409 100644 --- a/depends/packages/dbus.mk +++ b/depends/packages/dbus.mk @@ -1,8 +1,8 @@ package=dbus -$(package)_version=1.10.14 -$(package)_download_path=http://dbus.freedesktop.org/releases/dbus +$(package)_version=1.10.18 +$(package)_download_path=https://dbus.freedesktop.org/releases/dbus $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=23238f70353e38ce5ca183ebc9525c0d97ac00ef640ad29cf794782af6e6a083 +$(package)_sha256_hash=6049ddd5f3f3e2618f615f1faeda0a115104423a7996b7aa73e2f36e38cc514a $(package)_dependencies=expat define $(package)_set_vars diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 70f345e71d..00231d75d5 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -1,8 +1,8 @@ package=libevent -$(package)_version=2.1.7 +$(package)_version=2.1.8-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ -$(package)_file_name=release-$($(package)_version)-rc.tar.gz -$(package)_sha256_hash=548362d202e22fe24d4c3fad38287b4f6d683e6c21503341373b89785fa6f991 +$(package)_file_name=release-$($(package)_version).tar.gz +$(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d define $(package)_preprocess_cmds ./autogen.sh diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index e34cf7be2f..1bb8cb5d26 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -1,8 +1,8 @@ package=miniupnpc -$(package)_version=2.0 +$(package)_version=2.0.20170509 $(package)_download_path=http://miniupnp.free.fr/files $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=d434ceb8986efbe199c5ca53f90ed53eab290b1e6d0530b717eb6fa49d61f93b +$(package)_sha256_hash=d3c368627f5cdfb66d3ebd64ca39ba54d6ff14a61966dbecb8dd296b7039f16a define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" diff --git a/depends/packages/native_ccache.mk b/depends/packages/native_ccache.mk index 4ed61a49e9..966804ce8b 100644 --- a/depends/packages/native_ccache.mk +++ b/depends/packages/native_ccache.mk @@ -1,8 +1,8 @@ package=native_ccache -$(package)_version=3.3.3 +$(package)_version=3.3.4 $(package)_download_path=https://samba.org/ftp/ccache $(package)_file_name=ccache-$($(package)_version).tar.bz2 -$(package)_sha256_hash=2985bc5e32ebe38d2958d508eb54ddcad39eed909489c0c2988035214597ca54 +$(package)_sha256_hash=fa9d7f38367431bc86b19ad107d709ca7ecf1574fdacca01698bdf0a47cd8567 define $(package)_set_vars $(package)_config_opts= diff --git a/depends/packages/zlib.mk b/depends/packages/zlib.mk index 7ff5d00bbd..589490800f 100644 --- a/depends/packages/zlib.mk +++ b/depends/packages/zlib.mk @@ -7,8 +7,10 @@ $(package)_sha256_hash=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca11 define $(package)_set_vars $(package)_build_opts= CC="$($(package)_cc)" $(package)_build_opts+=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC" -$(package)_build_opts+=AR="$($(package)_ar)" $(package)_build_opts+=RANLIB="$($(package)_ranlib)" +$(package)_build_opts+=AR="$($(package)_ar)" +$(package)_build_opts_darwin+=AR="$($(package)_libtool)" +$(package)_build_opts_darwin+=ARFLAGS="-o" endef define $(package)_config_cmds diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 0000000000..38498103bb --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +Doxyfile diff --git a/doc/Doxyfile b/doc/Doxyfile.in index 45436a6b15..58c65fb7e2 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile.in @@ -38,7 +38,7 @@ PROJECT_NAME = "Bitcoin Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.99 +PROJECT_NUMBER = @PACKAGE_VERSION@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/doc/README.md b/doc/README.md index 09f32bc09e..275ae67e54 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,9 +1,9 @@ -Bitcoin Core 0.14.99 -===================== +Bitcoin Core +============= Setup --------------------- -Bitcoin Core is the original Bitcoin client and it builds the backbone of the network. However, it downloads and stores the entire history of Bitcoin transactions (which is currently several GBs); depending on the speed of your computer and network connection, the synchronization process can take anywhere from a few hours to a day or more. +Bitcoin Core is the original Bitcoin client and it builds the backbone of the network. It downloads and, by default, stores the entire history of Bitcoin transactions (which is currently more than 100 GBs); depending on the speed of your computer and network connection, the synchronization process can take anywhere from a few hours to a day or more. To download Bitcoin Core, visit [bitcoincore.org](https://bitcoincore.org/en/releases/). diff --git a/doc/README_windows.txt b/doc/README_windows.txt index 0e4c9ce04f..07d61b3bda 100644 --- a/doc/README_windows.txt +++ b/doc/README_windows.txt @@ -1,5 +1,5 @@ -Bitcoin Core 0.14.99
-=====================
+Bitcoin Core
+=============
Intro
-----
diff --git a/doc/build-osx.md b/doc/build-osx.md index a15bcd012c..32d7dbd69e 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -11,14 +11,14 @@ Install the OS X command line tools: When the popup appears, click `Install`. -Then install [Homebrew](http://brew.sh). +Then install [Homebrew](https://brew.sh). Dependencies ---------------------- - brew install automake berkeley-db4 libtool boost --c++11 miniupnpc openssl pkg-config protobuf --c++11 qt5 libevent + brew install automake berkeley-db4 libtool boost --c++11 miniupnpc openssl pkg-config protobuf qt libevent -In case you want to build the disk image with `make deploy` (.dmg / optional), you need RSVG +If you want to build the disk image with `make deploy` (.dmg / optional), you need RSVG brew install librsvg diff --git a/doc/build-unix.md b/doc/build-unix.md index 31a88a1b18..b7eae2a630 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -321,8 +321,10 @@ Clang is installed by default as `cc` compiler, this makes it easier to get started than on [OpenBSD](build-openbsd.md). Installing dependencies: pkg install autoconf automake libtool pkgconf - pkg install boost-libs openssl libevent2 + pkg install boost-libs openssl libevent + pkg install gmake +You need to use GNU make (`gmake`) instead of `make`. (`libressl` instead of `openssl` will also work) For the wallet (optional): @@ -338,7 +340,7 @@ Then build using: ./autogen.sh ./configure --with-incompatible-bdb BDB_CFLAGS="-I/usr/local/include/db5" BDB_LIBS="-L/usr/local/lib -ldb_cxx-5" - make + gmake *Note on debugging*: The version of `gdb` installed by default is [ancient and considered harmful](https://wiki.freebsd.org/GdbRetirement). It is not suitable for debugging a multi-threaded C++ program, not even for getting backtraces. Please install the package `gdb` and diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 3a7e4f73b3..ec6abda91e 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -3,41 +3,64 @@ Developer Notes Various coding styles have been used during the history of the codebase, and the result is not very consistent. However, we're now trying to converge to -a single style, so please use it in new code. Old code will be converted -gradually and you are encouraged to use the provided -[clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) -to clean up the patch automatically before submitting a pull request. +a single style, which is specified below. When writing patches, favor the new +style over attempting to mimick the surrounding style, except for move-only +commits. + +Do not submit patches solely to modify the style of existing code. -- Basic rules specified in [src/.clang-format](/src/.clang-format). +- **Indentation and whitespace rules** as specified in +[src/.clang-format](/src/.clang-format). You can use the provided +[clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) +tool to clean up patches automatically before submission. - Braces on new lines for namespaces, classes, functions, methods. - Braces on the same line for everything else. - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - No extra spaces inside parenthesis; don't do ( this ) - No space after function names; one space after `if`, `for` and `while`. - - If an `if` only has a single-statement then-clause, it can appear - on the same line as the if, without braces. In every other case, - braces are required, and the then and else clauses must appear + - If an `if` only has a single-statement `then`-clause, it can appear + on the same line as the `if`, without braces. In every other case, + braces are required, and the `then` and `else` clauses must appear correctly indented on a new line. + +- **Symbol naming conventions**. These are preferred in new code, but are not +required when doing so would need changes to significant pieces of existing +code. + - Variable and namespace names are all lowercase, and may use `_` to + separate words. + - Class member variables have a `m_` prefix. + - Global variables have a `g_` prefix. + - Constant names are all uppercase, and use `_` to separate words. + - Class names, function names and method names are CamelCase. Do not prefix + class names with `C`. + +- **Miscellaneous** - `++i` is preferred over `i++`. Block style example: ```c++ +int g_count = 0; + namespace foo { class Class { + std::string m_name; + +public: bool Function(const std::string& s, int n) { // Comment summarising what this section of code does for (int i = 0; i < n; ++i) { + int total_sum = 0; // When something fails, return early if (!Something()) return false; ... - if (SomethingElse()) { - DoMore(); + if (SomethingElse(i)) { + total_sum += ComputeSomething(g_count); } else { - DoLess(); + DoSomething(m_name, total_sum); } } @@ -132,7 +155,7 @@ Run with the -testnet option to run with "play bitcoins" on the test network, if are testing multi-machine code that needs to operate across the internet. If you are testing something that can run on one machine, run with the -regtest option. -In regression test mode, blocks can be created on-demand; see qa/rpc-tests/ for tests +In regression test mode, blocks can be created on-demand; see test/functional/ for tests that run in -regtest mode. **DEBUG_LOCKORDER** @@ -252,7 +275,7 @@ Wallet - *Rationale*: In RPC code that conditionally uses the wallet (such as `validateaddress`) it is easy to forget that global pointer `pwalletMain` - can be NULL. See `qa/rpc-tests/disablewallet.py` for functional tests + can be NULL. See `test/functional/disablewallet.py` for functional tests exercising the API with `-disablewallet` - Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set @@ -343,10 +366,9 @@ Strings and formatting Variable names -------------- -The shadowing warning (`-Wshadow`) is enabled by default. It prevents issues rising -from using a different variable with the same name. - -Please name variables so that their names do not shadow variables defined in the source code. +Although the shadowing warning (`-Wshadow`) is not enabled by default (it prevents issues rising +from using a different variable with the same name), +please name variables so that their names do not shadow variables defined in the source code. E.g. in member initializers, prepend `_` to the argument name shadowing the member name: @@ -495,3 +517,76 @@ Git and GitHub tips This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. + +RPC interface guidelines +-------------------------- + +A few guidelines for introducing and reviewing new RPC interfaces: + +- Method naming: use consecutive lower-case names such as `getrawtransaction` and `submitblock` + + - *Rationale*: Consistency with existing interface. + +- Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) + + - *Rationale*: Consistency with existing interface. + +- Use the JSON parser for parsing, don't manually parse integers or strings from + arguments unless absolutely necessary. + + - *Rationale*: Introduces hand-rolled string manipulation code at both the caller and callee sites, + which is error prone, and it is easy to get things such as escaping wrong. + JSON already supports nested data structures, no need to re-invent the wheel. + + - *Exception*: AmountToValue can parse amounts as string. This was introduced because many JSON + parsers and formatters hard-code handling decimal numbers as floating point + values, resulting in potential loss of precision. This is unacceptable for + monetary values. **Always** use `AmountToValue` and `ValueToAmount` when + inputting or outputting monetary values. The only exceptions to this are + `prioritisetransaction` and `getblocktemplate` because their interface + is specified as-is in BIP22. + +- Missing arguments and 'null' should be treated the same: as default values. If there is no + default value, both cases should fail in the same way. + + - *Rationale*: Avoids surprises when switching to name-based arguments. Missing name-based arguments + are passed as 'null'. + + - *Exception*: Many legacy exceptions to this exist, one of the worst ones is + `getbalance` which follows a completely different code path based on the + number of arguments. We are still in the process of cleaning these up. Do not introduce + new ones. + +- Try not to overload methods on argument type. E.g. don't make `getblock(true)` and `getblock("hash")` + do different things. + + - *Rationale*: This is impossible to use with `bitcoin-cli`, and can be surprising to users. + + - *Exception*: Some RPC calls can take both an `int` and `bool`, most notably when a bool was switched + to a multi-value, or due to other historical reasons. **Always** have false map to 0 and + true to 1 in this case. + +- Don't forget to fill in the argument names correctly in the RPC command table. + + - *Rationale*: If not, the call can not be used with name-based arguments. + +- Set okSafeMode in the RPC command table to a sensible value: safe mode is when the + blockchain is regarded to be in a confused state, and the client deems it unsafe to + do anything irreversible such as send. Anything that just queries should be permitted. + + - *Rationale*: Troubleshooting a node in safe mode is difficult if half the + RPCs don't work. + +- Add every non-string RPC argument `(method, idx, name)` to the table `vRPCConvertParams` in `rpc/client.cpp`. + + - *Rationale*: `bitcoin-cli` and the GUI debug console use this table to determine how to + convert a plaintext command line to JSON. If the types don't match, the method can be unusable + from there. + +- A RPC method must either be a wallet method or a non-wallet method. Do not + introduce new methods such as `getinfo` and `signrawtransaction` that differ + in behavior based on presence of a wallet. + + - *Rationale*: as well as complicating the implementation and interfering + with the introduction of multi-wallet, wallet and non-wallet code should be + separated to avoid introducing circular dependencies between code units. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index bf3ad17861..5dedcb51c8 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -32,6 +32,13 @@ We disable ccache because we don't want to pollute the ccache with instrumented objects, and similarly don't want to use non-instrumented cached objects linked in. +The fuzzing can be sped up significantly (~200x) by using `afl-clang-fast` and +`afl-clang-fast++` in place of `afl-gcc` and `afl-g++` when compiling. When +compiling using `afl-clang-fast`/`afl-clang-fast++` the resulting +`test_bitcoin_fuzzy` binary will be instrumented in such a way that the AFL +features "persistent mode" and "deferred forkserver" can be used. See +https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. + Preparing fuzzing ------------------ @@ -63,4 +70,3 @@ $AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/test_bitcoin_fuzzy You may have to change a few kernel parameters to test optimally - `afl-fuzz` will print an error and suggestion if so. - diff --git a/doc/gitian-building.md b/doc/gitian-building.md index 442d2d605f..c2f55b5796 100644 --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -336,7 +336,7 @@ There will be a lot of warnings printed during the build of the image. These can Getting and building the inputs -------------------------------- -At this point you have two options, you can either use the automated script (found in [contrib/gitian-build.sh](/contrib/gitian-build.sh)) or you could manually do everything by following this guide. If you're using the automated script, then run it with the "--setup" command. Afterwards, run it with the "--build" command (example: "contrib/gitian-building.sh -b signer 0.13.0"). Otherwise ignore this. +At this point you have two options, you can either use the automated script (found in [contrib/gitian-build.sh](/contrib/gitian-build.sh)) or you could manually do everything by following this guide. If you're using the automated script, then run it with the "--setup" command. Afterwards, run it with the "--build" command (example: "contrib/gitian-build.sh -b signer 0.13.0"). Otherwise ignore this. Follow the instructions in [doc/release-process.md](release-process.md#fetch-and-create-inputs-first-time-or-when-dependency-versions-change) in the bitcoin repository under 'Fetch and create inputs' to install sources which require diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am index 5252ef4a19..08ff4d6ac1 100644 --- a/doc/man/Makefile.am +++ b/doc/man/Makefile.am @@ -1 +1,13 @@ -dist_man1_MANS=bitcoind.1 bitcoin-qt.1 bitcoin-cli.1 bitcoin-tx.1 +dist_man1_MANS= + +if BUILD_BITCOIND + dist_man1_MANS+=bitcoind.1 +endif + +if ENABLE_QT + dist_man1_MANS+=bitcoin-qt.1 +endif + +if BUILD_BITCOIN_UTILS + dist_man1_MANS+=bitcoin-cli.1 bitcoin-tx.1 +endif diff --git a/doc/release-notes.md b/doc/release-notes.md index eaa0b330eb..a13ede2dd5 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -33,6 +33,70 @@ frequently tested on them. Notable changes =============== +Low-level RPC changes +--------------------- + +- The new database model no longer stores information about transaction + versions of unspent outputs. This means that: + - The `gettxout` RPC no longer has a `version` field in the response. + - The `gettxoutsetinfo` RPC reports `hash_serialized_2` instead of `hash_serialized`, + which does not commit to the transaction versions of unspent outputs, but does + commit to the height and coinbase information. + - The `gettxoutsetinfo` response now contains `disk_size` and `bogosize` instead of + `bytes_serialized`. The first is a more accurate estimate of actual disk usage, but + is not deterministic. The second is unrelated to disk usage, but is a + database-independent metric of UTXO set size: it counts every UTXO entry as 50 + the + length of its scriptPubKey. + - The `getutxos` REST path no longer reports the `txvers` field in JSON format, + and always reports 0 for transaction versions in the binary format + + +- Error codes have been updated to be more accurate for the following error cases: + - `getblock` now returns RPC_MISC_ERROR if the block can't be found on disk (for + example if the block has been pruned). Previously returned RPC_INTERNAL_ERROR. + - `pruneblockchain` now returns RPC_MISC_ERROR if the blocks cannot be pruned + because the node is not in pruned mode. Previously returned RPC_METHOD_NOT_FOUND. + - `pruneblockchain` now returns RPC_INVALID_PARAMETER if the blocks cannot be pruned + because the supplied timestamp is too late. Previously returned RPC_INTERNAL_ERROR. + - `pruneblockchain` now returns RPC_MISC_ERROR if the blocks cannot be pruned + because the blockchain is too short. Previously returned RPC_INTERNAL_ERROR. + - `setban` now returns RPC_CLIENT_INVALID_IP_OR_SUBNET if the supplied IP address + or subnet is invalid. Previously returned RPC_CLIENT_NODE_ALREADY_ADDED. + - `setban` now returns RPC_CLIENT_INVALID_IP_OR_SUBNET if the user tries to unban + a node that has not previously been banned. Previously returned RPC_MISC_ERROR. + - `removeprunedfunds` now returns RPC_WALLET_ERROR if bitcoind is unable to remove + the transaction. Previously returned RPC_INTERNAL_ERROR. + - `removeprunedfunds` now returns RPC_INVALID_PARAMETER if the transaction does not + exist in the wallet. Previously returned RPC_INTERNAL_ERROR. + - `fundrawtransaction` now returns RPC_INVALID_ADDRESS_OR_KEY if an invalid change + address is provided. Previously returned RPC_INVALID_PARAMETER. + - `fundrawtransaction` now returns RPC_WALLET_ERROR if bitcoind is unable to create + the transaction. The error message provides further details. Previously returned + RPC_INTERNAL_ERROR. + - `bumpfee` now returns RPC_INVALID_PARAMETER if the provided transaction has + descendants in the wallet. Previously returned RPC_MISC_ERROR. + - `bumpfee` now returns RPC_INVALID_PARAMETER if the provided transaction has + descendants in the mempool. Previously returned RPC_MISC_ERROR. + - `bumpfee` now returns RPC_WALLET_ERROR if the provided transaction has + has been mined or conflicts with a mined transaction. Previously returned + RPC_INVALID_ADDRESS_OR_KEY. + - `bumpfee` now returns RPC_WALLET_ERROR if the provided transaction is not + BIP 125 replaceable. Previously returned RPC_INVALID_ADDRESS_OR_KEY. + - `bumpfee` now returns RPC_WALLET_ERROR if the provided transaction has already + been bumped by a different transaction. Previously returned RPC_INVALID_REQUEST. + - `bumpfee` now returns RPC_WALLET_ERROR if the provided transaction contains + inputs which don't belong to this wallet. Previously returned RPC_INVALID_ADDRESS_OR_KEY. + - `bumpfee` now returns RPC_WALLET_ERROR if the provided transaction has multiple change + outputs. Previously returned RPC_MISC_ERROR. + - `bumpfee` now returns RPC_WALLET_ERROR if the provided transaction has no change + output. Previously returned RPC_MISC_ERROR. + - `bumpfee` now returns RPC_WALLET_ERROR if the fee is too high. Previously returned + RPC_MISC_ERROR. + - `bumpfee` now returns RPC_WALLET_ERROR if the fee is too low. Previously returned + RPC_MISC_ERROR. + - `bumpfee` now returns RPC_WALLET_ERROR if the change output is too small to bump the + fee. Previously returned RPC_MISC_ERROR. + Credits ======= diff --git a/doc/release-notes/release-notes-0.14.0.md b/doc/release-notes/release-notes-0.14.0.md new file mode 100644 index 0000000000..c41f22979b --- /dev/null +++ b/doc/release-notes/release-notes-0.14.0.md @@ -0,0 +1,873 @@ +Bitcoin Core version 0.14.0 is now available from: + + <https://bitcoin.org/bin/bitcoin-core-0.14.0/> + +This is a new major version release, including new features, various bugfixes +and performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at github: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +Compatibility +============== + +Bitcoin Core is extensively tested on multiple operating systems using +the Linux kernel, macOS 10.8+, and Windows Vista and later. + +Microsoft ended support for Windows XP on [April 8th, 2014](https://www.microsoft.com/en-us/WindowsForBusiness/end-of-xp-support), +No attempt is made to prevent installing or running the software on Windows XP, you +can still do so at your own risk but be aware that there are known instabilities and issues. +Please do not report issues about Windows XP to the issue tracker. + +Bitcoin Core should also work on most other Unix-like systems but is not +frequently tested on them. + +Notable changes +=============== + +Performance Improvements +-------------- + +Validation speed and network propagation performance have been greatly +improved, leading to much shorter sync and initial block download times. + +- The script signature cache has been reimplemented as a "cuckoo cache", + allowing for more signatures to be cached and faster lookups. +- Assumed-valid blocks have been introduced which allows script validation to + be skipped for ancestors of known-good blocks, without changing the security + model. See below for more details. +- In some cases, compact blocks are now relayed before being fully validated as + per BIP152. +- P2P networking has been refactored with a focus on concurrency and + throughput. Network operations are no longer bottlenecked by validation. As a + result, block fetching is several times faster than previous releases in many + cases. +- The UTXO cache now claims unused mempool memory. This speeds up initial block + download as UTXO lookups are a major bottleneck there, and there is no use for + the mempool at that stage. + + +Manual Pruning +-------------- + +Bitcoin Core has supported automatically pruning the blockchain since 0.11. Pruning +the blockchain allows for significant storage space savings as the vast majority of +the downloaded data can be discarded after processing so very little of it remains +on the disk. + +Manual block pruning can now be enabled by setting `-prune=1`. Once that is set, +the RPC command `pruneblockchain` can be used to prune the blockchain up to the +specified height or timestamp. + +`getinfo` Deprecated +-------------------- + +The `getinfo` RPC command has been deprecated. Each field in the RPC call +has been moved to another command's output with that command also giving +additional information that `getinfo` did not provide. The following table +shows where each field has been moved to: + +|`getinfo` field | Moved to | +|------------------|-------------------------------------------| +`"version"` | `getnetworkinfo()["version"]` +`"protocolversion"`| `getnetworkinfo()["protocolversion"]` +`"walletversion"` | `getwalletinfo()["walletversion"]` +`"balance"` | `getwalletinfo()["balance"]` +`"blocks"` | `getblockchaininfo()["blocks"]` +`"timeoffset"` | `getnetworkinfo()["timeoffset"]` +`"connections"` | `getnetworkinfo()["connections"]` +`"proxy"` | `getnetworkinfo()["networks"][0]["proxy"]` +`"difficulty"` | `getblockchaininfo()["difficulty"]` +`"testnet"` | `getblockchaininfo()["chain"] == "test"` +`"keypoololdest"` | `getwalletinfo()["keypoololdest"]` +`"keypoolsize"` | `getwalletinfo()["keypoolsize"]` +`"unlocked_until"` | `getwalletinfo()["unlocked_until"]` +`"paytxfee"` | `getwalletinfo()["paytxfee"]` +`"relayfee"` | `getnetworkinfo()["relayfee"]` +`"errors"` | `getnetworkinfo()["warnings"]` + +ZMQ On Windows +-------------- + +Previously the ZeroMQ notification system was unavailable on Windows +due to various issues with ZMQ. These have been fixed upstream and +now ZMQ can be used on Windows. Please see [this document](https://github.com/bitcoin/bitcoin/blob/master/doc/zmq.md) for +help with using ZMQ in general. + +Nested RPC Commands in Debug Console +------------------------------------ + +The ability to nest RPC commands has been added to the debug console. This +allows users to have the output of a command become the input to another +command without running the commands separately. + +The nested RPC commands use bracket syntax (i.e. `getwalletinfo()`) and can +be nested (i.e. `getblock(getblockhash(1))`). Simple queries can be +done with square brackets where object values are accessed with either an +array index or a non-quoted string (i.e. `listunspent()[0][txid]`). Both +commas and spaces can be used to separate parameters in both the bracket syntax +and normal RPC command syntax. + +Network Activity Toggle +----------------------- + +A RPC command and GUI toggle have been added to enable or disable all p2p +network activity. The network status icon in the bottom right hand corner +is now the GUI toggle. Clicking the icon will either enable or disable all +p2p network activity. If network activity is disabled, the icon will +be grayed out with an X on top of it. + +Additionally the `setnetworkactive` RPC command has been added which does +the same thing as the GUI icon. The command takes one boolean parameter, +`true` enables networking and `false` disables it. + +Out-of-sync Modal Info Layer +---------------------------- + +When Bitcoin Core is out-of-sync on startup, a semi-transparent information +layer will be shown over top of the normal display. This layer contains +details about the current sync progress and estimates the amount of time +remaining to finish syncing. This layer can also be hidden and subsequently +unhidden by clicking on the progress bar at the bottom of the window. + +Support for JSON-RPC Named Arguments +------------------------------------ + +Commands sent over the JSON-RPC interface and through the `bitcoin-cli` binary +can now use named arguments. This follows the [JSON-RPC specification](http://www.jsonrpc.org/specification) +for passing parameters by-name with an object. + +`bitcoin-cli` has been updated to support this by parsing `name=value` arguments +when the `-named` option is given. + +Some examples: + + src/bitcoin-cli -named help command="help" + src/bitcoin-cli -named getblockhash height=0 + src/bitcoin-cli -named getblock blockhash=000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f + src/bitcoin-cli -named sendtoaddress address="(snip)" amount="1.0" subtractfeefromamount=true + +The order of arguments doesn't matter in this case. Named arguments are also +useful to leave out arguments that should stay at their default value. The +rarely-used arguments `comment` and `comment_to` to `sendtoaddress`, for example, can +be left out. However, this is not yet implemented for many RPC calls, this is +expected to land in a later release. + +The RPC server remains fully backwards compatible with positional arguments. + +Opt into RBF When Sending +------------------------- + +A new startup option, `-walletrbf`, has been added to allow users to have all +transactions sent opt into RBF support. The default value for this option is +currently `false`, so transactions will not opt into RBF by default. The new +`bumpfee` RPC can be used to replace transactions that opt into RBF. + +Sensitive Data Is No Longer Stored In Debug Console History +----------------------------------------------------------- + +The debug console maintains a history of previously entered commands that can be +accessed by pressing the Up-arrow key so that users can easily reuse previously +entered commands. Commands which have sensitive information such as passphrases and +private keys will now have a `(...)` in place of the parameters when accessed through +the history. + +Retaining the Mempool Across Restarts +------------------------------------- + +The mempool will be saved to the data directory prior to shutdown +to a `mempool.dat` file. This file preserves the mempool so that when the node +restarts the mempool can be filled with transactions without waiting for new transactions +to be created. This will also preserve any changes made to a transaction through +commands such as `prioritisetransaction` so that those changes will not be lost. + +Final Alert +----------- + +The Alert System was [disabled and deprecated](https://bitcoin.org/en/alert/2016-11-01-alert-retirement) in Bitcoin Core 0.12.1 and removed in 0.13.0. +The Alert System was retired with a maximum sequence final alert which causes any nodes +supporting the Alert System to display a static hard-coded "Alert Key Compromised" message which also +prevents any other alerts from overriding it. This final alert is hard-coded into this release +so that all old nodes receive the final alert. + +GUI Changes +----------- + + - After resetting the options by clicking the `Reset Options` button + in the options dialog or with the `-resetguioptions` startup option, + the user will be prompted to choose the data directory again. This + is to ensure that custom data directories will be kept after the + option reset which clears the custom data directory set via the choose + datadir dialog. + + - Multiple peers can now be selected in the list of peers in the debug + window. This allows for users to ban or disconnect multiple peers + simultaneously instead of banning them one at a time. + + - An indicator has been added to the bottom right hand corner of the main + window to indicate whether the wallet being used is a HD wallet. This + icon will be grayed out with an X on top of it if the wallet is not a + HD wallet. + +Low-level RPC changes +---------------------- + + - `importprunedfunds` only accepts two required arguments. Some versions accept + an optional third arg, which was always ignored. Make sure to never pass more + than two arguments. + + - The first boolean argument to `getaddednodeinfo` has been removed. This is + an incompatible change. + + - RPC command `getmininginfo` loses the "testnet" field in favor of the more + generic "chain" (which has been present for years). + + - A new RPC command `preciousblock` has been added which marks a block as + precious. A precious block will be treated as if it were received earlier + than a competing block. + + - A new RPC command `importmulti` has been added which receives an array of + JSON objects representing the intention of importing a public key, a + private key, an address and script/p2sh + + - Use of `getrawtransaction` for retrieving confirmed transactions with unspent + outputs has been deprecated. For now this will still work, but in the future + it may change to only be able to retrieve information about transactions in + the mempool or if `txindex` is enabled. + + - A new RPC command `getmemoryinfo` has been added which will return information + about the memory usage of Bitcoin Core. This was added in conjunction with + optimizations to memory management. See [Pull #8753](https://github.com/bitcoin/bitcoin/pull/8753) + for more information. + + - A new RPC command `bumpfee` has been added which allows replacing an + unconfirmed wallet transaction that signaled RBF (see the `-walletrbf` + startup option above) with a new transaction that pays a higher fee, and + should be more likely to get confirmed quickly. + +HTTP REST Changes +----------------- + + - UTXO set query (`GET /rest/getutxos/<checkmempool>/<txid>-<n>/<txid>-<n> + /.../<txid>-<n>.<bin|hex|json>`) responses were changed to return status + code `HTTP_BAD_REQUEST` (400) instead of `HTTP_INTERNAL_SERVER_ERROR` (500) + when requests contain invalid parameters. + +Minimum Fee Rate Policies +------------------------- + +Since the changes in 0.12 to automatically limit the size of the mempool and improve the performance of block creation in mining code it has not been important for relay nodes or miners to set `-minrelaytxfee`. With this release the following concepts that were tied to this option have been separated out: +- incremental relay fee used for calculating BIP 125 replacement and mempool limiting. (1000 satoshis/kB) +- calculation of threshold for a dust output. (effectively 3 * 1000 satoshis/kB) +- minimum fee rate of a package of transactions to be included in a block created by the mining code. If miners wish to set this minimum they can use the new `-blockmintxfee` option. (defaults to 1000 satoshis/kB) + +The `-minrelaytxfee` option continues to exist but is recommended to be left unset. + +Fee Estimation Changes +---------------------- + +- Since 0.13.2 fee estimation for a confirmation target of 1 block has been + disabled. The fee slider will no longer be able to choose a target of 1 block. + This is only a minor behavior change as there was often insufficient + data for this target anyway. `estimatefee 1` will now always return -1 and + `estimatesmartfee 1` will start searching at a target of 2. + +- The default target for fee estimation is changed to 6 blocks in both the GUI + (previously 25) and for RPC calls (previously 2). + +Removal of Priority Estimation +------------------------------ + +- Estimation of "priority" needed for a transaction to be included within a target + number of blocks has been removed. The RPC calls are deprecated and will either + return -1 or 1e24 appropriately. The format for `fee_estimates.dat` has also + changed to no longer save these priority estimates. It will automatically be + converted to the new format which is not readable by prior versions of the + software. + +- Support for "priority" (coin age) transaction sorting for mining is + considered deprecated in Core and will be removed in the next major version. + This is not to be confused with the `prioritisetransaction` RPC which will remain + supported by Core for adding fee deltas to transactions. + +P2P connection management +-------------------------- + +- Peers manually added through the `-addnode` option or `addnode` RPC now have their own + limit of eight connections which does not compete with other inbound or outbound + connection usage and is not subject to the limitation imposed by the `-maxconnections` + option. + +- New connections to manually added peers are performed more quickly. + +Introduction of assumed-valid blocks +------------------------------------- + +- A significant portion of the initial block download time is spent verifying + scripts/signatures. Although the verification must pass to ensure the security + of the system, no other result from this verification is needed: If the node + knew the history of a given block were valid it could skip checking scripts + for its ancestors. + +- A new configuration option 'assumevalid' is provided to express this knowledge + to the software. Unlike the 'checkpoints' in the past this setting does not + force the use of a particular chain: chains that are consistent with it are + processed quicker, but other chains are still accepted if they'd otherwise + be chosen as best. Also unlike 'checkpoints' the user can configure which + block history is assumed true, this means that even outdated software can + sync more quickly if the setting is updated by the user. + +- Because the validity of a chain history is a simple objective fact it is much + easier to review this setting. As a result the software ships with a default + value adjusted to match the current chain shortly before release. The use + of this default value can be disabled by setting -assumevalid=0 + +Fundrawtransaction change address reuse +---------------------------------------- + +- Before 0.14, `fundrawtransaction` was by default wallet stateless. In + almost all cases `fundrawtransaction` does add a change-output to the + outputs of the funded transaction. Before 0.14, the used keypool key was + never marked as change-address key and directly returned to the keypool + (leading to address reuse). Before 0.14, calling `getnewaddress` + directly after `fundrawtransaction` did generate the same address as + the change-output address. + +- Since 0.14, fundrawtransaction does reserve the change-output-key from + the keypool by default (optional by setting `reserveChangeKey`, default = + `true`) + +- Users should also consider using `getrawchangeaddress()` in conjunction + with `fundrawtransaction`'s `changeAddress` option. + +Unused mempool memory used by coincache +---------------------------------------- + +- Before 0.14, memory reserved for mempool (using the `-maxmempool` option) + went unused during initial block download, or IBD. In 0.14, the UTXO DB cache + (controlled with the `-dbcache` option) borrows memory from the mempool + when there is extra memory available. This may result in an increase in + memory usage during IBD for those previously relying on only the `-dbcache` + option to limit memory during that time. + +0.14.0 Change log +================= + +Detailed release notes follow. This overview includes changes that affect +behavior, not code moves, minor refactors and string updates. For convenience +in locating the code changes and accompanying discussion, both the pull request +and git merge commit are mentioned. + +### RPC and other APIs +- #8421 `b77bb95` httpserver: drop boost dependency (theuni) +- #8638 `f061415` rest.cpp: change `HTTP_INTERNAL_SERVER_ERROR` to `HTTP_BAD_REQUEST` (djpnewton) +- #8272 `91990ee` Make the dummy argument to getaddednodeinfo optional (sipa) +- #8722 `bb843ad` bitcoin-cli: More detailed error reporting (laanwj) +- #6996 `7f71a3c` Add preciousblock RPC (sipa) +- #8788 `97c7f73` Give RPC commands more information about the RPC request (jonasschnelli) +- #7948 `5d2c8e5` Augment getblockchaininfo bip9\_softforks data (mruddy) +- #8980 `0e22855` importmulti: Avoid using boost::variant::operator!=, which is only in newer boost versions (luke-jr) +- #9025 `4d8558a` Getrawtransaction should take a bool for verbose (jnewbery) +- #8811 `5754e03` Add support for JSON-RPC named arguments (laanwj) +- #9520 `2456a83` Deprecate non-txindex getrawtransaction and better warning (sipa) +- #9518 `a65ced1` Return height of last block pruned by pruneblockchain RPC (ryanofsky) +- #9222 `7cb024e` Add 'subtractFeeFromAmount' option to 'fundrawtransaction' (dooglus) +- #8456 `2ef52d3` Simplified `bumpfee` command (mrbandrews) +- #9516 `727a798` Bug-fix: listsinceblock: use fork point as reference for blocks in reorg'd chains (kallewoof) +- #9640 `7bfb770` Bumpfee: bugfixes for error handling and feerate calculation (sdaftuar) +- #9673 `8d6447e` Set correct metadata on bumpfee wallet transactions (ryanofsky) +- #9650 `40f7e27` Better handle invalid parameters to signrawtransaction (TheBlueMatt) +- #9682 `edc9e63` Require timestamps for importmulti keys (ryanofsky) +- #9108 `d8e8b06` Use importmulti timestamp when importing watch only keys (on top of #9682) (ryanofsky) +- #9756 `7a93af8` Return error when importmulti called with invalid address (ryanofsky) +- #9778 `ad168ef` Add two hour buffer to manual pruning (morcos) +- #9761 `9828f9a` Use 2 hour grace period for key timestamps in importmulti rescans (ryanofsky) +- #9474 `48d7e0d` Mark the minconf parameter to move as ignored (sipa) +- #9619 `861cb0c` Bugfix: RPC/Mining: GBT should return 1 MB sizelimit before segwit activates (luke-jr) +- #9773 `9072395` Return errors from importmulti if complete rescans are not successful (ryanofsky) + +### Block and transaction handling +- #8391 `37d83bb` Consensus: Remove ISM (NicolasDorier) +- #8365 `618c9dd` Treat high-sigop transactions as larger rather than rejecting them (sipa) +- #8814 `14b7b3f` wallet, policy: ParameterInteraction: Don't allow 0 fee (MarcoFalke) +- #8515 `9bdf526` A few mempool removal optimizations (sipa) +- #8448 `101c642` Store mempool and prioritization data to disk (sipa) +- #7730 `3c03dc2` Remove priority estimation (morcos) +- #9111 `fb15610` Remove unused variable `UNLIKELY_PCT` from fees.h (fanquake) +- #9133 `434e683` Unset fImporting for loading mempool (morcos) +- #9179 `b9a87b4` Set `DEFAULT_LIMITFREERELAY` = 0 kB/minute (MarcoFalke) +- #9239 `3fbf079` Disable fee estimates for 1-block target (morcos) +- #7562 `1eef038` Bump transaction version default to 2 (btcdrak) +- #9313,#9367 If we don't allow free txs, always send a fee filter (morcos) +- #9346 `b99a093` Batch construct batches (sipa) +- #9262 `5a70572` Prefer coins that have fewer ancestors, sanity check txn before ATMP (instagibbs) +- #9288 `1ce7ede` Fix a bug if the min fee is 0 for FeeFilterRounder (morcos) +- #9395 `0fc1c31` Add test for `-walletrejectlongchains` (morcos) +- #9107 `7dac1e5` Safer modify new coins (morcos) +- #9312 `a72f76c` Increase mempool expiry time to 2 weeks (morcos) +- #8610 `c252685` Share unused mempool memory with coincache (sipa) +- #9138 `f646275` Improve fee estimation (morcos) +- #9408 `46b249e` Allow shutdown during LoadMempool, dump only when necessary (jonasschnelli) +- #9310 `8c87f17` Assert FRESH validity in CCoinsViewCache::BatchWrite (ryanofsky) +- #7871 `e2e624d` Manual block file pruning (mrbandrews) +- #9507 `0595042` Fix use-after-free in CTxMemPool::removeConflicts() (sdaftuar) +- #9380 `dd98f04` Separate different uses of minimum fees (morcos) +- #9596 `71148b8` bugfix save feeDelta instead of priorityDelta in DumpMempool (morcos) +- #9371 `4a1dc35` Notify on removal (morcos) +- #9519 `9b4d267` Exclude RBF replacement txs from fee estimation (morcos) +- #8606 `e2a1a1e` Fix some locks (sipa) +- #8681 `6898213` Performance Regression Fix: Pre-Allocate txChanged vector (JeremyRubin) +- #8223 `744d265` c++11: Use std::unique\_ptr for block creation (domob1812) +- #9125 `7490ae8` Make CBlock a vector of shared\_ptr of CTransactions (sipa) +- #8930 `93566e0` Move orphan processing to ActivateBestChain (TheBlueMatt) +- #8580 `46904ee` Make CTransaction actually immutable (sipa) +- #9240 `a1dcf2e` Remove txConflicted (morcos) +- #8589 `e8cfe1e` Inline CTxInWitness inside CTxIn (sipa) +- #9349 `2db4cbc` Make CScript (and prevector) c++11 movable (sipa) +- #9252 `ce5c1f4` Release cs\_main before calling ProcessNewBlock, or processing headers (cmpctblock handling) (sdaftuar) +- #9283 `869781c` A few more CTransactionRef optimizations (sipa) +- #9499 `9c9af5a` Use recent-rejects, orphans, and recently-replaced txn for compact-block-reconstruction (TheBlueMatt) +- #9813 `3972a8e` Read/write mempool.dat as a binary (paveljanik) + +### P2P protocol and network code +- #8128 `1030fa7` Turn net structures into dumb storage classes (theuni) +- #8282 `026c6ed` Feeler connections to increase online addrs in the tried table (EthanHeilman) +- #8462 `53f8f22` Move AdvertiseLocal debug output to net category (Mirobit) +- #8612 `84decb5` Check for compatibility with download in FindNextBlocksToDownload (sipa) +- #8594 `5b2ea29` Do not add random inbound peers to addrman (gmaxwell) +- #8085 `6423116` Begin encapsulation (theuni) +- #8715 `881d7ea` only delete CConnman if it's been created (theuni) +- #8707 `f07424a` Fix maxuploadtarget setting (theuni) +- #8661 `d2e4655` Do not set an addr time penalty when a peer advertises itself (gmaxwell) +- #8822 `9bc6a6b` Consistent checksum handling (laanwj) +- #8936 `1230890` Report NodeId in misbehaving debug (rebroad) +- #8968 `3cf496d` Don't hold cs\_main when calling ProcessNewBlock from a cmpctblock (TheBlueMatt) +- #9002 `e1d1f57` Make connect=0 disable automatic outbound connections (gmaxwell) +- #9050 `fcf61b8` Make a few values immutable, and use deterministic randomness for the localnonce (theuni) +- #8969 `3665483` Decouple peer-processing-logic from block-connection-logic (#2) (TheBlueMatt) +- #8708 `c8c572f` have CConnman handle message sending (theuni) +- #8709 `1e50d22` Allow filterclear messages for enabling TX relay only (rebroad) +- #9045 `9f554e0` Hash P2P messages as they are received instead of at process-time (TheBlueMatt) +- #9026 `dc6b940` Fix handling of invalid compact blocks (sdaftuar) +- #8996 `ab914a6` Network activity toggle (luke-jr) +- #9131 `62af164` fNetworkActive is not protected by a lock, use an atomic (jonasschnelli) +- #8872 `0c577f2` Remove block-request logic from INV message processing (TheBlueMatt) +- #8690 `791b58d` Do not fully sort all nodes for addr relay (sipa) +- #9128 `76fec09` Decouple CConnman and message serialization (theuni) +- #9226 `3bf06e9` Remove fNetworkNode and pnodeLocalHost (gmaxwell) +- #9352 `a7f7651` Attempt reconstruction from all compact block announcements (sdaftuar) +- #9319 `a55716a` Break addnode out from the outbound connection limits (gmaxwell) +- #9261 `2742568` Add unstored orphans with rejected parents to recentRejects (morcos) +- #9441 `8b66bf7` Massive speedup. Net locks overhaul (theuni) +- #9375 `3908fc4` Relay compact block messages prior to full block connection (TheBlueMatt) +- #9400 `8a445c5` Set peers as HB peers upon full block validation (instagibbs) +- #9561 `6696b46` Wake message handling thread when we receive a new block (TheBlueMatt) +- #9535 `82274c0` Split CNode::cs\_vSend: message processing and message sending (TheBlueMatt) +- #9606 `3f9f962` Consistently use GetTimeMicros() for inactivity checks (sdaftuar) +- #9594 `fd70211` Send final alert message to older peers after connecting (gmaxwell) +- #9626 `36966a1` Clean up a few CConnman cs\_vNodes/CNode things (TheBlueMatt) +- #9609 `4966917` Fix remaining net assertions (theuni) +- #9671 `7821db3` Fix super-unlikely race introduced in 236618061a445d2cb11e72 (TheBlueMatt) +- #9730 `33f3b21` Remove bitseed.xf2.org form the dns seed list (jonasschnelli) +- #9698 `2447c10` Fix socket close race (theuni) +- #9708 `a06ede9` Clean up all known races/platform-specific UB at the time PR was opened (TheBlueMatt) +- #9715 `b08656e` Disconnect peers which we do not receive VERACKs from within 60 sec (TheBlueMatt) +- #9720 `e87ce95` Fix banning and disallow sending messages before receiving verack (theuni) +- #9268 `09c4fd1` Fix rounding privacy leak introduced in #9260 (TheBlueMatt) +- #9075 `9346f84` Decouple peer-processing-logic from block-connection-logic (#3) (TheBlueMatt) +- #8688 `047ded0` Move static global randomizer seeds into CConnman (sipa) +- #9289 `d9ae1ce` net: drop boost::thread\_group (theuni) + +### Validation +- #9014 `d04aeba` Fix block-connection performance regression (TheBlueMatt) +- #9299 `d52ce89` Remove no longer needed check for premature v2 txs (morcos) +- #9273 `b68685a` Remove unused `CDiskBlockPos*` argument from ProcessNewBlock (TheBlueMatt) +- #8895 `b83264d` Better SigCache Implementation (JeremyRubin) +- #9490 `e126d0c` Replace FindLatestBefore used by importmulti with FindEarliestAtLeast (gmaxwell) +- #9484 `812714f` Introduce assumevalid setting to skip validation presumed valid scripts (gmaxwell) +- #9511 `7884956` Don't overwrite validation state with corruption check (morcos) +- #9765 `1e92e04` Harden against mistakes handling invalid blocks (sdaftuar) +- #9779 `3c02b95` Update nMinimumChainWork and defaultAssumeValid (gmaxwell) +- #8524 `19b0f33` Precompute sighashes (sipa) +- #9791 `1825a03` Avoid VLA in hash.h (sipa) + +### Build system +- #8238 `6caf3ee` ZeroMQ 4.1.5 && ZMQ on Windows (fanquake) +- #8520 `b40e19c` Remove check for `openssl/ec.h` (laanwj) +- #8617 `de07fdc` Include instructions to extract Mac OS X SDK on Linux using 7zip and SleuthKit (luke-jr) +- #8566 `7b98895` Easy to use gitian building script (achow101) +- #8604 `f256843` build,doc: Update for 0.13.0+ and OpenBSD 5.9 (laanwj) +- #8640 `2663e51` depends: Remove Qt46 package (fanquake) +- #8645 `8ea4440` Remove unused Qt 4.6 patch (droark) +- #8608 `7e9ab95` Install manpages via make install, also add some autogenerated manpages (nomnombtc) +- #8781 `ca69ef4` contrib: delete `qt_translations.py` (MarcoFalke) +- #8783 `64dc645` share: remove qt/protobuf.pri (MarcoFalke) +- #8423 `3166dff` depends: expat 2.2.0, ccache 3.3.1, fontconfig 2.12.1 (fanquake) +- #8791 `b694b0d` travis: cross-mac: explicitly enable gui (MarcoFalke) +- #8820 `dc64141` depends: Fix Qt compilation with Xcode 8 (fanquake) +- #8730 `489a6ab` depends: Add libevent compatibility patch for windows (laanwj) +- #8819 `c841816` depends: Boost 1.61.0 (fanquake) +- #8826 `f560d95` Do not include `env_win.cc` on non-Windows systems (paveljanik) +- #8948 `e077e00` Reorder Windows gitian build order to match Linux (Michagogo) +- #8568 `078900d` new var `DIST_CONTRIB` adds useful things for packagers from contrib (nomnombtc) +- #9114 `21e6c6b` depends: Set `OSX_MIN_VERSION` to 10.8 (fanquake) +- #9140 `018a4eb` Bugfix: Correctly replace generated headers and fail cleanly (luke-jr) +- #9156 `a8b2a82` Add compile and link options echo to configure (jonasschnelli) +- #9393 `03d85f6` Include cuckoocache header in Makefile (MarcoFalke) +- #9420 `bebe369` Fix linker error when configured with --enable-lcov (droark) +- #9412 `53442af` Fix 'make deploy' for OSX (jonasschnelli) +- #9475 `7014506` Let autoconf detect presence of `EVP_MD_CTX_new` (luke-jr) +- #9513 `bbf193f` Fix qt distdir builds (theuni) +- #9471 `ca615e6` depends: libevent 2.1.7rc (fanquake) +- #9468 `f9117f2` depends: Dependency updates for 0.14.0 (fanquake) +- #9469 `01c4576` depends: Qt 5.7.1 (fanquake) +- #9574 `5ac6687` depends: Fix QT build on OSX (fanquake) +- #9646 `720b579` depends: Fix cross build for qt5.7 (theuni) +- #9705 `6a55515` Add options to override BDB cflags/libs (laanwj) +- #8249 `4e1567a` Enable (and check for) 64-bit ASLR on Windows (laanwj) +- #9758 `476cc47` Selectively suppress deprecation warnings (jonasschnelli) +- #9783 `6d61a2b` release: bump gitian descriptors for a new 0.14 package cache (theuni) +- #9789 `749fe95` build: add --enable-werror and warn on vla's (theuni) +- #9831 `99fd85c` build: force a c++ standard to be specified (theuni) + +### GUI +- #8192 `c503863` Remove URLs from About dialog translations (fanquake) +- #8540 `36404ae` Fix random segfault when closing "Choose data directory" dialog (laanwj) +- #8517 `2468292` Show wallet HD state in statusbar (jonasschnelli) +- #8463 `62a5a8a` Remove Priority from coincontrol dialog (MarcoFalke) +- #7579 `0606f95` Show network/chain errors in the GUI (jonasschnelli) +- #8583 `c19f8a4` Show XTHIN in GUI (rebroad) +- #7783 `4335d5a` RPC-Console: support nested commands and simple value queries (jonasschnelli) +- #8672 `6052d50` Show transaction size in transaction details window (Cocosoft) +- #8777 `fec6af7` WalletModel: Expose disablewallet (MarcoFalke) +- #8371 `24f72e9` Add out-of-sync modal info layer (jonasschnelli) +- #8885 `b2fec4e` Fix ban from qt console (theuni) +- #8821 `bf8e68a` sync-overlay: Don't block during reindex (MarcoFalke) +- #8906 `088d1f4` sync-overlay: Don't show progress twice (MarcoFalke) +- #8918 `47ace42` Add "Copy URI" to payment request context menu (luke-jr) +- #8925 `f628d9a` Display minimum ping in debug window (rebroad) +- #8774 `3e942a7` Qt refactors to better abstract wallet access (luke-jr) +- #8985 `7b1bfa3` Use pindexBestHeader instead of setBlockIndexCandidates for NotifyHeaderTip() (jonasschnelli) +- #8989 `d2143dc` Overhaul smart-fee slider, adjust default confirmation target (jonasschnelli) +- #9043 `273bde3` Return useful error message on ATMP failure (MarcoFalke) +- #9088 `4e57824` Reduce ambiguity of warning message (rebroad) +- #8874 `e984730` Multiple Selection for peer and ban tables (achow101) +- #9145 `924745d` Make network disabled icon 50% opaque (MarcoFalke) +- #9130 `ac489b2` Mention the new network toggle functionality in the tooltip (paveljanik) +- #9218 `4d955fc` Show progress overlay when clicking spinner icon (laanwj) +- #9280 `e15660c` Show ModalOverlay by pressing the progress bar, allow hiding (jonasschnelli) +- #9296 `fde7d99` Fix missed change to WalletTx structure (morcos) +- #9266 `2044e37` Bugfix: Qt/RPCConsole: Put column enum in the right places (luke-jr) +- #9255 `9851a84` layoutAboutToChange signal is called layoutAboutToBeChanged (laanwj) +- #9330 `47e6a19` Console: add security warning (jonasschnelli) +- #9329 `db45ad8` Console: allow empty arguments (jonasschnelli) +- #8877 `6dc4c43` Qt RPC console: history sensitive-data filter, and saving input line when browsing history (luke-jr) +- #9462 `649cf5f` Do not translate tilde character (MarcoFalke) +- #9457 `123ea73` Select more files for translation (MarcoFalke) +- #9413 `fd7d8c7` CoinControl: Allow non-wallet owned change addresses (jonasschnelli) +- #9461 `b250686` Improve progress display during headers-sync and peer-finding (jonasschnelli) +- #9588 `5086452` Use nPowTargetSpacing constant (MarcoFalke) +- #9637 `d9e4d1d` Fix transaction details output-index to reflect vout index (jonasschnelli) +- #9718 `36f9d3a` Qt/Intro: Various fixes (luke-jr) +- #9735 `ec66d06` devtools: Handle Qt formatting characters edge-case in update-translations.py (laanwj) +- #9755 `a441db0` Bugfix: Qt/Options: Restore persistent "restart required" notice (luke-jr) +- #9817 `7d75a5a` Fix segfault crash when shutdown the GUI in disablewallet mode (jonasschnelli) + +### Wallet +- #8152 `b9c1cd8` Remove `CWalletDB*` parameter from CWallet::AddToWallet (pstratem) +- #8432 `c7e05b3` Make CWallet::fFileBacked private (pstratem) +- #8445 `f916700` Move CWallet::setKeyPool to private section of CWallet (pstratem) +- #8564 `0168019` Remove unused code/conditions in ReadAtCursor (jonasschnelli) +- #8601 `37ac678` Add option to opt into full-RBF when sending funds (rebase, original by petertodd) (laanwj) +- #8494 `a5b20ed` init, wallet: ParameterInteraction() iff wallet enabled (MarcoFalke) +- #8760 `02ac669` init: Get rid of some `ENABLE_WALLET` (MarcoFalke) +- #8696 `a1f8d3e` Wallet: Remove last external reference to CWalletDB (pstratem) +- #8768 `886e8c9` init: Get rid of fDisableWallet (MarcoFalke) +- #8486 `ab0b411` Add high transaction fee warnings (MarcoFalke) +- #8851 `940748b` Move key derivation logic from GenerateNewKey to DeriveNewChildKey (pstratem) +- #8287 `e10af96` Set fLimitFree = true (MarcoFalke) +- #8928 `c587577` Fix init segfault where InitLoadWallet() calls ATMP before genesis (TheBlueMatt) +- #7551 `f2d7056` Add importmulti RPC call (pedrobranco) +- #9016 `0dcb888` Return useful error message on ATMP failure (instagibbs) +- #8753 `f8723d2` Locked memory manager (laanwj) +- #8828 `a4fd8df` Move CWalletDB::ReorderTransactions to CWallet (pstratem) +- #8977 `6a1343f` Refactor wallet/init interaction (Reaccept wtx, flush thread) (jonasschnelli) +- #9036 `ed0cc50` Change default confirm target from 2 to 6 (laanwj) +- #9071 `d1871da` Declare wallet.h functions inline (sipa) +- #9132 `f54e460` Make strWalletFile const (jonasschnelli) +- #9141 `5ea5e04` Remove unnecessary calls to CheckFinalTx (jonasschnelli) +- #9165 `c01f16a` SendMoney: use already-calculated balance (instagibbs) +- #9311 `a336d13` Flush wallet after abandontransaction (morcos) +- #8717 `38e4887` Addition of ImmatureCreditCached to MarkDirty() (spencerlievens) +- #9446 `510c0d9` SetMerkleBranch: remove unused code, remove cs\_main lock requirement (jonasschnelli) +- #8776 `2a524b8` Wallet refactoring leading up to multiwallet (luke-jr) +- #9465 `a7d55c9` Do not perform ECDSA signing in the fee calculation inner loop (gmaxwell) +- #9404 `12e3112` Smarter coordination of change and fee in CreateTransaction (morcos) +- #9377 `fb75cd0` fundrawtransaction: Keep change-output keys by default, make it optional (jonasschnelli) +- #9578 `923dc44` Add missing mempool lock for CalculateMemPoolAncestors (TheBlueMatt) +- #9227 `02464da` Make nWalletDBUpdated atomic to avoid a potential race (pstratem) +- #9764 `f8af89a` Prevent "overrides a member function but is not marked 'override'" warnings (laanwj) +- #9771 `e43a585` Add missing cs\_wallet lock that triggers new lock held assertion (ryanofsky) +- #9316 `3097ea4` Disable free transactions when relay is disabled (MarcoFalke) +- #9615 `d2c9e4d` Wallet incremental fee (morcos) +- #9760 `40c754c` Remove importmulti always-true check (ryanofsky) + +### Tests and QA +- #8270 `6e5e5ab` Tests: Use portable #! in python scripts (/usr/bin/env) (ChoHag) +- #8534,#8504 Remove java comparison tool (laanwj,MarcoFalke) +- #8482 `740cff5` Use single cache dir for chains (MarcoFalke) +- #8450 `21857d2` Replace `rpc_wallet_tests.cpp` with python RPC unit tests (pstratem) +- #8671 `ddc3080` Minimal fix to slow prevector tests as stopgap measure (JeremyRubin) +- #8680 `666eaf0` Address Travis spurious failures (theuni) +- #8789 `e31a43c` pull-tester: Only print output when failed (MarcoFalke) +- #8810 `14e8f99` tests: Add exception error message for JSONRPCException (laanwj) +- #8830 `ef0801b` test: Add option to run bitcoin-util-test.py manually (jnewbery) +- #8881 `e66cc1d` Add some verbose logging to bitcoin-util-test.py (jnewbery) +- #8922 `0329511` Send segwit-encoded blocktxn messages in p2p-compactblocks (TheBlueMatt) +- #8873 `74dc388` Add microbenchmarks to profile more code paths (ryanofsky) +- #9032 `6a8be7b` test: Add format-dependent comparison to bctest (laanwj) +- #9023 `774db92` Add logging to bitcoin-util-test.py (jnewbery) +- #9065 `c9bdf9a` Merge `doc/unit-tests.md` into `src/test/README.md` (laanwj) +- #9069 `ed64bce` Clean up bctest.py and bitcoin-util-test.py (jnewbery) +- #9095 `b8f43e3` test: Fix test\_random includes (MarcoFalke) +- #8894 `faec09b` Testing: Include fRelay in mininode version messages (jnewbery) +- #9097 `e536499` Rework `sync_*` and preciousblock.py (MarcoFalke) +- #9049 `71bc39e` Remove duplicatable duplicate-input check from CheckTransaction (TheBlueMatt) +- #9136 `b422913` sync\_blocks cleanup (ryanofsky) +- #9151 `4333b1c` proxy\_test: Calculate hardcoded port numbers (MarcoFalke) +- #9206 `e662d28` Make test constant consistent with consensus.h (btcdrak) +- #9139 `0de7fd3` Change sync\_blocks to pick smarter maxheight (on top of #9196) (ryanofsky) +- #9100 `97ec6e5` tx\_valid: re-order inputs to how they are encoded (dcousens) +- #9202 `e56cf67` bench: Add support for measuring CPU cycles (laanwj) +- #9223 `5412c08` unification of Bloom filter representation (s-matthew-english) +- #9257 `d7ba4a2` Dump debug logs on travis failures (sdaftuar) +- #9221 `9e4bb31` Get rid of duplicate code (MarcoFalke) +- #9274 `919db03` Use cached utxo set to fix performance regression (MarcoFalke) +- #9276 `ea33f19` Some minor testing cleanups (morcos) +- #9291 `8601784` Remove mapOrphanTransactionsByPrev from DoS\_tests (sipa) +- #9309 `76fcd9d` Wallet needs to stay unlocked for whole test (morcos) +- #9172 `5bc209c` Resurrect pstratem's "Simple fuzzing framework" (laanwj) +- #9331 `c6fd923` Add test for rescan feature of wallet key import RPCs (ryanofsky) +- #9354 `b416095` Make fuzzer actually test CTxOutCompressor (sipa) +- #9390,#9416 travis: make distdir (MarcoFalke) +- #9308 `0698639` test: Add CCoinsViewCache Access/Modify/Write tests (ryanofsky) +- #9406 `0f921e6` Re-enable a blank v1 Tx JSON test (droark) +- #9435 `dbc8a8c` Removed unused variable in test, fixing warning (ryanofsky) +- #9436 `dce853e` test: Include tx data in `EXTRA_DIST` (MarcoFalke) +- #9525 `02e5308` test: Include tx data in `EXTRA_DIST` (MarcoFalke) +- #9498 `054d664` Basic CCheckQueue Benchmarks (JeremyRubin) +- #9554 `0b96abc` test: Avoid potential NULL pointer dereference in `addrman_tests.cpp` (practicalswift) +- #9628 `f895023` Increase a sync\_blocks timeout in pruning.py (sdaftuar) +- #9638 `a7ea2f8` Actually test assertions in pruning.py (MarcoFalke) +- #9647 `e99f0d7` Skip RAII event tests if libevent is built without `event_set_mem_functions` (luke-jr) +- #9691 `fc67cd2` Init ECC context for `test_bitcoin_fuzzy` (gmaxwell) +- #9712 `d304fef` bench: Fix initialization order in registration (laanwj) +- #9707 `b860915` Fix RPC failure testing (jnewbery) +- #9269 `43e8150` Align struct COrphan definition (sipa) +- #9820 `599c69a` Fix pruning test broken by 2 hour manual prune window (ryanofsky) +- #9824 `260c71c` qa: Check return code when stopping nodes (MarcoFalke) +- #9875 `50953c2` tests: Fix dangling pwalletMain pointer in wallet tests (laanwj) +- #9839 `eddaa6b` [qa] Make import-rescan.py watchonly check reliable (ryanofsky) + +### Documentation +- #8332 `806b9e7` Clarify witness branches in transaction.h serialization (dcousens) +- #8935 `0306978` Documentation: Building on Windows with WSL (pooleja) +- #9144 `c98f6b3` Correct waitforblockheight example help text (fanquake) +- #9407 `041331e` Added missing colons in when running help command (anditto) +- #9378 `870cd2b` Add documentation for CWalletTx::fFromMe member (ryanofsky) +- #9297 `0b73807` Various RPC help outputs updated (Mirobit) +- #9613 `07421cf` Clarify getbalance help string to explain interaction with bumpfee (ryanofsky) +- #9663 `e30d928` Clarify listunspent amount description (instagibbs) +- #9396 `d65a13b` Updated listsinceblock rpc documentation (accraze) +- #8747 `ce43630` rpc: Fix transaction size comments and RPC help text (jnewbery) +- #8058 `bbd9740` Doc: Add issue template (AmirAbrams) +- #8567 `85d4e21` Add default port numbers to REST doc (djpnewton) +- #8624 `89de153` build: Mention curl (MarcoFalke) +- #8786 `9da7366` Mandatory copyright agreement (achow101) +- #8823 `7b05af6` Add privacy recommendation when running hidden service (laanwj) +- #9433 `caa2f10` Update the Windows build notes (droark) +- #8879 `f928050` Rework docs (MarcoFalke) +- #8887 `61d191f` Improve GitHub issue template (fanquake) +- #8787 `279bbad` Add missing autogen to example builds (AmirAbrams) +- #8892 `d270c30` Add build instructions for FreeBSD (laanwj) +- #8890 `c71a654` Update Doxygen configuration file (fanquake) +- #9207 `fa1f944` Move comments above bash command in build-unix (AmirAbrams) +- #9219 `c4522e7` Improve windows build instructions using Linux subsystem (laanwj) +- #8954 `932d02a` contrib: Add README for pgp keys (MarcoFalke) +- #9093 `2fae5b9` release-process: Mention GitHub release and archived release notes (MarcoFalke) +- #8743 `bae178f` Remove old manpages from contrib/debian in favour of doc/man (fanquake) +- #9550 `4105cb6` Trim down the XP notice and say more about what we support (gmaxwell) +- #9246 `9851498` Developer docs about existing subtrees (gmaxwell) +- #9401 `c2ea1e6` Make rpcauth help message clearer, add example in example .conf (instagibbs) +- #9022,#9033 Document dropping OS X 10.7 support (fanquake, MarcoFalke) +- #8771 `bc9e3ab` contributing: Mention not to open several pulls (luke-jr) +- #8852 `7b784cc` Mention Gitian building script in doc (Laudaa) (laanwj) +- #8915 `03dd707` Add copyright/patent issues to possible NACK reasons (petertodd) +- #8965 `23e03f8` Mention that PPA doesn't support Debian (anduck) +- #9115 `bfc7aad` Mention reporting security issues responsibly (paveljanik) +- #9840 `08e0690` Update sendfrom RPC help to correct coin selection misconception (ryanofsky) +- #9865 `289204f` Change bitcoin address in RPC help message (marijnfs) + +### Miscellaneous +- #8274 `7a2d402` util: Update tinyformat (laanwj) +- #8291 `5cac8b1` util: CopyrightHolders: Check for untranslated substitution (MarcoFalke) +- #8557 `44691f3` contrib: Rework verifybinaries (MarcoFalke) +- #8621 `e8ed6eb` contrib: python: Don't use shell=True (MarcoFalke) +- #8813 `fb24d7e` bitcoind: Daemonize using daemon(3) (laanwj) +- #9004 `67728a3` Clarify `listenonion` (unsystemizer) +- #8674 `bae81b8` tools for analyzing, updating and adding copyright headers in source files (isle2983) +- #8976 `8c6218a` libconsensus: Add input validation of flags (laanwj) +- #9112 `46027e8` Avoid ugly exception in log on unknown inv type (laanwj) +- #8837 `2108911` Allow bitcoin-tx to parse partial transactions (jnewbery) +- #9204 `74ced54` Clarify CreateTransaction error messages (instagibbs) +- #9265 `31bcc66` bitcoin-cli: Make error message less confusing (laanwj) +- #9303 `72bf1b3` Update comments in ctaes (sipa) +- #9417 `c4b7d4f` Do not evaluate hidden LogPrint arguments (sipa) +- #9506 `593a00c` RFC: Improve style for if indentation (sipa) +- #8883 `d5d4ad8` Add all standard TXO types to bitcoin-tx (jnewbery) +- #9531 `23281a4` Release notes for estimation changes (morcos) +- #9486 `f62bc10` Make peer=%d log prints consistent (TheBlueMatt) +- #9552 `41cb05c` Add IPv6 support to qos.sh (jamesmacwhite) +- #9542 `e9e7993` Docs: Update CONTRIBUTING.md (jnewbery) +- #9649 `53ab12d` Remove unused clang format dev script (MarcoFalke) +- #9625 `77bd8c4` Increase minimum debug.log size to 10MB after shrink (morcos) +- #9070 `7b22e50` Lockedpool fixes (kazcw) +- #8779 `7008e28` contrib: Delete spendfrom (MarcoFalke) +- #9587,#8793,#9496,#8191,#8109,#8655,#8472,#8677,#8981,#9124 Avoid shadowing of variables (paveljanik) +- #9063 `f2a6e82` Use deprecated `MAP_ANON` if `MAP_ANONYMOUS` is not defined (paveljanik) +- #9060 `1107653` Fix bloom filter init to isEmpty = true (robmcl4) +- #8613 `613bda4` LevelDB 1.19 (sipa) +- #9225 `5488514` Fix some benign races (TheBlueMatt) +- #8736 `5fa7b07` base58: Improve DecodeBase58 performance (wjx) +- #9039 `e81df49` Various serialization simplifcations and optimizations (sipa) +- #9010 `a143b88` Split up AppInit2 into multiple phases, daemonize after datadir lock errors (laanwj) +- #9230 `c79e52a` Fix some benign races in timestamp logging (TheBlueMatt) +- #9183,#9260 Mrs Peacock in The Library with The Candlestick (killed main.{h,cpp}) (TheBlueMatt) +- #9236 `7f72568` Fix races for strMiscWarning and `fLargeWork*Found`, make QT runawayException use GetWarnings (gmaxwell) +- #9243 `7aa7004` Clean up mapArgs and mapMultiArgs Usage (TheBlueMatt) +- #9387 `cfe41d7` RAII of libevent stuff using unique ptrs with deleters (kallewoof) +- #9472 `fac0f30` Disentangle progress estimation from checkpoints and update it (sipa) +- #9512 `6012967` Fix various things -fsanitize complains about (sipa) +- #9373,#9580 Various linearization script issues (droark) +- #9674 `dd163f5` Lock debugging: Always enforce strict lock ordering (try or not) (TheBlueMatt) +- #8453,#9334 Update to latest libsecp256k1 (laanwj,sipa) +- #9656 `7c93952` Check verify-commits on pushes to master (TheBlueMatt) +- #9679 `a351162` Access WorkQueue::running only within the cs lock (TheBlueMatt) +- #9777 `8dee822` Handle unusual maxsigcachesize gracefully (jnewbery) +- #8863,#8807 univalue: Pull subtree (MarcoFalke) +- #9798 `e22c067` Fix Issue #9775 (Check returned value of fopen) (kirit93) +- #9856 `69832aa` Terminate immediately when allocation fails (theuni) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- accraze +- adlawren +- Alex Morcos +- Alexey Vesnin +- Amir Abrams +- Anders Øyvind Urke-Sætre +- Anditto Heristyo +- Andrew Chow +- anduck +- Anthony Towns +- Brian Deery +- BtcDrak +- Chris Moore +- Chris Stewart +- Christian Barcenas +- Christian Decker +- Cory Fields +- crowning- +- CryptAxe +- CryptoVote +- Dagur Valberg Johannsson +- Daniel Cousens +- Daniel Kraft +- Derek Miller +- djpnewton +- Don Patterson +- Doug +- Douglas Roark +- Ethan Heilman +- fsb4000 +- Gaurav Rana +- Geoffrey Tsui +- Greg Walker +- Gregory Maxwell +- Gregory Sanders +- Hampus Sjöberg +- isle2983 +- Ivo van der Sangen +- James White +- Jameson Lopp +- Jeremy Rubin +- Jiaxing Wang +- jnewbery +- John Newbery +- Johnson Lau +- Jon Lund Steffensen +- Jonas Schnelli +- jonnynewbs +- Jorge Timón +- Justin Camarena +- Karl-Johan Alm +- Kaz Wesley +- kirit93 +- Koki Takahashi +- Lauda +- leijurv +- lizhi +- Luke Dashjr +- maiiz +- MarcoFalke +- Marijn Stollenga +- Marty Jones +- Masahiko Hyuga +- Matt Corallo +- Matthew King +- matthias +- Micha +- Michael Ford +- Michael Rotarius +- Mitchell Cash +- mrbandrews +- mruddy +- Nicolas DORIER +- nomnombtc +- Patrick Strateman +- Pavel JanÃk +- Pedro Branco +- Peter Todd +- Pieter Wuille +- poole\_party +- practicalswift +- R E Broadley +- randy-waterhouse +- Richard Kiss +- Robert McLaughlin +- rodasmith +- Russell Yanofsky +- S. Matthew English +- Sev +- Spencer Lievens +- Stanislas Marion +- Steven +- Suhas Daftuar +- Thomas Snider +- UdjinM6 +- unsystemizer +- whythat +- Will Binns +- Wladimir J. van der Laan +- wodry +- Zak Wilcox + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/doc/release-notes/release-notes-0.14.1.md b/doc/release-notes/release-notes-0.14.1.md new file mode 100644 index 0000000000..ef072afd4d --- /dev/null +++ b/doc/release-notes/release-notes-0.14.1.md @@ -0,0 +1,143 @@ +Bitcoin Core version 0.14.1 is now available from: + + <https://bitcoin.org/bin/bitcoin-core-0.14.1/> + +This is a new minor version release, including various bugfixes and +performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at github: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +Compatibility +============== + +Bitcoin Core is extensively tested on multiple operating systems using +the Linux kernel, macOS 10.8+, and Windows Vista and later. + +Microsoft ended support for Windows XP on [April 8th, 2014](https://www.microsoft.com/en-us/WindowsForBusiness/end-of-xp-support), +No attempt is made to prevent installing or running the software on Windows XP, you +can still do so at your own risk but be aware that there are known instabilities and issues. +Please do not report issues about Windows XP to the issue tracker. + +Bitcoin Core should also work on most other Unix-like systems but is not +frequently tested on them. + +Notable changes +=============== + +RPC changes +----------- + +- The first positional argument of `createrawtransaction` was renamed from + `transactions` to `inputs`. + +- The argument of `disconnectnode` was renamed from `node` to `address`. + +These interface changes break compatibility with 0.14.0, when the named +arguments functionality, introduced in 0.14.0, is used. Client software +using these calls with named arguments needs to be updated. + +Mining +------ + +In previous versions, getblocktemplate required segwit support from downstream +clients/miners once the feature activated on the network. In this version, it +now supports non-segwit clients even after activation, by removing all segwit +transactions from the returned block template. This allows non-segwit miners to +continue functioning correctly even after segwit has activated. + +Due to the limitations in previous versions, getblocktemplate also recommended +non-segwit clients to not signal for the segwit version-bit. Since this is no +longer an issue, getblocktemplate now always recommends signalling segwit for +all miners. This is safe because ability to enforce the rule is the only +required criteria for safe activation, not actually producing segwit-enabled +blocks. + +UTXO memory accounting +---------------------- + +Memory usage for the UTXO cache is being calculated more accurately, so that +the configured limit (`-dbcache`) will be respected when memory usage peaks +during cache flushes. The memory accounting in prior releases is estimated to +only account for half the actual peak utilization. + +The default `-dbcache` has also been changed in this release to 450MiB. Users +who currently set `-dbcache` to a high value (e.g. to keep the UTXO more fully +cached in memory) should consider increasing this setting in order to achieve +the same cache performance as prior releases. Users on low-memory systems +(such as systems with 1GB or less) should consider specifying a lower value for +this parameter. + +Additional information relating to running on low-memory systems can be found +here: +[reducing-bitcoind-memory-usage.md](https://gist.github.com/laanwj/efe29c7661ce9b6620a7). + +0.14.1 Change log +================= + +Detailed release notes follow. This overview includes changes that affect +behavior, not code moves, refactors and string updates. For convenience in locating +the code changes and accompanying discussion, both the pull request and +git merge commit are mentioned. + +### RPC and other APIs +- #10084 `142fbb2` Rename first named arg of createrawtransaction (MarcoFalke) +- #10139 `f15268d` Remove auth cookie on shutdown (practicalswift) +- #10146 `2fea10a` Better error handling for submitblock (rawodb, gmaxwell) +- #10144 `d947afc` Prioritisetransaction wasn't always updating ancestor fee (sdaftuar) +- #10204 `3c79602` Rename disconnectnode argument (jnewbery) + +### Block and transaction handling +- #10126 `0b5e162` Compensate for memory peak at flush time (sipa) +- #9912 `fc3d7db` Optimize GetWitnessHash() for non-segwit transactions (sdaftuar) +- #10133 `ab864d3` Clean up calculations of pcoinsTip memory usage (morcos) + +### P2P protocol and network code +- #9953/#10013 `d2548a4` Fix shutdown hang with >= 8 -addnodes set (TheBlueMatt) +- #10176 `30fa231` net: gracefully handle NodeId wrapping (theuni) + +### Build system +- #9973 `e9611d1` depends: fix zlib build on osx (theuni) + +### GUI +- #10060 `ddc2dd1` Ensure an item exists on the rpcconsole stack before adding (achow101) + +### Mining +- #9955/#10006 `569596c` Don't require segwit in getblocktemplate for segwit signalling or mining (sdaftuar) +- #9959/#10127 `b5c3440` Prevent slowdown in CreateNewBlock on large mempools (sdaftuar) + +### Tests and QA +- #10157 `55f641c` Fix the `mempool_packages.py` test (sdaftuar) + +### Miscellaneous +- #10037 `4d8e660` Trivial: Fix typo in help getrawtransaction RPC (keystrike) +- #10120 `e4c9a90` util: Work around (virtual) memory exhaustion on 32-bit w/ glibc (laanwj) +- #10130 `ecc5232` bitcoin-tx input verification (awemany, jnewbery) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Alex Morcos +- Andrew Chow +- Awemany +- Cory Fields +- Gregory Maxwell +- James Evans +- John Newbery +- MarcoFalke +- Matt Corallo +- Pieter Wuille +- practicalswift +- rawodb +- Suhas Daftuar +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). + diff --git a/doc/release-process.md b/doc/release-process.md index 399ed25c91..5a99b726f1 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -10,7 +10,7 @@ Before every release candidate: Before every minor and major release: * Update [bips.md](bips.md) to account for changes since the last release. -* Update version in sources (see below) +* Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_IS_RELEASE` to `true`) * Write release notes (see below) * Update `src/chainparams.cpp` nMinimumChainWork with information from the getblockchaininfo rpc. * Update `src/chainparams.cpp` defaultAssumeValid with information from the getblockhash rpc. @@ -23,6 +23,8 @@ Before every major release: * Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/7415) for an example. * Update [`BLOCK_CHAIN_SIZE`](/src/qt/intro.cpp) to the current size plus some overhead. +* Update `src/chainparams.cpp` chainTxData with statistics about the transaction count and rate. +* Update version of `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release ### First time / New builders @@ -36,23 +38,7 @@ Check out the source code in the following directory hierarchy. git clone https://github.com/devrandom/gitian-builder.git git clone https://github.com/bitcoin/bitcoin.git -### Bitcoin maintainers/release engineers, update version in sources - -Update the following: - -- `configure.ac`: - - `_CLIENT_VERSION_MAJOR` - - `_CLIENT_VERSION_MINOR` - - `_CLIENT_VERSION_REVISION` - - Don't forget to set `_CLIENT_VERSION_IS_RELEASE` to `true` -- `src/clientversion.h`: (this mirrors `configure.ac` - see issue #3539) - - `CLIENT_VERSION_MAJOR` - - `CLIENT_VERSION_MINOR` - - `CLIENT_VERSION_REVISION` - - Don't forget to set `CLIENT_VERSION_IS_RELEASE` to `true` -- `doc/README.md` and `doc/README_windows.txt` -- `doc/Doxyfile`: `PROJECT_NUMBER` contains the full version -- `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release +### Bitcoin maintainers/release engineers, suggestion for writing release notes Write release notes. git shortlog helps a lot, for example: @@ -175,7 +161,38 @@ Commit your signature to gitian.sigs: git push # Assuming you can push to the gitian.sigs tree popd -Wait for Windows/OS X detached signatures: +Codesigner only: Create Windows/OS X detached signatures: +- Only one person handles codesigning. Everyone else should skip to the next step. +- Only once the Windows/OS X builds each have 3 matching signatures may they be signed with their respective release keys. + +Codesigner only: Sign the osx binary: + + transfer bitcoin-osx-unsigned.tar.gz to osx for signing + tar xf bitcoin-osx-unsigned.tar.gz + ./detached-sig-create.sh -s "Key ID" + Enter the keychain password and authorize the signature + Move signature-osx.tar.gz back to the gitian host + +Codesigner only: Sign the windows binaries: + + tar xf bitcoin-win-unsigned.tar.gz + ./detached-sig-create.sh -key /path/to/codesign.key + Enter the passphrase for the key when prompted + signature-win.tar.gz will be created + +Codesigner only: Commit the detached codesign payloads: + + cd ~/bitcoin-detached-sigs + checkout the appropriate branch for this release series + rm -rf * + tar xf signature-osx.tar.gz + tar xf signature-win.tar.gz + git add -a + git commit -m "point to ${VERSION}" + git tag -s v${VERSION} HEAD + git push the current branch and new tag + +Non-codesigners: wait for Windows/OS X detached signatures: - Once the Windows/OS X builds each have 3 matching signatures, they will be signed with their respective release keys. - Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin-core/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. diff --git a/qa/README.md b/qa/README.md deleted file mode 100644 index f4dce7af5c..0000000000 --- a/qa/README.md +++ /dev/null @@ -1,87 +0,0 @@ -The [pull-tester](/qa/pull-tester/) folder contains a script to call -multiple tests from the [rpc-tests](/qa/rpc-tests/) folder. - -Every pull request to the bitcoin repository is built and run through -the regression test suite. You can also run all or only individual -tests locally. - -Test dependencies -================= -Before running the tests, the following must be installed. - -Unix ----- -The python3-zmq library is required. On Ubuntu or Debian it can be installed via: -``` -sudo apt-get install python3-zmq -``` - -OS X ------- -``` -pip3 install pyzmq -``` - -Running tests -============= - -You can run any single test by calling - - qa/pull-tester/rpc-tests.py <testname> - -Or you can run any combination of tests by calling - - qa/pull-tester/rpc-tests.py <testname1> <testname2> <testname3> ... - -Run the regression test suite with - - qa/pull-tester/rpc-tests.py - -Run all possible tests with - - qa/pull-tester/rpc-tests.py --extended - -By default, tests will be run in parallel. To specify how many jobs to run, -append `--jobs=n` (default n=4). - -If you want to create a basic coverage report for the RPC test suite, append `--coverage`. - -Possible options, which apply to each individual test run: - -``` - -h, --help show this help message and exit - --nocleanup Leave bitcoinds and test.* datadir on exit or error - --noshutdown Don't stop bitcoinds after the test execution - --srcdir=SRCDIR Source directory containing bitcoind/bitcoin-cli - (default: ../../src) - --tmpdir=TMPDIR Root directory for datadirs - --tracerpc Print out all RPC calls as they are made - --coveragedir=COVERAGEDIR - Write tested RPC commands into this directory -``` - -If you set the environment variable `PYTHON_DEBUG=1` you will get some debug -output (example: `PYTHON_DEBUG=1 qa/pull-tester/rpc-tests.py wallet`). - -A 200-block -regtest blockchain and wallets for four nodes -is created the first time a regression test is run and -is stored in the cache/ directory. Each node has 25 mature -blocks (25*50=1250 BTC) in its wallet. - -After the first run, the cache/ blockchain and wallets are -copied into a temporary directory and used as the initial -test state. - -If you get into a bad state, you should be able -to recover with: - -```bash -rm -rf cache -killall bitcoind -``` - -Writing tests -============= -You are encouraged to write tests for new or existing features. -Further information about the test framework and individual RPC -tests is found in [qa/rpc-tests](/qa/rpc-tests). diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py deleted file mode 100755 index 67b5826840..0000000000 --- a/qa/rpc-tests/blockchain.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test RPCs related to blockchainstate. - -Test the following RPCs: - - gettxoutsetinfo - - verifychain - -Tests correspond to code in rpc/blockchain.cpp. -""" - -from decimal import Decimal - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.authproxy import JSONRPCException -from test_framework.util import ( - assert_equal, - assert_raises, - assert_is_hex_string, - assert_is_hash_string, - start_nodes, - connect_nodes_bi, -) - - -class BlockchainTest(BitcoinTestFramework): - - def __init__(self): - super().__init__() - self.setup_clean_chain = False - self.num_nodes = 2 - - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes, 0, 1) - self.is_network_split = False - self.sync_all() - - def run_test(self): - self._test_gettxoutsetinfo() - self._test_getblockheader() - self.nodes[0].verifychain(4, 0) - - def _test_gettxoutsetinfo(self): - node = self.nodes[0] - res = node.gettxoutsetinfo() - - assert_equal(res['total_amount'], Decimal('8725.00000000')) - assert_equal(res['transactions'], 200) - assert_equal(res['height'], 200) - assert_equal(res['txouts'], 200) - assert_equal(res['bytes_serialized'], 13924), - assert_equal(len(res['bestblock']), 64) - assert_equal(len(res['hash_serialized']), 64) - - def _test_getblockheader(self): - node = self.nodes[0] - - assert_raises( - JSONRPCException, lambda: node.getblockheader('nonsense')) - - besthash = node.getbestblockhash() - secondbesthash = node.getblockhash(199) - header = node.getblockheader(besthash) - - assert_equal(header['hash'], besthash) - assert_equal(header['height'], 200) - assert_equal(header['confirmations'], 1) - assert_equal(header['previousblockhash'], secondbesthash) - assert_is_hex_string(header['chainwork']) - assert_is_hash_string(header['hash']) - assert_is_hash_string(header['previousblockhash']) - assert_is_hash_string(header['merkleroot']) - assert_is_hash_string(header['bits'], length=None) - assert isinstance(header['time'], int) - assert isinstance(header['mediantime'], int) - assert isinstance(header['nonce'], int) - assert isinstance(header['version'], int) - assert isinstance(int(header['versionHex'], 16), int) - assert isinstance(header['difficulty'], Decimal) - -if __name__ == '__main__': - BlockchainTest().main() diff --git a/qa/rpc-tests/maxblocksinflight.py b/qa/rpc-tests/maxblocksinflight.py deleted file mode 100755 index 8ea405b4d5..0000000000 --- a/qa/rpc-tests/maxblocksinflight.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test nodes responses to having many blocks in flight. - -In this test we connect to one node over p2p, send it numerous inv's, and -compare the resulting number of getdata requests to a max allowed value. We -test for exceeding 128 blocks in flight, which was the limit an 0.9 client will -reach. [0.10 clients shouldn't request more than 16 from a single peer.] -""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * -import logging - -MAX_REQUESTS = 128 - -class TestManager(NodeConnCB): - # set up NodeConnCB callbacks, overriding base class - def on_getdata(self, conn, message): - self.log.debug("got getdata %s" % repr(message)) - # Log the requests - for inv in message.inv: - if inv.hash not in self.blockReqCounts: - self.blockReqCounts[inv.hash] = 0 - self.blockReqCounts[inv.hash] += 1 - - def on_close(self, conn): - if not self.disconnectOkay: - raise EarlyDisconnectError(0) - - def __init__(self): - NodeConnCB.__init__(self) - self.log = logging.getLogger("BlockRelayTest") - - def add_new_connection(self, connection): - self.connection = connection - self.blockReqCounts = {} - self.disconnectOkay = False - - def run(self): - self.connection.rpc.generate(1) # Leave IBD - - numBlocksToGenerate = [8, 16, 128, 1024] - for count in range(len(numBlocksToGenerate)): - current_invs = [] - for i in range(numBlocksToGenerate[count]): - current_invs.append(CInv(2, random.randrange(0, 1 << 256))) - if len(current_invs) >= 50000: - self.connection.send_message(msg_inv(current_invs)) - current_invs = [] - if len(current_invs) > 0: - self.connection.send_message(msg_inv(current_invs)) - - # Wait and see how many blocks were requested - time.sleep(2) - - total_requests = 0 - with mininode_lock: - for key in self.blockReqCounts: - total_requests += self.blockReqCounts[key] - if self.blockReqCounts[key] > 1: - raise AssertionError("Error, test failed: block %064x requested more than once" % key) - if total_requests > MAX_REQUESTS: - raise AssertionError("Error, too many blocks (%d) requested" % total_requests) - print("Round %d: success (total requests: %d)" % (count, total_requests)) - - self.disconnectOkay = True - self.connection.disconnect_node() - - -class MaxBlocksInFlightTest(BitcoinTestFramework): - def add_options(self, parser): - parser.add_option("--testbinary", dest="testbinary", - default=os.getenv("BITCOIND", "bitcoind"), - help="Binary to test max block requests behavior") - - def __init__(self): - super().__init__() - self.setup_clean_chain = True - self.num_nodes = 1 - - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1']], - binary=[self.options.testbinary]) - - def run_test(self): - test = TestManager() - test.add_new_connection(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test)) - NetworkThread().start() # Start up network handling in another thread - test.run() - -if __name__ == '__main__': - MaxBlocksInFlightTest().main() diff --git a/qa/rpc-tests/nodehandling.py b/qa/rpc-tests/nodehandling.py deleted file mode 100755 index 89059d2760..0000000000 --- a/qa/rpc-tests/nodehandling.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test node handling.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -import urllib.parse - -class NodeHandlingTest (BitcoinTestFramework): - - def __init__(self): - super().__init__() - self.num_nodes = 4 - self.setup_clean_chain = False - - def run_test(self): - ########################### - # setban/listbanned tests # - ########################### - assert_equal(len(self.nodes[2].getpeerinfo()), 4) #we should have 4 nodes at this point - self.nodes[2].setban("127.0.0.1", "add") - time.sleep(3) #wait till the nodes are disconected - assert_equal(len(self.nodes[2].getpeerinfo()), 0) #all nodes must be disconnected at this point - assert_equal(len(self.nodes[2].listbanned()), 1) - self.nodes[2].clearbanned() - assert_equal(len(self.nodes[2].listbanned()), 0) - self.nodes[2].setban("127.0.0.0/24", "add") - assert_equal(len(self.nodes[2].listbanned()), 1) - try: - self.nodes[2].setban("127.0.0.1", "add") #throws exception because 127.0.0.1 is within range 127.0.0.0/24 - except: - pass - assert_equal(len(self.nodes[2].listbanned()), 1) #still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 - try: - self.nodes[2].setban("127.0.0.1", "remove") - except: - pass - assert_equal(len(self.nodes[2].listbanned()), 1) - self.nodes[2].setban("127.0.0.0/24", "remove") - assert_equal(len(self.nodes[2].listbanned()), 0) - self.nodes[2].clearbanned() - assert_equal(len(self.nodes[2].listbanned()), 0) - - ##test persisted banlist - self.nodes[2].setban("127.0.0.0/32", "add") - self.nodes[2].setban("127.0.0.0/24", "add") - self.nodes[2].setban("192.168.0.1", "add", 1) #ban for 1 seconds - self.nodes[2].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) #ban for 1000 seconds - listBeforeShutdown = self.nodes[2].listbanned() - assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) #must be here - time.sleep(2) #make 100% sure we expired 192.168.0.1 node time - - #stop node - stop_node(self.nodes[2], 2) - - self.nodes[2] = start_node(2, self.options.tmpdir) - listAfterShutdown = self.nodes[2].listbanned() - assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) - assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) - assert_equal("/19" in listAfterShutdown[2]['address'], True) - - ########################### - # RPC disconnectnode test # - ########################### - url = urllib.parse.urlparse(self.nodes[1].url) - self.nodes[0].disconnectnode(url.hostname+":"+str(p2p_port(1))) - time.sleep(2) #disconnecting a node needs a little bit of time - for node in self.nodes[0].getpeerinfo(): - assert(node['addr'] != url.hostname+":"+str(p2p_port(1))) - - connect_nodes_bi(self.nodes,0,1) #reconnect the node - found = False - for node in self.nodes[0].getpeerinfo(): - if node['addr'] == url.hostname+":"+str(p2p_port(1)): - found = True - assert(found) - -if __name__ == '__main__': - NodeHandlingTest ().main () diff --git a/qa/rpc-tests/p2p-mempool.py b/qa/rpc-tests/p2p-mempool.py deleted file mode 100755 index c6b1490211..0000000000 --- a/qa/rpc-tests/p2p-mempool.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test p2p mempool message. - -Test that nodes are disconnected if they send mempool messages when bloom -filters are not enabled. -""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -class TestNode(NodeConnCB): - def __init__(self): - NodeConnCB.__init__(self) - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() - self.block_receive_map = {} - - def add_connection(self, conn): - self.connection = conn - self.peer_disconnected = False - - def on_inv(self, conn, message): - pass - - # Track the last getdata message we receive (used in the test) - def on_getdata(self, conn, message): - self.last_getdata = message - - def on_block(self, conn, message): - message.block.calc_sha256() - try: - self.block_receive_map[message.block.sha256] += 1 - except KeyError as e: - self.block_receive_map[message.block.sha256] = 1 - - # Spin until verack message is received from the node. - # We use this to signal that our test can begin. This - # is called from the testing thread, so it needs to acquire - # the global lock. - def wait_for_verack(self): - def veracked(): - return self.verack_received - return wait_until(veracked, timeout=10) - - def wait_for_disconnect(self): - def disconnected(): - return self.peer_disconnected - return wait_until(disconnected, timeout=10) - - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_pong(self, conn, message): - self.last_pong = message - - def on_close(self, conn): - self.peer_disconnected = True - - # Sync up with the node after delivery of a block - def sync_with_ping(self, timeout=30): - def received_pong(): - return (self.last_pong.nonce == self.ping_counter) - self.connection.send_message(msg_ping(nonce=self.ping_counter)) - success = wait_until(received_pong, timeout=timeout) - self.ping_counter += 1 - return success - - def send_mempool(self): - self.lastInv = [] - self.send_message(msg_mempool()) - -class P2PMempoolTests(BitcoinTestFramework): - - def __init__(self): - super().__init__() - self.setup_clean_chain = True - self.num_nodes = 2 - - def setup_network(self): - # Start a node with maxuploadtarget of 200 MB (/24h) - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-peerbloomfilters=0"])) - - def run_test(self): - #connect a mininode - aTestNode = TestNode() - node = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], aTestNode) - aTestNode.add_connection(node) - NetworkThread().start() - aTestNode.wait_for_verack() - - #request mempool - aTestNode.send_mempool() - aTestNode.wait_for_disconnect() - - #mininode must be disconnected at this point - assert_equal(len(self.nodes[0].getpeerinfo()), 0) - -if __name__ == '__main__': - P2PMempoolTests().main() diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py deleted file mode 100755 index c5e9c6a658..0000000000 --- a/qa/rpc-tests/test_framework/test_framework.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Base class for RPC testing.""" - -import logging -import optparse -import os -import sys -import shutil -import tempfile -import traceback - -from .util import ( - initialize_chain, - start_nodes, - connect_nodes_bi, - sync_blocks, - sync_mempools, - stop_nodes, - stop_node, - enable_coverage, - check_json_precision, - initialize_chain_clean, - PortSeed, -) -from .authproxy import JSONRPCException - - -class BitcoinTestFramework(object): - - def __init__(self): - self.num_nodes = 4 - self.setup_clean_chain = False - self.nodes = None - - def run_test(self): - raise NotImplementedError - - def add_options(self, parser): - pass - - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - if self.setup_clean_chain: - initialize_chain_clean(self.options.tmpdir, self.num_nodes) - else: - initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir) - - def stop_node(self, num_node): - stop_node(self.nodes[num_node], num_node) - - def setup_nodes(self): - return start_nodes(self.num_nodes, self.options.tmpdir) - - def setup_network(self, split = False): - self.nodes = self.setup_nodes() - - # Connect the nodes as a "chain". This allows us - # to split the network between nodes 1 and 2 to get - # two halves that can work on competing chains. - - # If we joined network halves, connect the nodes from the joint - # on outward. This ensures that chains are properly reorganised. - if not split: - connect_nodes_bi(self.nodes, 1, 2) - sync_blocks(self.nodes[1:3]) - sync_mempools(self.nodes[1:3]) - - connect_nodes_bi(self.nodes, 0, 1) - connect_nodes_bi(self.nodes, 2, 3) - self.is_network_split = split - self.sync_all() - - def split_network(self): - """ - Split the network of four nodes into nodes 0/1 and 2/3. - """ - assert not self.is_network_split - stop_nodes(self.nodes) - self.setup_network(True) - - def sync_all(self): - if self.is_network_split: - sync_blocks(self.nodes[:2]) - sync_blocks(self.nodes[2:]) - sync_mempools(self.nodes[:2]) - sync_mempools(self.nodes[2:]) - else: - sync_blocks(self.nodes) - sync_mempools(self.nodes) - - def join_network(self): - """ - Join the (previously split) network halves together. - """ - assert self.is_network_split - stop_nodes(self.nodes) - self.setup_network(False) - - def main(self): - - parser = optparse.OptionParser(usage="%prog [options]") - parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true", - help="Leave bitcoinds and test.* datadir on exit or error") - parser.add_option("--noshutdown", dest="noshutdown", default=False, action="store_true", - help="Don't stop bitcoinds after the test execution") - parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"), - help="Source directory containing bitcoind/bitcoin-cli (default: %default)") - parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"), - help="Directory for caching pregenerated datadirs") - parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), - help="Root directory for datadirs") - parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true", - help="Print out all RPC calls as they are made") - parser.add_option("--portseed", dest="port_seed", default=os.getpid(), type='int', - help="The seed to use for assigning port numbers (default: current process id)") - parser.add_option("--coveragedir", dest="coveragedir", - help="Write tested RPC commands into this directory") - self.add_options(parser) - (self.options, self.args) = parser.parse_args() - - # backup dir variable for removal at cleanup - self.options.root, self.options.tmpdir = self.options.tmpdir, self.options.tmpdir + '/' + str(self.options.port_seed) - - if self.options.trace_rpc: - logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) - - if self.options.coveragedir: - enable_coverage(self.options.coveragedir) - - PortSeed.n = self.options.port_seed - - os.environ['PATH'] = self.options.srcdir+":"+self.options.srcdir+"/qt:"+os.environ['PATH'] - - check_json_precision() - - success = False - try: - os.makedirs(self.options.tmpdir, exist_ok=False) - self.setup_chain() - self.setup_network() - self.run_test() - success = True - except JSONRPCException as e: - print("JSONRPC error: "+e.error['message']) - traceback.print_tb(sys.exc_info()[2]) - except AssertionError as e: - print("Assertion failed: " + str(e)) - traceback.print_tb(sys.exc_info()[2]) - except KeyError as e: - print("key not found: "+ str(e)) - traceback.print_tb(sys.exc_info()[2]) - except Exception as e: - print("Unexpected exception caught during testing: " + repr(e)) - traceback.print_tb(sys.exc_info()[2]) - except KeyboardInterrupt as e: - print("Exiting after " + repr(e)) - - if not self.options.noshutdown: - print("Stopping nodes") - stop_nodes(self.nodes) - else: - print("Note: bitcoinds were not stopped and may still be running") - - if not self.options.nocleanup and not self.options.noshutdown and success: - print("Cleaning up") - shutil.rmtree(self.options.tmpdir) - if not os.listdir(self.options.root): - os.rmdir(self.options.root) - else: - print("Not cleaning up dir %s" % self.options.tmpdir) - if os.getenv("PYTHON_DEBUG", ""): - # Dump the end of the debug logs, to aid in debugging rare - # travis failures. - import glob - filenames = glob.glob(self.options.tmpdir + "/node*/regtest/debug.log") - MAX_LINES_TO_PRINT = 1000 - for f in filenames: - print("From" , f, ":") - from collections import deque - print("".join(deque(open(f), MAX_LINES_TO_PRINT))) - if success: - print("Tests successful") - sys.exit(0) - else: - print("Failed") - sys.exit(1) - - -# Test framework for doing p2p comparison testing, which sets up some bitcoind -# binaries: -# 1 binary: test binary -# 2 binaries: 1 test binary, 1 ref binary -# n>2 binaries: 1 test binary, n-1 ref binaries - -class ComparisonTestFramework(BitcoinTestFramework): - - def __init__(self): - super().__init__() - self.num_nodes = 2 - self.setup_clean_chain = True - - def add_options(self, parser): - parser.add_option("--testbinary", dest="testbinary", - default=os.getenv("BITCOIND", "bitcoind"), - help="bitcoind binary to test") - parser.add_option("--refbinary", dest="refbinary", - default=os.getenv("BITCOIND", "bitcoind"), - help="bitcoind binary to use for reference nodes (if any)") - - def setup_network(self): - self.nodes = start_nodes( - self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1']] * self.num_nodes, - binary=[self.options.testbinary] + - [self.options.refbinary]*(self.num_nodes-1)) diff --git a/share/genbuild.sh b/share/genbuild.sh index eecac4bd00..32ef2a5755 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -17,9 +17,13 @@ else exit 1 fi +git_check_in_repo() { + ! { git status --porcelain -uall --ignored "$@" 2>/dev/null || echo '??'; } | grep -q '?' +} + DESC="" SUFFIX="" -if [ -e "$(which git 2>/dev/null)" -a "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then +if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" -a -e "$(which git 2>/dev/null)" -a "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ] && git_check_in_repo share/genbuild.sh; then # clean 'dirty' status of touched files that haven't been modified git diff >/dev/null 2>/dev/null diff --git a/src/Makefile.am b/src/Makefile.am index e8d22313dc..ae2eb29c94 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -95,9 +95,11 @@ BITCOIN_CORE_H = \ compat/sanity.h \ compressor.h \ consensus/consensus.h \ + consensus/tx_verify.h \ core_io.h \ core_memusage.h \ cuckoocache.h \ + fs.h \ httprpc.h \ httpserver.h \ indirectmap.h \ @@ -115,6 +117,7 @@ BITCOIN_CORE_H = \ netbase.h \ netmessagemaker.h \ noui.h \ + policy/feerate.h \ policy/fees.h \ policy/policy.h \ policy/rbf.h \ @@ -122,6 +125,7 @@ BITCOIN_CORE_H = \ protocol.h \ random.h \ reverselock.h \ + rpc/blockchain.h \ rpc/client.h \ rpc/protocol.h \ rpc/server.h \ @@ -155,6 +159,7 @@ BITCOIN_CORE_H = \ wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ + wallet/feebumper.h \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ @@ -167,20 +172,21 @@ BITCOIN_CORE_H = \ obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj - @$(top_srcdir)/share/genbuild.sh $(abs_top_builddir)/src/obj/build.h \ - $(abs_top_srcdir) + @$(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ + "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # server: shared between bitcoind and bitcoin-qt libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ - addrman.cpp \ addrdb.cpp \ + addrman.cpp \ bloom.cpp \ blockencodings.cpp \ chain.cpp \ checkpoints.cpp \ + consensus/tx_verify.cpp \ httprpc.cpp \ httpserver.cpp \ init.cpp \ @@ -229,6 +235,7 @@ libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ wallet/crypter.cpp \ wallet/db.cpp \ + wallet/feebumper.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ @@ -242,6 +249,8 @@ crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ + crypto/chacha20.h \ + crypto/chacha20.cpp \ crypto/common.h \ crypto/hmac_sha256.cpp \ crypto/hmac_sha256.h \ @@ -295,7 +304,6 @@ libbitcoin_consensus_a_SOURCES = \ libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ - amount.cpp \ base58.cpp \ chainparams.cpp \ coins.cpp \ @@ -306,6 +314,7 @@ libbitcoin_common_a_SOURCES = \ keystore.cpp \ netaddress.cpp \ netbase.cpp \ + policy/feerate.cpp \ protocol.cpp \ scheduler.cpp \ script/sign.cpp \ @@ -325,6 +334,7 @@ libbitcoin_util_a_SOURCES = \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ + fs.cpp \ random.cpp \ rpc/protocol.cpp \ support/cleanse.cpp \ @@ -457,6 +467,14 @@ DISTCLEANFILES = obj/build.h EXTRA_DIST = $(CTAES_DIST) + +config/bitcoin-config.h: config/stamp-h1 + @$(MAKE) -C $(top_builddir) $(subdir)/$(@) +config/stamp-h1: $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in $(top_builddir)/config.status + $(AM_V_at)$(MAKE) -C $(top_builddir) $(subdir)/$(@) +$(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) + $(AM_V_at)$(MAKE) -C $(top_srcdir) $(subdir)/config/bitcoin-config.h.in + clean-local: -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 8c699c2f8c..3bcecab596 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -25,7 +25,8 @@ bench_bench_bitcoin_SOURCES = \ bench/base58.cpp \ bench/lockedpool.cpp \ bench/perf.cpp \ - bench/perf.h + bench/perf.h \ + bench/prevector_destructor.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index edc3c4b292..48411f29ec 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -122,6 +122,7 @@ QT_MOC_CPP = \ qt/moc_bitcoinamountfield.cpp \ qt/moc_bitcoingui.cpp \ qt/moc_bitcoinunits.cpp \ + qt/moc_callback.cpp \ qt/moc_clientmodel.cpp \ qt/moc_coincontroldialog.cpp \ qt/moc_coincontroltreewidget.cpp \ @@ -167,6 +168,7 @@ BITCOIN_MM = \ QT_MOC = \ qt/bitcoin.moc \ qt/bitcoinamountfield.moc \ + qt/callback.moc \ qt/intro.moc \ qt/overviewpage.moc \ qt/rpcconsole.moc @@ -189,6 +191,7 @@ BITCOIN_QT_H = \ qt/bitcoinamountfield.h \ qt/bitcoingui.h \ qt/bitcoinunits.h \ + qt/callback.h \ qt/clientmodel.h \ qt/coincontroldialog.h \ qt/coincontroltreewidget.h \ diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 039f8ac547..391b9ebdf6 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -11,7 +11,9 @@ TEST_QT_MOC_CPP = \ qt/test/moc_uritests.cpp if ENABLE_WALLET -TEST_QT_MOC_CPP += qt/test/moc_paymentservertests.cpp +TEST_QT_MOC_CPP += \ + qt/test/moc_paymentservertests.cpp \ + qt/test/moc_wallettests.cpp endif TEST_QT_H = \ @@ -19,7 +21,16 @@ TEST_QT_H = \ qt/test/rpcnestedtests.h \ qt/test/uritests.h \ qt/test/paymentrequestdata.h \ - qt/test/paymentservertests.h + qt/test/paymentservertests.h \ + qt/test/wallettests.h + +TEST_BITCOIN_CPP = \ + test/test_bitcoin.cpp \ + test/testutil.cpp + +TEST_BITCOIN_H = \ + test/test_bitcoin.h \ + test/testutil.h qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(PROTOBUF_CFLAGS) @@ -29,10 +40,14 @@ qt_test_test_bitcoin_qt_SOURCES = \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ qt/test/uritests.cpp \ - $(TEST_QT_H) + $(TEST_QT_H) \ + $(TEST_BITCOIN_CPP) \ + $(TEST_BITCOIN_H) if ENABLE_WALLET qt_test_test_bitcoin_qt_SOURCES += \ - qt/test/paymentservertests.cpp + qt/test/paymentservertests.cpp \ + qt/test/wallettests.cpp \ + wallet/test/wallet_test_fixture.cpp endif nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 4d44b35bb6..ee1c11ff1f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -8,60 +8,6 @@ noinst_PROGRAMS += test/test_bitcoin_fuzzy TEST_SRCDIR = test TEST_BINARY=test/test_bitcoin$(EXEEXT) - -EXTRA_DIST += \ - test/bctest.py \ - test/bitcoin-util-test.py \ - test/data/bitcoin-util-test.json \ - test/data/blanktxv1.hex \ - test/data/blanktxv1.json \ - test/data/blanktxv2.hex \ - test/data/blanktxv2.json \ - test/data/tt-delin1-out.hex \ - test/data/tt-delin1-out.json \ - test/data/tt-delout1-out.hex \ - test/data/tt-delout1-out.json \ - test/data/tt-locktime317000-out.hex \ - test/data/tt-locktime317000-out.json \ - test/data/tx394b54bb.hex \ - test/data/txcreate1.hex \ - test/data/txcreate1.json \ - test/data/txcreate2.hex \ - test/data/txcreate2.json \ - test/data/txcreatedata1.hex \ - test/data/txcreatedata1.json \ - test/data/txcreatedata2.hex \ - test/data/txcreatedata2.json \ - test/data/txcreatedata_seq0.hex \ - test/data/txcreatedata_seq0.json \ - test/data/txcreatedata_seq1.hex \ - test/data/txcreatedata_seq1.json \ - test/data/txcreatemultisig1.hex \ - test/data/txcreatemultisig1.json \ - test/data/txcreatemultisig2.hex \ - test/data/txcreatemultisig2.json \ - test/data/txcreatemultisig3.hex \ - test/data/txcreatemultisig3.json \ - test/data/txcreatemultisig4.hex \ - test/data/txcreatemultisig4.json \ - test/data/txcreateoutpubkey1.hex \ - test/data/txcreateoutpubkey1.json \ - test/data/txcreateoutpubkey2.hex \ - test/data/txcreateoutpubkey2.json \ - test/data/txcreateoutpubkey3.hex \ - test/data/txcreateoutpubkey3.json \ - test/data/txcreatescript1.hex \ - test/data/txcreatescript1.json \ - test/data/txcreatescript2.hex \ - test/data/txcreatescript2.json \ - test/data/txcreatescript3.hex \ - test/data/txcreatescript3.json \ - test/data/txcreatescript4.hex \ - test/data/txcreatescript4.json \ - test/data/txcreatesignv1.hex \ - test/data/txcreatesignv1.json \ - test/data/txcreatesignv2.hex - JSON_TEST_FILES = \ test/data/script_tests.json \ test/data/base58_keys_valid.json \ @@ -89,6 +35,7 @@ BITCOIN_TESTS =\ test/blockencodings_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ + test/checkqueue_tests.cpp \ test/coins_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ @@ -111,6 +58,7 @@ BITCOIN_TESTS =\ test/pow_tests.cpp \ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ + test/random_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ @@ -125,10 +73,12 @@ BITCOIN_TESTS =\ test/streams_tests.cpp \ test/test_bitcoin.cpp \ test/test_bitcoin.h \ + test/test_bitcoin_main.cpp \ test/test_random.h \ test/testutil.cpp \ test/testutil.h \ test/timedata_tests.cpp \ + test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ @@ -148,7 +98,7 @@ endif test_test_bitcoin_SOURCES = $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ - $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) + $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) @@ -188,9 +138,6 @@ CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno $(GENERATED_TEST_FILES) CLEANFILES += $(CLEAN_BITCOIN_TEST) -# This file is problematic for out-of-tree builds if it exists. -DISTCLEANFILES += test/buildenv.pyc - bitcoin_test: $(TEST_BINARY) bitcoin_test_check: $(TEST_BINARY) FORCE @@ -200,8 +147,8 @@ bitcoin_test_clean : FORCE rm -f $(CLEAN_BITCOIN_TEST) $(test_test_bitcoin_OBJECTS) $(TEST_BINARY) check-local: - @echo "Running test/bitcoin-util-test.py..." - $(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(PYTHON) $(srcdir)/test/bitcoin-util-test.py + @echo "Running test/util/bitcoin-util-test.py..." + $(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check if EMBEDDED_UNIVALUE $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 224d3921cc..a3743cd0d4 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -8,13 +8,13 @@ #include "addrman.h" #include "chainparams.h" #include "clientversion.h" +#include "fs.h" #include "hash.h" #include "random.h" #include "streams.h" #include "tinyformat.h" #include "util.h" -#include <boost/filesystem.hpp> CBanDB::CBanDB() { @@ -36,8 +36,8 @@ bool CBanDB::Write(const banmap_t& banSet) ssBanlist << hash; // open temp output file, and associate with CAutoFile - boost::filesystem::path pathTmp = GetDataDir() / tmpfn; - FILE *file = fopen(pathTmp.string().c_str(), "wb"); + fs::path pathTmp = GetDataDir() / tmpfn; + FILE *file = fsbridge::fopen(pathTmp, "wb"); CAutoFile fileout(file, SER_DISK, CLIENT_VERSION); if (fileout.IsNull()) return error("%s: Failed to open file %s", __func__, pathTmp.string()); @@ -62,13 +62,13 @@ bool CBanDB::Write(const banmap_t& banSet) bool CBanDB::Read(banmap_t& banSet) { // open input file, and associate with CAutoFile - FILE *file = fopen(pathBanlist.string().c_str(), "rb"); + FILE *file = fsbridge::fopen(pathBanlist, "rb"); CAutoFile filein(file, SER_DISK, CLIENT_VERSION); if (filein.IsNull()) return error("%s: Failed to open file %s", __func__, pathBanlist.string()); // use file size to size memory buffer - uint64_t fileSize = boost::filesystem::file_size(pathBanlist); + uint64_t fileSize = fs::file_size(pathBanlist); uint64_t dataSize = 0; // Don't try to resize to a negative number if file is small if (fileSize >= sizeof(uint256)) @@ -133,8 +133,8 @@ bool CAddrDB::Write(const CAddrMan& addr) ssPeers << hash; // open temp output file, and associate with CAutoFile - boost::filesystem::path pathTmp = GetDataDir() / tmpfn; - FILE *file = fopen(pathTmp.string().c_str(), "wb"); + fs::path pathTmp = GetDataDir() / tmpfn; + FILE *file = fsbridge::fopen(pathTmp, "wb"); CAutoFile fileout(file, SER_DISK, CLIENT_VERSION); if (fileout.IsNull()) return error("%s: Failed to open file %s", __func__, pathTmp.string()); @@ -159,13 +159,13 @@ bool CAddrDB::Write(const CAddrMan& addr) bool CAddrDB::Read(CAddrMan& addr) { // open input file, and associate with CAutoFile - FILE *file = fopen(pathAddr.string().c_str(), "rb"); + FILE *file = fsbridge::fopen(pathAddr, "rb"); CAutoFile filein(file, SER_DISK, CLIENT_VERSION); if (filein.IsNull()) return error("%s: Failed to open file %s", __func__, pathAddr.string()); // use file size to size memory buffer - uint64_t fileSize = boost::filesystem::file_size(pathAddr); + uint64_t fileSize = fs::file_size(pathAddr); uint64_t dataSize = 0; // Don't try to resize to a negative number if file is small if (fileSize >= sizeof(uint256)) diff --git a/src/addrdb.h b/src/addrdb.h index ab985b10cb..c3d509bd3a 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H +#include "fs.h" #include "serialize.h" #include <string> #include <map> -#include <boost/filesystem/path.hpp> class CSubNet; class CAddrMan; @@ -80,7 +80,7 @@ typedef std::map<CSubNet, CBanEntry> banmap_t; class CAddrDB { private: - boost::filesystem::path pathAddr; + fs::path pathAddr; public: CAddrDB(); bool Write(const CAddrMan& addr); @@ -92,7 +92,7 @@ public: class CBanDB { private: - boost::filesystem::path pathBanlist; + fs::path pathBanlist; public: CBanDB(); bool Write(const banmap_t& banSet); diff --git a/src/addrman.cpp b/src/addrman.cpp index b6ab4c6305..4a408b9beb 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -233,7 +233,7 @@ void CAddrMan::Good_(const CService& addr, int64_t nTime) if (nUBucket == -1) return; - LogPrint("addrman", "Moving %s to tried\n", addr.ToString()); + LogPrint(BCLog::ADDRMAN, "Moving %s to tried\n", addr.ToString()); // move nId to the tried tables MakeTried(info, nId); @@ -351,8 +351,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.rand32()) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE; + nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; + nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nId) == 1); @@ -368,8 +368,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.rand32()) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE; + nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; + nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvNew[nUBucket][nUBucketPos]; assert(mapInfo.count(nId) == 1); diff --git a/src/addrman.h b/src/addrman.h index 6e5f946bf2..70d907488f 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -136,13 +136,13 @@ public: */ //! total number of buckets for tried addresses -#define ADDRMAN_TRIED_BUCKET_COUNT 256 +#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8 //! total number of buckets for new addresses -#define ADDRMAN_NEW_BUCKET_COUNT 1024 +#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10 //! maximum allowed number of entries in buckets for new and tried addresses -#define ADDRMAN_BUCKET_SIZE 64 +#define ADDRMAN_BUCKET_SIZE_LOG2 6 //! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread #define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8 @@ -171,6 +171,11 @@ public: //! the maximum number of nodes to return in a getaddr call #define ADDRMAN_GETADDR_MAX 2500 +//! Convenience +#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) +#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) +#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2) + /** * Stochastical (IP) address manager */ @@ -442,7 +447,7 @@ public: } } if (nLost + nLostUnk > 0) { - LogPrint("addrman", "addrman lost %i new and %i tried addresses due to collisions\n", nLostUnk, nLost); + LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions\n", nLostUnk, nLost); } Check(); @@ -507,8 +512,9 @@ public: Check(); fRet |= Add_(addr, source, nTimePenalty); Check(); - if (fRet) - LogPrint("addrman", "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); + if (fRet) { + LogPrint(BCLog::ADDRMAN, "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); + } return fRet; } @@ -521,8 +527,9 @@ public: for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; Check(); - if (nAdd) - LogPrint("addrman", "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); + if (nAdd) { + LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); + } return nAdd > 0; } diff --git a/src/amount.h b/src/amount.h index 93060f7193..2bd367cba2 100644 --- a/src/amount.h +++ b/src/amount.h @@ -6,10 +6,7 @@ #ifndef BITCOIN_AMOUNT_H #define BITCOIN_AMOUNT_H -#include "serialize.h" - -#include <stdlib.h> -#include <string> +#include <stdint.h> /** Amount in satoshis (Can be negative) */ typedef int64_t CAmount; @@ -17,8 +14,6 @@ typedef int64_t CAmount; static const CAmount COIN = 100000000; static const CAmount CENT = 1000000; -extern const std::string CURRENCY_UNIT; - /** No amount larger than this (in satoshi) is valid. * * Note that this constant is *not* the total money supply, which in Bitcoin @@ -31,42 +26,4 @@ extern const std::string CURRENCY_UNIT; static const CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -/** - * Fee rate in satoshis per kilobyte: CAmount / kB - */ -class CFeeRate -{ -private: - CAmount nSatoshisPerK; // unit is satoshis-per-1,000-bytes -public: - /** Fee rate of 0 satoshis per kB */ - CFeeRate() : nSatoshisPerK(0) { } - explicit CFeeRate(const CAmount& _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } - /** Constructor for a fee rate in satoshis per kB. The size in bytes must not exceed (2^63 - 1)*/ - CFeeRate(const CAmount& nFeePaid, size_t nBytes); - CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } - /** - * Return the fee in satoshis for the given size in bytes. - */ - CAmount GetFee(size_t nBytes) const; - /** - * Return the fee in satoshis for a size of 1000 bytes - */ - CAmount GetFeePerK() const { return GetFee(1000); } - friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } - friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } - friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } - friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } - friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } - CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } - std::string ToString() const; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nSatoshisPerK); - } -}; - #endif // BITCOIN_AMOUNT_H diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index f9f2d19e68..dd34a313b7 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -173,9 +173,9 @@ unsigned int base_uint<BITS>::bits() const { for (int pos = WIDTH - 1; pos >= 0; pos--) { if (pn[pos]) { - for (int bits = 31; bits > 0; bits--) { - if (pn[pos] & 1 << bits) - return 32 * pos + bits + 1; + for (int nbits = 31; nbits > 0; nbits--) { + if (pn[pos] & 1 << nbits) + return 32 * pos + nbits + 1; } return 32 * pos + 1; } diff --git a/src/base58.cpp b/src/base58.cpp index e2f475bb7f..36b3523692 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -134,7 +134,7 @@ bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet) vchRet.clear(); return false; } - // re-calculate the checksum, insure it matches the included 4-byte checksum + // re-calculate the checksum, ensure it matches the included 4-byte checksum uint256 hash = Hash(vchRet.begin(), vchRet.end() - 4); if (memcmp(&hash, &vchRet.end()[-4], 4) != 0) { vchRet.clear(); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 3c9df4f713..33631d2d15 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -5,6 +5,7 @@ #include "bench.h" #include "perf.h" +#include <assert.h> #include <iostream> #include <iomanip> #include <sys/time.h> @@ -92,6 +93,8 @@ bool benchmark::State::KeepRunning() --count; + assert(count != 0 && "count == 0 => (now == 0 && beginTime == 0) => return above"); + // Output results double average = (now-beginTime)/count; int64_t averageCycles = (nowCycles-beginCycles)/count; diff --git a/src/bench/bench.h b/src/bench/bench.h index 0e7605c726..1f36f2a4bc 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,16 +5,17 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <functional> +#include <limits> #include <map> #include <string> -#include <boost/function.hpp> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> // Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark // framework (see https://github.com/google/benchmark) -// Wny not use the Google Benchmark framework? Because adding Yet Another Dependency +// Why not use the Google Benchmark framework? Because adding Yet Another Dependency // (that uses cmake as its build system and has lots of features we don't need) isn't // worth it. @@ -59,7 +60,7 @@ namespace benchmark { bool KeepRunning(); }; - typedef boost::function<void(State&)> BenchFunction; + typedef std::function<void(State&)> BenchFunction; class BenchRunner { diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 1e8e3d462f..5aab3381fd 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -35,14 +35,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50 * CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21 * CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22 * CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 230e4ca773..195388839e 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -22,7 +22,7 @@ static void DeserializeBlockTest(benchmark::State& state) CDataStream stream((const char*)block_bench::block413567, (const char*)&block_bench::block413567[sizeof(block_bench::block413567)], SER_NETWORK, PROTOCOL_VERSION); - char a; + char a = '\0'; stream.write(&a, 1); // Prevent compaction while (state.KeepRunning()) { @@ -37,10 +37,10 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) CDataStream stream((const char*)block_bench::block413567, (const char*)&block_bench::block413567[sizeof(block_bench::block413567)], SER_NETWORK, PROTOCOL_VERSION); - char a; + char a = '\0'; stream.write(&a, 1); // Prevent compaction - Consensus::Params params = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); while (state.KeepRunning()) { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here @@ -48,7 +48,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) assert(stream.Rewind(sizeof(block_bench::block413567))); CValidationState validationState; - assert(CheckBlock(block, validationState, params)); + assert(CheckBlock(block, validationState, chainParams->GetConsensus())); } } diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 6fa9fe4fe8..88a2a570f9 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -68,7 +68,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) PrevectorJob(){ } PrevectorJob(FastRandomContext& insecure_rand){ - p.resize(insecure_rand.rand32() % (PREVECTOR_SIZE*2)); + p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } bool operator()() { diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 29fbd34631..42891f345b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -20,7 +20,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<CO CWalletTx* wtx = new CWalletTx(&wallet, MakeTransactionRef(std::move(tx))); int nAge = 6 * 24; - COutput output(wtx, nInput, nAge, true, true); + COutput output(wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); vCoins.push_back(output); } @@ -48,7 +48,7 @@ static void CoinSelection(benchmark::State& state) addCoin(1000 * COIN, wallet, vCoins); addCoin(3 * COIN, wallet, vCoins); - std::set<std::pair<const CWalletTx*, unsigned int> > setCoinsRet; + std::set<CInputCoin> setCoinsRet; CAmount nValueRet; bool success = wallet.SelectCoinsMinConf(1003 * COIN, 1, 6, 0, vCoins, setCoinsRet, nValueRet); assert(success); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 737d3572ae..2914a36c7b 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -7,6 +7,7 @@ #include "bench.h" #include "bloom.h" #include "hash.h" +#include "random.h" #include "uint256.h" #include "utiltime.h" #include "crypto/ripemd160.h" @@ -69,6 +70,28 @@ static void SipHash_32b(benchmark::State& state) } } +static void FastRandom_32bit(benchmark::State& state) +{ + FastRandomContext rng(true); + uint32_t x = 0; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + x += rng.rand32(); + } + } +} + +static void FastRandom_1bit(benchmark::State& state) +{ + FastRandomContext rng(true); + uint32_t x = 0; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + x += rng.randbool(); + } + } +} + BENCHMARK(RIPEMD160); BENCHMARK(SHA1); BENCHMARK(SHA256); @@ -76,3 +99,5 @@ BENCHMARK(SHA512); BENCHMARK(SHA256_32b); BENCHMARK(SipHash_32b); +BENCHMARK(FastRandom_32bit); +BENCHMARK(FastRandom_1bit); diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index 5df5b1ac6e..43a1422795 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -13,7 +13,7 @@ #define BITER 5000 #define MSIZE 2048 -static void LockedPool(benchmark::State& state) +static void BenchLockedPool(benchmark::State& state) { void *synth_base = reinterpret_cast<void*>(0x08000000); const size_t synth_size = 1024*1024; @@ -43,5 +43,5 @@ static void LockedPool(benchmark::State& state) addr.clear(); } -BENCHMARK(LockedPool); +BENCHMARK(BenchLockedPool); diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 5790d51a82..073bbde016 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -12,14 +12,13 @@ static void AddTx(const CTransaction& tx, const CAmount& nFee, CTxMemPool& pool) { int64_t nTime = 0; - double dPriority = 10.0; unsigned int nHeight = 1; bool spendsCoinbase = false; unsigned int sigOpCost = 4; LockPoints lp; pool.addUnchecked(tx.GetHash(), CTxMemPoolEntry( - MakeTransactionRef(tx), nFee, nTime, dPriority, nHeight, - tx.GetValueOut(), spendsCoinbase, sigOpCost, lp)); + MakeTransactionRef(tx), nFee, nTime, nHeight, + spendsCoinbase, sigOpCost, lp)); } // Right now this is only testing eviction performance in an extremely small @@ -97,7 +96,7 @@ static void MempoolEviction(benchmark::State& state) tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; - CTxMemPool pool(CFeeRate(1000)); + CTxMemPool pool; while (state.KeepRunning()) { AddTx(tx1, 10000LL, pool); diff --git a/src/bench/prevector_destructor.cpp b/src/bench/prevector_destructor.cpp new file mode 100644 index 0000000000..55af3de4fe --- /dev/null +++ b/src/bench/prevector_destructor.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2015-2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" +#include "prevector.h" + +static void PrevectorDestructor(benchmark::State& state) +{ + while (state.KeepRunning()) { + for (auto x = 0; x < 1000; ++x) { + prevector<28, unsigned char> t0; + prevector<28, unsigned char> t1; + t0.resize(28); + t1.resize(29); + } + } +} + +static void PrevectorClear(benchmark::State& state) +{ + + while (state.KeepRunning()) { + for (auto x = 0; x < 1000; ++x) { + prevector<28, unsigned char> t0; + prevector<28, unsigned char> t1; + t0.resize(28); + t0.clear(); + t1.resize(29); + t0.clear(); + } + } +} + +BENCHMARK(PrevectorDestructor); +BENCHMARK(PrevectorClear); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 1c330cf5ea..885b787b4d 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -9,12 +9,12 @@ #include "chainparamsbase.h" #include "clientversion.h" +#include "fs.h" #include "rpc/client.h" #include "rpc/protocol.h" #include "util.h" #include "utilstrencodings.h" -#include <boost/filesystem/operations.hpp> #include <stdio.h> #include <event2/buffer.h> @@ -30,6 +30,8 @@ static const int CONTINUE_EXECUTION=-1; std::string HelpMessageCli() { + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); std::string strUsage; strUsage += HelpMessageGroup(_("Options:")); strUsage += HelpMessageOpt("-?", _("This help message")); @@ -38,11 +40,11 @@ std::string HelpMessageCli() AppendParamsHelpMessages(strUsage); strUsage += HelpMessageOpt("-named", strprintf(_("Pass named instead of positional arguments (default: %s)"), DEFAULT_NAMED)); strUsage += HelpMessageOpt("-rpcconnect=<ip>", strprintf(_("Send commands to node running on <ip> (default: %s)"), DEFAULT_RPCCONNECT)); - strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); + strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); - strUsage += HelpMessageOpt("-rpcclienttimeout=<n>", strprintf(_("Timeout during HTTP requests (default: %d)"), DEFAULT_HTTP_CLIENT_TIMEOUT)); + strUsage += HelpMessageOpt("-rpcclienttimeout=<n>", strprintf(_("Timeout in seconds during HTTP requests, or 0 for no timeout. (default: %d)"), DEFAULT_HTTP_CLIENT_TIMEOUT)); strUsage += HelpMessageOpt("-stdin", _("Read extra arguments from standard input, one per line until EOF/Ctrl-D (recommended for sensitive information such as passphrases)")); return strUsage; @@ -96,7 +98,7 @@ static int AppInitRPC(int argc, char* argv[]) } return EXIT_SUCCESS; } - if (!boost::filesystem::is_directory(GetDataDir(false))) { + if (!fs::is_directory(GetDataDir(false))) { fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", GetArg("-datadir", "").c_str()); return EXIT_FAILURE; } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 3c3646523a..cf280f485c 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -242,6 +242,9 @@ static void MutateTxAddOutAddr(CMutableTransaction& tx, const std::string& strIn std::vector<std::string> vStrInputParts; boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + if (vStrInputParts.size() != 2) + throw std::runtime_error("TX output missing or too many separators"); + // Extract and validate VALUE CAmount value = ExtractAndValidateValue(vStrInputParts[0]); @@ -264,6 +267,9 @@ static void MutateTxAddOutPubKey(CMutableTransaction& tx, const std::string& str std::vector<std::string> vStrInputParts; boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + if (vStrInputParts.size() < 2 || vStrInputParts.size() > 3) + throw std::runtime_error("TX output missing or too many separators"); + // Extract and validate VALUE CAmount value = ExtractAndValidateValue(vStrInputParts[0]); @@ -483,22 +489,6 @@ static bool findSighashFlags(int& flags, const std::string& flagStr) return false; } -uint256 ParseHashUO(std::map<std::string,UniValue>& o, std::string strKey) -{ - if (!o.count(strKey)) - return uint256(); - return ParseHashUV(o[strKey], strKey); -} - -std::vector<unsigned char> ParseHexUO(std::map<std::string,UniValue>& o, std::string strKey) -{ - if (!o.count(strKey)) { - std::vector<unsigned char> emptyVec; - return emptyVec; - } - return ParseHexUV(o[strKey], strKey); -} - static CAmount AmountFromValue(const UniValue& value) { if (!value.isNum() && !value.isStr()) @@ -566,24 +556,26 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (nOut < 0) throw std::runtime_error("vout must be positive"); + COutPoint out(txid, nOut); std::vector<unsigned char> pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw std::runtime_error(err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(prevOut["amount"]); + newcoin.out.nValue = AmountFromValue(prevOut["amount"]); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and private keys given, @@ -605,13 +597,13 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (!coins || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { fComplete = false; continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: @@ -667,11 +659,13 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, MutateTxDelOutput(tx, commandVal); else if (command == "outaddr") MutateTxAddOutAddr(tx, commandVal); - else if (command == "outpubkey") + else if (command == "outpubkey") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxAddOutPubKey(tx, commandVal); - else if (command == "outmultisig") + } else if (command == "outmultisig") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxAddOutMultiSig(tx, commandVal); - else if (command == "outscript") + } else if (command == "outscript") MutateTxAddOutScript(tx, commandVal); else if (command == "outdata") MutateTxAddOutData(tx, commandVal); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index a3d02afcdb..5922e45801 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -10,6 +10,7 @@ #include "chainparams.h" #include "clientversion.h" #include "compat.h" +#include "fs.h" #include "rpc/server.h" #include "init.h" #include "noui.h" @@ -20,7 +21,6 @@ #include "utilstrencodings.h" #include <boost/algorithm/string/predicate.hpp> -#include <boost/filesystem.hpp> #include <boost/thread.hpp> #include <stdio.h> @@ -97,7 +97,7 @@ bool AppInit(int argc, char* argv[]) try { - if (!boost::filesystem::is_directory(GetDataDir(false))) + if (!fs::is_directory(GetDataDir(false))) { fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", GetArg("-datadir", "").c_str()); return false; @@ -117,17 +117,14 @@ bool AppInit(int argc, char* argv[]) return false; } - // Command-line RPC - bool fCommandLine = false; - for (int i = 1; i < argc; i++) - if (!IsSwitchChar(argv[i][0]) && !boost::algorithm::istarts_with(argv[i], "bitcoin:")) - fCommandLine = true; - - if (fCommandLine) - { - fprintf(stderr, "Error: There is no RPC client functionality in bitcoind anymore. Use the bitcoin-cli utility instead.\n"); - exit(EXIT_FAILURE); + // Error out when loose non-argument tokens are encountered on command line + for (int i = 1; i < argc; i++) { + if (!IsSwitchChar(argv[i][0])) { + fprintf(stderr, "Error: Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i]); + exit(EXIT_FAILURE); + } } + // -server defaults to true for bitcoind but not for the GUI so do this here SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 4a311cbba2..ee2c654980 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -147,7 +147,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c // request it. // This should be rare enough that the extra bandwidth doesn't matter, // but eating a round-trip due to FillBlock failure would be annoying - // Note that we dont want duplication between extra_txn and mempool to + // Note that we don't want duplication between extra_txn and mempool to // trigger this case, so we compare witness hashes first if (txn_available[idit->second] && txn_available[idit->second]->GetWitnessHash() != extra_txn[i].second->GetWitnessHash()) { @@ -164,7 +164,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c break; } - LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); + LogPrint(BCLog::CMPCTBLOCK, "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); return READ_STATUS_OK; } @@ -209,10 +209,11 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_CHECKBLOCK_FAILED; } - LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool (incl at least %lu from extra pool) and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, extra_count, vtx_missing.size()); + LogPrint(BCLog::CMPCTBLOCK, "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool (incl at least %lu from extra pool) and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, extra_count, vtx_missing.size()); if (vtx_missing.size() < 5) { - for (const auto& tx : vtx_missing) - LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetHash().ToString()); + for (const auto& tx : vtx_missing) { + LogPrint(BCLog::CMPCTBLOCK, "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetHash().ToString()); + } } return READ_STATUS_OK; diff --git a/src/bloom.cpp b/src/bloom.cpp index 8d47cb76e8..7ed982c984 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -19,7 +19,7 @@ #define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 #define LN2 0.6931471805599453094172321214581765680755001343602552 -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn, unsigned char nFlagsIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn, unsigned char nFlagsIn) : /** * The ideal size for a bloom filter with a given number of elements and false positive rate is: * - nElements * log(fp rate) / ln(2)^2 @@ -40,7 +40,7 @@ CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int } // Private constructor used by CRollingBloomFilter -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn) : vData((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)) / 8), isFull(false), isEmpty(true), @@ -120,7 +120,7 @@ void CBloomFilter::clear() isEmpty = true; } -void CBloomFilter::reset(unsigned int nNewTweak) +void CBloomFilter::reset(const unsigned int nNewTweak) { clear(); nTweak = nNewTweak; @@ -214,7 +214,7 @@ void CBloomFilter::UpdateEmptyFull() isEmpty = empty; } -CRollingBloomFilter::CRollingBloomFilter(unsigned int nElements, double fpRate) +CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const double fpRate) { double logFpRate = log(fpRate); /* The optimal number of hash functions is log(fpRate) / log(0.5), but @@ -254,8 +254,8 @@ void CRollingBloomFilter::insert(const std::vector<unsigned char>& vKey) if (nGeneration == 4) { nGeneration = 1; } - uint64_t nGenerationMask1 = -(uint64_t)(nGeneration & 1); - uint64_t nGenerationMask2 = -(uint64_t)(nGeneration >> 1); + uint64_t nGenerationMask1 = 0 - (uint64_t)(nGeneration & 1); + uint64_t nGenerationMask2 = 0 - (uint64_t)(nGeneration >> 1); /* Wipe old entries that used this generation number. */ for (uint32_t p = 0; p < data.size(); p += 2) { uint64_t p1 = data[p], p2 = data[p + 1]; diff --git a/src/bloom.h b/src/bloom.h index 5ad727c330..7ca9682239 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -54,7 +54,7 @@ private: unsigned int Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const; // Private constructor for CRollingBloomFilter, no restrictions on size - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak); + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak); friend class CRollingBloomFilter; public: @@ -67,7 +67,7 @@ public: * It should generally always be a random value (and is largely only exposed for unit testing) * nFlags should be one of the BLOOM_UPDATE_* enums (not _MASK) */ - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak, unsigned char nFlagsIn); + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn); CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {} ADD_SERIALIZE_METHODS; @@ -89,7 +89,7 @@ public: bool contains(const uint256& hash) const; void clear(); - void reset(unsigned int nNewTweak); + void reset(const unsigned int nNewTweak); //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) @@ -122,7 +122,7 @@ public: // A random bloom filter calls GetRand() at creation time. // Don't create global CRollingBloomFilter objects, as they may be // constructed before the randomizer is properly initialized. - CRollingBloomFilter(unsigned int nElements, double nFPRate); + CRollingBloomFilter(const unsigned int nElements, const double nFPRate); void insert(const std::vector<unsigned char>& vKey); void insert(const uint256& hash); diff --git a/src/chain.h b/src/chain.h index acb29b667b..de120d2d75 100644 --- a/src/chain.h +++ b/src/chain.h @@ -14,6 +14,20 @@ #include <vector> +/** + * Maximum amount of time that a block timestamp is allowed to exceed the + * current network-adjusted time before the block will be accepted. + */ +static const int64_t MAX_FUTURE_BLOCK_TIME = 2 * 60 * 60; + +/** + * Timestamp window used as a grace period by code that compares external + * timestamps (such as timestamps passed to RPCs, or wallet key creation times) + * to block timestamps. This should be set at least as high as + * MAX_FUTURE_BLOCK_TIME. + */ +static const int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; + class CBlockFileInfo { public: @@ -367,9 +381,9 @@ public: template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - int nVersion = s.GetVersion(); + int _nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) - READWRITE(VARINT(nVersion)); + READWRITE(VARINT(_nVersion)); READWRITE(VARINT(nHeight)); READWRITE(VARINT(nStatus)); diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 8c38558829..04a2f680fa 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -55,6 +55,12 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits return CreateGenesisBlock(pszTimestamp, genesisOutputScript, nTime, nNonce, nBits, nVersion, genesisReward); } +void CChainParams::UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +{ + consensus.vDeployments[d].nStartTime = nStartTime; + consensus.vDeployments[d].nTimeout = nTimeout; +} + /** * Main network */ @@ -135,7 +141,6 @@ public: vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main)); - fMiningRequiresPeers = true; fDefaultConsistencyChecks = false; fRequireStandard = true; fMineBlocksOnDemand = false; @@ -166,7 +171,6 @@ public: }; } }; -static CMainParams mainParams; /** * Testnet (v3) @@ -235,7 +239,6 @@ public: vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_test, pnSeed6_test + ARRAYLEN(pnSeed6_test)); - fMiningRequiresPeers = true; fDefaultConsistencyChecks = false; fRequireStandard = false; fMineBlocksOnDemand = false; @@ -255,7 +258,6 @@ public: } }; -static CTestNetParams testNetParams; /** * Regression test @@ -307,7 +309,6 @@ public: vFixedSeeds.clear(); //!< Regtest mode doesn't have any fixed seeds. vSeeds.clear(); //!< Regtest mode doesn't have any DNS seeds. - fMiningRequiresPeers = false; fDefaultConsistencyChecks = true; fRequireStandard = false; fMineBlocksOnDemand = true; @@ -329,42 +330,33 @@ public: base58Prefixes[EXT_PUBLIC_KEY] = boost::assign::list_of(0x04)(0x35)(0x87)(0xCF).convert_to_container<std::vector<unsigned char> >(); base58Prefixes[EXT_SECRET_KEY] = boost::assign::list_of(0x04)(0x35)(0x83)(0x94).convert_to_container<std::vector<unsigned char> >(); } - - void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) - { - consensus.vDeployments[d].nStartTime = nStartTime; - consensus.vDeployments[d].nTimeout = nTimeout; - } }; -static CRegTestParams regTestParams; -static CChainParams *pCurrentParams = 0; +static std::unique_ptr<CChainParams> globalChainParams; const CChainParams &Params() { - assert(pCurrentParams); - return *pCurrentParams; + assert(globalChainParams); + return *globalChainParams; } -CChainParams& Params(const std::string& chain) +std::unique_ptr<CChainParams> CreateChainParams(const std::string& chain) { if (chain == CBaseChainParams::MAIN) - return mainParams; + return std::unique_ptr<CChainParams>(new CMainParams()); else if (chain == CBaseChainParams::TESTNET) - return testNetParams; + return std::unique_ptr<CChainParams>(new CTestNetParams()); else if (chain == CBaseChainParams::REGTEST) - return regTestParams; - else - throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); + return std::unique_ptr<CChainParams>(new CRegTestParams()); + throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } void SelectParams(const std::string& network) { SelectBaseParams(network); - pCurrentParams = &Params(network); + globalChainParams = CreateChainParams(network); } -void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +void UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) { - regTestParams.UpdateBIP9Parameters(d, nStartTime, nTimeout); + globalChainParams->UpdateVersionBitsParameters(d, nStartTime, nTimeout); } - diff --git a/src/chainparams.h b/src/chainparams.h index db524e8f8e..a2f136171b 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -11,6 +11,7 @@ #include "primitives/block.h" #include "protocol.h" +#include <memory> #include <vector> struct CDNSSeedData { @@ -61,8 +62,6 @@ public: int GetDefaultPort() const { return nDefaultPort; } const CBlock& GenesisBlock() const { return genesis; } - /** Make miner wait to have peers to avoid wasting work */ - bool MiningRequiresPeers() const { return fMiningRequiresPeers; } /** Default value for -checkmempool and -checkblockindex argument */ bool DefaultConsistencyChecks() const { return fDefaultConsistencyChecks; } /** Policy: Filter transactions that do not match well-defined patterns */ @@ -77,6 +76,7 @@ public: const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; } const CCheckpointData& Checkpoints() const { return checkpointData; } const ChainTxData& TxData() const { return chainTxData; } + void UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); protected: CChainParams() {} @@ -89,7 +89,6 @@ protected: std::string strNetworkID; CBlock genesis; std::vector<SeedSpec6> vFixedSeeds; - bool fMiningRequiresPeers; bool fDefaultConsistencyChecks; bool fRequireStandard; bool fMineBlocksOnDemand; @@ -98,15 +97,17 @@ protected: }; /** - * Return the currently selected parameters. This won't change after app - * startup, except for unit tests. + * Creates and returns a std::unique_ptr<CChainParams> of the chosen chain. + * @returns a CChainParams* of the chosen chain. + * @throws a std::runtime_error if the chain is not supported. */ -const CChainParams &Params(); +std::unique_ptr<CChainParams> CreateChainParams(const std::string& chain); /** - * @returns CChainParams for the given BIP70 chain name. + * Return the currently selected parameters. This won't change after app + * startup, except for unit tests. */ -CChainParams& Params(const std::string& chain); +const CChainParams &Params(); /** * Sets the params returned by Params() to those for the given BIP70 chain name. @@ -115,8 +116,8 @@ CChainParams& Params(const std::string& chain); void SelectParams(const std::string& chain); /** - * Allows modifying the BIP9 regtest parameters. + * Allows modifying the Version Bits regtest parameters. */ -void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); +void UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index cb71a8b550..43c9a13c54 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -35,7 +35,6 @@ public: nRPCPort = 8332; } }; -static CBaseMainParams mainParams; /** * Testnet (v3) @@ -49,7 +48,6 @@ public: strDataDir = "testnet3"; } }; -static CBaseTestNetParams testNetParams; /* * Regression test @@ -63,31 +61,30 @@ public: strDataDir = "regtest"; } }; -static CBaseRegTestParams regTestParams; -static CBaseChainParams* pCurrentBaseParams = 0; +static std::unique_ptr<CBaseChainParams> globalChainBaseParams; const CBaseChainParams& BaseParams() { - assert(pCurrentBaseParams); - return *pCurrentBaseParams; + assert(globalChainBaseParams); + return *globalChainBaseParams; } -CBaseChainParams& BaseParams(const std::string& chain) +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain) { if (chain == CBaseChainParams::MAIN) - return mainParams; + return std::unique_ptr<CBaseChainParams>(new CBaseMainParams()); else if (chain == CBaseChainParams::TESTNET) - return testNetParams; + return std::unique_ptr<CBaseChainParams>(new CBaseTestNetParams()); else if (chain == CBaseChainParams::REGTEST) - return regTestParams; + return std::unique_ptr<CBaseChainParams>(new CBaseRegTestParams()); else throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } void SelectBaseParams(const std::string& chain) { - pCurrentBaseParams = &BaseParams(chain); + globalChainBaseParams = CreateBaseChainParams(chain); } std::string ChainNameFromCommandLine() @@ -103,8 +100,3 @@ std::string ChainNameFromCommandLine() return CBaseChainParams::TESTNET; return CBaseChainParams::MAIN; } - -bool AreBaseParamsConfigured() -{ - return pCurrentBaseParams != NULL; -} diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index 59493afb9b..fc101f5b77 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_CHAINPARAMSBASE_H #define BITCOIN_CHAINPARAMSBASE_H +#include <memory> #include <string> #include <vector> @@ -31,6 +32,13 @@ protected: }; /** + * Creates and returns a std::unique_ptr<CBaseChainParams> of the chosen chain. + * @returns a CBaseChainParams* of the chosen chain. + * @throws a std::runtime_error if the chain is not supported. + */ +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain); + +/** * Append the help messages for the chainparams options to the * parameter string. */ @@ -42,8 +50,6 @@ void AppendParamsHelpMessages(std::string& strUsage, bool debugHelp=true); */ const CBaseChainParams& BaseParams(); -CBaseChainParams& BaseParams(const std::string& chain); - /** Sets the params returned by Params() to those for the given network. */ void SelectBaseParams(const std::string& chain); @@ -53,10 +59,4 @@ void SelectBaseParams(const std::string& chain); */ std::string ChainNameFromCommandLine(); -/** - * Return true if SelectBaseParamsFromCommandLine() has been called to select - * a network. - */ -bool AreBaseParamsConfigured(); - #endif // BITCOIN_CHAINPARAMSBASE_H diff --git a/src/checkqueue.h b/src/checkqueue.h index 32e25d5c8c..63c104c02a 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CHECKQUEUE_H #define BITCOIN_CHECKQUEUE_H +#include "sync.h" + #include <algorithm> #include <vector> @@ -127,6 +129,9 @@ private: } public: + //! Mutex to ensure only one concurrent CCheckQueueControl + boost::mutex ControlMutex; + //! Create a new check queue CCheckQueue(unsigned int nBatchSizeIn) : nIdle(0), nTotal(0), fAllOk(true), nTodo(0), fQuit(false), nBatchSize(nBatchSizeIn) {} @@ -161,12 +166,6 @@ public: { } - bool IsIdle() - { - boost::unique_lock<boost::mutex> lock(mutex); - return (nTotal == nIdle && nTodo == 0 && fAllOk == true); - } - }; /** @@ -177,16 +176,18 @@ template <typename T> class CCheckQueueControl { private: - CCheckQueue<T>* pqueue; + CCheckQueue<T> * const pqueue; bool fDone; public: - CCheckQueueControl(CCheckQueue<T>* pqueueIn) : pqueue(pqueueIn), fDone(false) + CCheckQueueControl() = delete; + CCheckQueueControl(const CCheckQueueControl&) = delete; + CCheckQueueControl& operator=(const CCheckQueueControl&) = delete; + explicit CCheckQueueControl(CCheckQueue<T> * const pqueueIn) : pqueue(pqueueIn), fDone(false) { // passed queue is supposed to be unused, or NULL if (pqueue != NULL) { - bool isIdle = pqueue->IsIdle(); - assert(isIdle); + ENTER_CRITICAL_SECTION(pqueue->ControlMutex); } } @@ -209,6 +210,9 @@ public: { if (!fDone) Wait(); + if (pqueue != NULL) { + LEAVE_CRITICAL_SECTION(pqueue->ControlMutex); + } } }; diff --git a/src/clientversion.h b/src/clientversion.h index 69154d546d..3d5392619b 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -7,29 +7,13 @@ #if defined(HAVE_CONFIG_H) #include "config/bitcoin-config.h" -#else - -/** - * client versioning and copyright year - */ - -//! These need to be macros, as clientversion.cpp's and bitcoin*-res.rc's voodoo requires it -#define CLIENT_VERSION_MAJOR 0 -#define CLIENT_VERSION_MINOR 14 -#define CLIENT_VERSION_REVISION 99 -#define CLIENT_VERSION_BUILD 0 - -//! Set to true for release, false for prerelease or test build -#define CLIENT_VERSION_IS_RELEASE false - -/** - * Copyright year (2009-this) - * Todo: update this when changing our copyright comments in the source - */ -#define COPYRIGHT_YEAR 2017 - #endif //HAVE_CONFIG_H +// Check that required client information is defined +#if !defined(CLIENT_VERSION_MAJOR) || !defined(CLIENT_VERSION_MINOR) || !defined(CLIENT_VERSION_REVISION) || !defined(CLIENT_VERSION_BUILD) || !defined(CLIENT_VERSION_IS_RELEASE) || !defined(COPYRIGHT_YEAR) +#error Client version information missing: version is not defined by bitcoin-config.h or in any other way +#endif + /** * Converts the parameter X to a string after macro replacement on X has been performed. * Don't merge these into one macro! diff --git a/src/coins.cpp b/src/coins.cpp index 4d0e4bc0ad..5b7c562678 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -4,174 +4,126 @@ #include "coins.h" +#include "consensus/consensus.h" #include "memusage.h" #include "random.h" #include <assert.h> -/** - * calculate number of bytes for the bitmask, and its number of non-zero bytes - * each bit in the bitmask represents the availability of one output, but the - * availabilities of the first two outputs are encoded separately - */ -void CCoins::CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const { - unsigned int nLastUsedByte = 0; - for (unsigned int b = 0; 2+b*8 < vout.size(); b++) { - bool fZero = true; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) { - if (!vout[2+b*8+i].IsNull()) { - fZero = false; - continue; - } - } - if (!fZero) { - nLastUsedByte = b + 1; - nNonzeroBytes++; - } - } - nBytes += nLastUsedByte; -} - -bool CCoins::Spend(uint32_t nPos) -{ - if (nPos >= vout.size() || vout[nPos].IsNull()) - return false; - vout[nPos].SetNull(); - Cleanup(); - return true; -} - -bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { return false; } -bool CCoinsView::HaveCoins(const uint256 &txid) const { return false; } +bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } +bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return 0; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } -bool CCoinsViewBacked::GetCoins(const uint256 &txid, CCoins &coins) const { return base->GetCoins(txid, coins); } -bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { return base->HaveCoins(txid); } +bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } +bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } +size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } - -CCoinsViewCache::~CCoinsViewCache() -{ - assert(!hasModifier); -} +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { - CCoinsMap::iterator it = cacheCoins.find(txid); +CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { + CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) return it; - CCoins tmp; - if (!base->GetCoins(txid, tmp)) + Coin tmp; + if (!base->GetCoin(outpoint, tmp)) return cacheCoins.end(); - CCoinsMap::iterator ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())).first; - tmp.swap(ret->second.coins); - if (ret->second.coins.IsPruned()) { - // The parent only has an empty entry for this txid; we can consider our + CCoinsMap::iterator ret = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))).first; + if (ret->second.coin.IsSpent()) { + // The parent only has an empty entry for this outpoint; we can consider our // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } - cachedCoinsUsage += ret->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } -bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it != cacheCoins.end()) { - coins = it->second.coins; + coin = it->second.coin; return true; } return false; } -CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { - assert(!hasModifier); - std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - size_t cachedCoinUsage = 0; - if (ret.second) { - if (!base->GetCoins(txid, ret.first->second.coins)) { - // The parent view does not have this entry; mark it as fresh. - ret.first->second.coins.Clear(); - ret.first->second.flags = CCoinsCacheEntry::FRESH; - } else if (ret.first->second.coins.IsPruned()) { - // The parent view only has a pruned entry for this; mark it as fresh. - ret.first->second.flags = CCoinsCacheEntry::FRESH; +void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { + assert(!coin.IsSpent()); + if (coin.out.scriptPubKey.IsUnspendable()) return; + CCoinsMap::iterator it; + bool inserted; + std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); + bool fresh = false; + if (!inserted) { + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + } + if (!possible_overwrite) { + if (!it->second.coin.IsSpent()) { + throw std::logic_error("Adding new coin that replaces non-pruned entry"); } - } else { - cachedCoinUsage = ret.first->second.coins.DynamicMemoryUsage(); + fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); + } + it->second.coin = std::move(coin); + it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); + cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); +} + +void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { + bool fCoinbase = tx.IsCoinBase(); + const uint256& txid = tx.GetHash(); + for (size_t i = 0; i < tx.vout.size(); ++i) { + // Pass fCoinbase as the possible_overwrite flag to AddCoin, in order to correctly + // deal with the pre-BIP30 occurrances of duplicate coinbase transactions. + cache.AddCoin(COutPoint(txid, i), Coin(tx.vout[i], nHeight, fCoinbase), fCoinbase); } - // Assume that whenever ModifyCoins is called, the entry will be modified. - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, cachedCoinUsage); } -/* ModifyNewCoins allows for faster coin modification when creating the new - * outputs from a transaction. It assumes that BIP 30 (no duplicate txids) - * applies and has already been tested for (or the test is not required due to - * BIP 34, height in coinbase). If we can assume BIP 30 then we know that any - * non-coinbase transaction we are adding to the UTXO must not already exist in - * the utxo unless it is fully spent. Thus we can check only if it exists DIRTY - * at the current level of the cache, in which case it is not safe to mark it - * FRESH (b/c then its spentness still needs to flushed). If it's not dirty and - * doesn't exist or is pruned in the current cache, we know it either doesn't - * exist or is pruned in parent caches, which is the definition of FRESH. The - * exception to this is the two historical violations of BIP 30 in the chain, - * both of which were coinbases. We do not mark these fresh so we we can ensure - * that they will still be properly overwritten when spent. - */ -CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid, bool coinbase) { - assert(!hasModifier); - std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - if (!coinbase) { - // New coins must not already exist. - if (!ret.first->second.coins.IsPruned()) - throw std::logic_error("ModifyNewCoins should not find pre-existing coins on a non-coinbase unless they are pruned!"); - - if (!(ret.first->second.flags & CCoinsCacheEntry::DIRTY)) { - // If the coin is known to be pruned (have no unspent outputs) in - // the current view and the cache entry is not dirty, we know the - // coin also must be pruned in the parent view as well, so it is safe - // to mark this fresh. - ret.first->second.flags |= CCoinsCacheEntry::FRESH; - } +void CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { + CCoinsMap::iterator it = FetchCoin(outpoint); + if (it == cacheCoins.end()) return; + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + if (moveout) { + *moveout = std::move(it->second.coin); + } + if (it->second.flags & CCoinsCacheEntry::FRESH) { + cacheCoins.erase(it); + } else { + it->second.flags |= CCoinsCacheEntry::DIRTY; + it->second.coin.Clear(); } - ret.first->second.coins.Clear(); - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, 0); } -const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +static const Coin coinEmpty; + +const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { - return NULL; + return coinEmpty; } else { - return &it->second.coins; + return it->second.coin; } } -bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - // We're using vtx.empty() instead of IsPruned here for performance reasons, - // as we only care about the case where a transaction was replaced entirely - // in a reorganization (which wipes vout entirely, as opposed to spending - // which just cleans individual outputs). - return (it != cacheCoins.end() && !it->second.coins.vout.empty()); +bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + return (it != cacheCoins.end() && !it->second.coin.IsSpent()); } -bool CCoinsViewCache::HaveCoinsInCache(const uint256 &txid) const { - CCoinsMap::const_iterator it = cacheCoins.find(txid); +bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return it != cacheCoins.end(); } @@ -186,19 +138,18 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - assert(!hasModifier); for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { // The parent cache does not have an entry, while the child does // We can ignore it if it's both FRESH and pruned in the child - if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coins.IsPruned())) { + if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { // Otherwise we will need to create it in the parent // and move the data up and mark it as dirty CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coins.swap(it->second.coins); - cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the child // Otherwise it might have just been flushed from the parent's cache @@ -211,21 +162,21 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // parent cache entry has unspent outputs. If this ever happens, // it means the FRESH flag was misapplied and there is a logic // error in the calling code. - if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coins.IsPruned()) + if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); // Found the entry in the parent cache - if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { + if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); - itUs->second.coins.swap(it->second.coins); - cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in // the event the entry we found in the parent is pruned. But @@ -249,11 +200,11 @@ bool CCoinsViewCache::Flush() { return fOk; } -void CCoinsViewCache::Uncache(const uint256& hash) +void CCoinsViewCache::Uncache(const COutPoint& hash) { CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } @@ -262,13 +213,6 @@ unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } -const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn& input) const -{ - const CCoins* coins = AccessCoins(input.prevout.hash); - assert(coins && coins->IsAvailable(input.prevout.n)); - return coins->vout[input.prevout.n]; -} - CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const { if (tx.IsCoinBase()) @@ -276,7 +220,7 @@ CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const CAmount nResult = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) - nResult += GetOutputFor(tx.vin[i]).nValue; + nResult += AccessCoin(tx.vin[i].prevout).out.nValue; return nResult; } @@ -285,9 +229,7 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { for (unsigned int i = 0; i < tx.vin.size(); i++) { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = AccessCoins(prevout.hash); - if (!coins || !coins->IsAvailable(prevout.n)) { + if (!HaveCoin(tx.vin[i].prevout)) { return false; } } @@ -295,44 +237,15 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } -double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight, CAmount &inChainInputValue) const -{ - inChainInputValue = 0; - if (tx.IsCoinBase()) - return 0.0; - double dResult = 0.0; - BOOST_FOREACH(const CTxIn& txin, tx.vin) - { - const CCoins* coins = AccessCoins(txin.prevout.hash); - assert(coins); - if (!coins->IsAvailable(txin.prevout.n)) continue; - if (coins->nHeight <= nHeight) { - dResult += (double)(coins->vout[txin.prevout.n].nValue) * (nHeight-coins->nHeight); - inChainInputValue += coins->vout[txin.prevout.n].nValue; - } - } - return tx.ComputePriority(dResult); -} - -CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage) : cache(cache_), it(it_), cachedCoinUsage(usage) { - assert(!cache.hasModifier); - cache.hasModifier = true; -} +static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. -CCoinsModifier::~CCoinsModifier() +const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) { - assert(cache.hasModifier); - cache.hasModifier = false; - it->second.coins.Cleanup(); - cache.cachedCoinsUsage -= cachedCoinUsage; // Subtract the old usage - if ((it->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { - cache.cacheCoins.erase(it); - } else { - // If the coin still exists after the modification, add the new usage - cache.cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); + COutPoint iter(txid, 0); + while (iter.n < MAX_OUTPUTS_PER_BLOCK) { + const Coin& alternate = view.AccessCoin(iter); + if (!alternate.IsSpent()) return alternate; + ++iter.n; } -} - -CCoinsViewCursor::~CCoinsViewCursor() -{ + return coinEmpty; } diff --git a/src/coins.h b/src/coins.h index 902cb57f69..476db8f37c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H +#include "primitives/transaction.h" #include "compressor.h" #include "core_memusage.h" #include "hash.h" @@ -17,137 +18,39 @@ #include <stdint.h> #include <boost/foreach.hpp> -#include <boost/unordered_map.hpp> +#include <unordered_map> -/** - * Pruned version of CTransaction: only retains metadata and unspent transaction outputs +/** + * A UTXO entry. * * Serialized format: - * - VARINT(nVersion) - * - VARINT(nCode) - * - unspentness bitvector, for vout[2] and further; least significant byte first - * - the non-spent CTxOuts (via CTxOutCompressor) - * - VARINT(nHeight) - * - * The nCode value consists of: - * - bit 0: IsCoinBase() - * - bit 1: vout[0] is not spent - * - bit 2: vout[1] is not spent - * - The higher bits encode N, the number of non-zero bytes in the following bitvector. - * - In case both bit 1 and bit 2 are unset, they encode N-1, as there must be at - * least one non-spent output). - * - * Example: 0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e - * <><><--------------------------------------------><----> - * | \ | / - * version code vout[1] height - * - * - version = 1 - * - code = 4 (vout[1] is not spent, and 0 non-zero bytes of bitvector follow) - * - unspentness bitvector: as 0 non-zero bytes follow, it has length 0 - * - vout[1]: 835800816115944e077fe7c803cfa57f29b36bf87c1d35 - * * 8358: compact amount representation for 60000000000 (600 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 816115944e077fe7c803cfa57f29b36bf87c1d35: address uint160 - * - height = 203998 - * - * - * Example: 0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b - * <><><--><--------------------------------------------------><----------------------------------------------><----> - * / \ \ | | / - * version code unspentness vout[4] vout[16] height - * - * - version = 1 - * - code = 9 (coinbase, neither vout[0] or vout[1] are unspent, - * 2 (1, +1 because both bit 1 and bit 2 are unset) non-zero bitvector bytes follow) - * - unspentness bitvector: bits 2 (0x04) and 14 (0x4000) are set, so vout[2+2] and vout[14+2] are unspent - * - vout[4]: 86ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4ee - * * 86ef97d579: compact amount representation for 234925952 (2.35 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 61b01caab50f1b8e9c50a5057eb43c2d9563a4ee: address uint160 - * - vout[16]: bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4 - * * bbd123: compact amount representation for 110397 (0.001 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 8c988f1a4a4de2161e0f50aac7f17e7f9555caa4: address uint160 - * - height = 120891 + * - VARINT((coinbase ? 1 : 0) | (height << 1)) + * - the non-spent CTxOut (via CTxOutCompressor) */ -class CCoins +class Coin { public: - //! whether transaction is a coinbase - bool fCoinBase; + //! unspent transaction output + CTxOut out; - //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped - std::vector<CTxOut> vout; + //! whether containing transaction was a coinbase + unsigned int fCoinBase : 1; - //! at which height this transaction was included in the active block chain - int nHeight; + //! at which height this containing transaction was included in the active block chain + uint32_t nHeight : 31; - //! version of the CTransaction; accesses to this value should probably check for nHeight as well, - //! as new tx version will probably only be introduced at certain heights - int nVersion; - - void FromTx(const CTransaction &tx, int nHeightIn) { - fCoinBase = tx.IsCoinBase(); - vout = tx.vout; - nHeight = nHeightIn; - nVersion = tx.nVersion; - ClearUnspendable(); - } - - //! construct a CCoins from a CTransaction, at a given height - CCoins(const CTransaction &tx, int nHeightIn) { - FromTx(tx, nHeightIn); - } + //! construct a Coin from a CTxOut and height/coinbase information. + Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : out(std::move(outIn)), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {} + Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : out(outIn), fCoinBase(fCoinBaseIn),nHeight(nHeightIn) {} void Clear() { + out.SetNull(); fCoinBase = false; - std::vector<CTxOut>().swap(vout); nHeight = 0; - nVersion = 0; } //! empty constructor - CCoins() : fCoinBase(false), vout(0), nHeight(0), nVersion(0) { } - - //!remove spent outputs at the end of vout - void Cleanup() { - while (vout.size() > 0 && vout.back().IsNull()) - vout.pop_back(); - if (vout.empty()) - std::vector<CTxOut>().swap(vout); - } - - void ClearUnspendable() { - BOOST_FOREACH(CTxOut &txout, vout) { - if (txout.scriptPubKey.IsUnspendable()) - txout.SetNull(); - } - Cleanup(); - } - - void swap(CCoins &to) { - std::swap(to.fCoinBase, fCoinBase); - to.vout.swap(vout); - std::swap(to.nHeight, nHeight); - std::swap(to.nVersion, nVersion); - } - - //! equality test - friend bool operator==(const CCoins &a, const CCoins &b) { - // Empty CCoins objects are always equal. - if (a.IsPruned() && b.IsPruned()) - return true; - return a.fCoinBase == b.fCoinBase && - a.nHeight == b.nHeight && - a.nVersion == b.nVersion && - a.vout == b.vout; - } - friend bool operator!=(const CCoins &a, const CCoins &b) { - return !(a == b); - } - - void CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const; + Coin() : fCoinBase(false), nHeight(0) { } bool IsCoinBase() const { return fCoinBase; @@ -155,115 +58,52 @@ public: template<typename Stream> void Serialize(Stream &s) const { - unsigned int nMaskSize = 0, nMaskCode = 0; - CalcMaskSize(nMaskSize, nMaskCode); - bool fFirst = vout.size() > 0 && !vout[0].IsNull(); - bool fSecond = vout.size() > 1 && !vout[1].IsNull(); - assert(fFirst || fSecond || nMaskCode); - unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); - // version - ::Serialize(s, VARINT(this->nVersion)); - // header code - ::Serialize(s, VARINT(nCode)); - // spentness bitmask - for (unsigned int b = 0; b<nMaskSize; b++) { - unsigned char chAvail = 0; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) - if (!vout[2+b*8+i].IsNull()) - chAvail |= (1 << i); - ::Serialize(s, chAvail); - } - // txouts themself - for (unsigned int i = 0; i < vout.size(); i++) { - if (!vout[i].IsNull()) - ::Serialize(s, CTxOutCompressor(REF(vout[i]))); - } - // coinbase height - ::Serialize(s, VARINT(nHeight)); + assert(!IsSpent()); + uint32_t code = nHeight * 2 + fCoinBase; + ::Serialize(s, VARINT(code)); + ::Serialize(s, CTxOutCompressor(REF(out))); } template<typename Stream> void Unserialize(Stream &s) { - unsigned int nCode = 0; - // version - ::Unserialize(s, VARINT(this->nVersion)); - // header code - ::Unserialize(s, VARINT(nCode)); - fCoinBase = nCode & 1; - std::vector<bool> vAvail(2, false); - vAvail[0] = (nCode & 2) != 0; - vAvail[1] = (nCode & 4) != 0; - unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); - // spentness bitmask - while (nMaskCode > 0) { - unsigned char chAvail = 0; - ::Unserialize(s, chAvail); - for (unsigned int p = 0; p < 8; p++) { - bool f = (chAvail & (1 << p)) != 0; - vAvail.push_back(f); - } - if (chAvail != 0) - nMaskCode--; - } - // txouts themself - vout.assign(vAvail.size(), CTxOut()); - for (unsigned int i = 0; i < vAvail.size(); i++) { - if (vAvail[i]) - ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); - } - // coinbase height - ::Unserialize(s, VARINT(nHeight)); - Cleanup(); - } - - //! mark a vout spent - bool Spend(uint32_t nPos); - - //! check whether a particular output is still available - bool IsAvailable(unsigned int nPos) const { - return (nPos < vout.size() && !vout[nPos].IsNull()); + uint32_t code = 0; + ::Unserialize(s, VARINT(code)); + nHeight = code >> 1; + fCoinBase = code & 1; + ::Unserialize(s, REF(CTxOutCompressor(out))); } - //! check whether the entire CCoins is spent - //! note that only !IsPruned() CCoins can be serialized - bool IsPruned() const { - BOOST_FOREACH(const CTxOut &out, vout) - if (!out.IsNull()) - return false; - return true; + bool IsSpent() const { + return out.IsNull(); } size_t DynamicMemoryUsage() const { - size_t ret = memusage::DynamicUsage(vout); - BOOST_FOREACH(const CTxOut &out, vout) { - ret += RecursiveDynamicUsage(out.scriptPubKey); - } - return ret; + return memusage::DynamicUsage(out.scriptPubKey); } }; -class SaltedTxidHasher +class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: - SaltedTxidHasher(); + SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). */ - size_t operator()(const uint256& txid) const { - return SipHashUint256(k0, k1, txid); + size_t operator()(const COutPoint& id) const { + return SipHashUint256Extra(k0, k1, id.hash, id.n); } }; struct CCoinsCacheEntry { - CCoins coins; // The actual cached data. + Coin coin; // The actual cached data. unsigned char flags; enum Flags { @@ -276,21 +116,21 @@ struct CCoinsCacheEntry */ }; - CCoinsCacheEntry() : coins(), flags(0) {} + CCoinsCacheEntry() : flags(0) {} + explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {} }; -typedef boost::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap; +typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {} - virtual ~CCoinsViewCursor(); + virtual ~CCoinsViewCursor() {} - virtual bool GetKey(uint256 &key) const = 0; - virtual bool GetValue(CCoins &coins) const = 0; - /* Don't care about GetKeySize here */ + virtual bool GetKey(COutPoint &key) const = 0; + virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; @@ -306,17 +146,17 @@ private: class CCoinsView { public: - //! Retrieve the CCoins (unspent transaction outputs) for a given txid - virtual bool GetCoins(const uint256 &txid, CCoins &coins) const; + //! Retrieve the Coin (unspent transaction output) for a given outpoint. + virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; - //! Just check whether we have data for a given txid. - //! This may (but cannot always) return true for fully spent transactions - virtual bool HaveCoins(const uint256 &txid) const; + //! Just check whether we have data for a given outpoint. + //! This may (but cannot always) return true for spent outputs. + virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; - //! Do a bulk modification (multiple CCoins changes + BestBlock change). + //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -325,6 +165,9 @@ public: //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} + + //! Estimate database size (0 if not implemented) + virtual size_t EstimateSize() const { return 0; } }; @@ -336,45 +179,20 @@ protected: public: CCoinsViewBacked(CCoinsView *viewIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; void SetBackend(CCoinsView &viewIn); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + size_t EstimateSize() const override; }; -class CCoinsViewCache; - -/** - * A reference to a mutable cache entry. Encapsulating it allows us to run - * cleanup code after the modification is finished, and keeping track of - * concurrent modifications. - */ -class CCoinsModifier -{ -private: - CCoinsViewCache& cache; - CCoinsMap::iterator it; - size_t cachedCoinUsage; // Cached memory usage of the CCoins object before modification - CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage); - -public: - CCoins* operator->() { return &it->second.coins; } - CCoins& operator*() { return it->second.coins; } - ~CCoinsModifier(); - friend class CCoinsViewCache; -}; - /** CCoinsView that adds a memory cache for transactions to another CCoinsView */ class CCoinsViewCache : public CCoinsViewBacked { protected: - /* Whether this cache has an active modifier. */ - bool hasModifier; - - /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". @@ -382,51 +200,45 @@ protected: mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; - /* Cached dynamic memory usage for the inner CCoins objects. */ + /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; public: CCoinsViewCache(CCoinsView *baseIn); - ~CCoinsViewCache(); // Standard CCoinsView methods - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); /** - * Check if we have the given tx already loaded in this cache. - * The semantics are the same as HaveCoins(), but no calls to + * Check if we have the given utxo already loaded in this cache. + * The semantics are the same as HaveCoin(), but no calls to * the backing CCoinsView are made. */ - bool HaveCoinsInCache(const uint256 &txid) const; + bool HaveCoinInCache(const COutPoint &outpoint) const; /** - * Return a pointer to CCoins in the cache, or NULL if not found. This is - * more efficient than GetCoins. Modifications to other cache entries are + * Return a reference to Coin in the cache, or a pruned one if not found. This is + * more efficient than GetCoin. Modifications to other cache entries are * allowed while accessing the returned pointer. */ - const CCoins* AccessCoins(const uint256 &txid) const; + const Coin& AccessCoin(const COutPoint &output) const; /** - * Return a modifiable reference to a CCoins. If no entry with the given - * txid exists, a new one is created. Simultaneous modifications are not - * allowed. + * Add a coin. Set potential_overwrite to true if a non-pruned version may + * already exist. */ - CCoinsModifier ModifyCoins(const uint256 &txid); + void AddCoin(const COutPoint& outpoint, Coin&& coin, bool potential_overwrite); /** - * Return a modifiable reference to a CCoins. Assumes that no entry with the given - * txid exists and creates a new one. This saves a database access in the case where - * the coins were to be wiped out by FromTx anyway. This should not be called with - * the 2 historical coinbase duplicate pairs because the new coins are marked fresh, and - * in the event the duplicate coinbase was spent before a flush, the now pruned coins - * would not properly overwrite the first coinbase of the pair. Simultaneous modifications - * are not allowed. + * Spend a coin. Pass moveto in order to get the deleted data. + * If no unspent output exists for the passed outpoint, this call + * has no effect. */ - CCoinsModifier ModifyNewCoins(const uint256 &txid, bool coinbase); + void SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); /** * Push the modifications applied to this cache to its base. @@ -436,12 +248,12 @@ public: bool Flush(); /** - * Removes the transaction with the given hash from the cache, if it is + * Removes the UTXO with the given outpoint from the cache, if it is * not modified. */ - void Uncache(const uint256 &txid); + void Uncache(const COutPoint &outpoint); - //! Calculate the size of the cache (in number of transactions) + //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) @@ -460,19 +272,8 @@ public: //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; - /** - * Return priority of tx at height nHeight. Also calculate the sum of the values of the inputs - * that are already in the chain. These are the inputs that will age and increase priority as - * new blocks are added to the chain. - */ - double GetPriority(const CTransaction &tx, int nHeight, CAmount &inChainInputValue) const; - - const CTxOut &GetOutputFor(const CTxIn& input) const; - - friend class CCoinsModifier; - private: - CCoinsMap::const_iterator FetchCoins(const uint256 &txid) const; + CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it when one intends to create a cache on top of a base cache. @@ -480,4 +281,13 @@ private: CCoinsViewCache(const CCoinsViewCache &); }; +//! Utility function to add all of a transaction's outputs to a cache. +// It assumes that overwrites are only possible for coinbase transactions, +// TODO: pass in a boolean to limit these possible overwrites to known +// (pre-BIP34) cases. +void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight); + +//! Utility function to find any unspent output with a given txid. +const Coin& AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); + #endif // BITCOIN_COINS_H diff --git a/src/compat.h b/src/compat.h index 28aa77eea2..e76ab94c82 100644 --- a/src/compat.h +++ b/src/compat.h @@ -47,10 +47,8 @@ #include <unistd.h> #endif -#ifdef WIN32 -#define MSG_DONTWAIT 0 -#else -typedef u_int SOCKET; +#ifndef WIN32 +typedef unsigned int SOCKET; #include "errno.h" #define WSAGetLastError() errno #define WSAEINVAL EINVAL @@ -74,11 +72,6 @@ typedef u_int SOCKET; #define MAX_PATH 1024 #endif -// As Solaris does not have the MSG_NOSIGNAL flag for send(2) syscall, it is defined as 0 -#if !defined(HAVE_MSG_NOSIGNAL) && !defined(MSG_NOSIGNAL) -#define MSG_NOSIGNAL 0 -#endif - #if HAVE_DECL_STRNLEN == 0 size_t strnlen( const char *start, size_t max_len); #endif // HAVE_DECL_STRNLEN diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp new file mode 100644 index 0000000000..bf68f8754b --- /dev/null +++ b/src/consensus/tx_verify.cpp @@ -0,0 +1,246 @@ +// Copyright (c) 2017-2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "tx_verify.h" + +#include "consensus.h" +#include "primitives/transaction.h" +#include "script/interpreter.h" +#include "validation.h" + +// TODO remove the following dependencies +#include "chain.h" +#include "coins.h" +#include "utilmoneystr.h" + +bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) +{ + if (tx.nLockTime == 0) + return true; + if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) + return true; + for (const auto& txin : tx.vin) { + if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) + return false; + } + return true; +} + +std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) +{ + assert(prevHeights->size() == tx.vin.size()); + + // Will be set to the equivalent height- and time-based nLockTime + // values that would be necessary to satisfy all relative lock- + // time constraints given our view of block chain history. + // The semantics of nLockTime are the last invalid height/time, so + // use -1 to have the effect of any height or time being valid. + int nMinHeight = -1; + int64_t nMinTime = -1; + + // tx.nVersion is signed integer so requires cast to unsigned otherwise + // we would be doing a signed comparison and half the range of nVersion + // wouldn't support BIP 68. + bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 + && flags & LOCKTIME_VERIFY_SEQUENCE; + + // Do not enforce sequence numbers as a relative lock time + // unless we have been instructed to + if (!fEnforceBIP68) { + return std::make_pair(nMinHeight, nMinTime); + } + + for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { + const CTxIn& txin = tx.vin[txinIndex]; + + // Sequence numbers with the most significant bit set are not + // treated as relative lock-times, nor are they given any + // consensus-enforced meaning at this point. + if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { + // The height of this input is not relevant for sequence locks + (*prevHeights)[txinIndex] = 0; + continue; + } + + int nCoinHeight = (*prevHeights)[txinIndex]; + + if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { + int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); + // NOTE: Subtract 1 to maintain nLockTime semantics + // BIP 68 relative lock times have the semantics of calculating + // the first block or time at which the transaction would be + // valid. When calculating the effective block time or height + // for the entire transaction, we switch to using the + // semantics of nLockTime which is the last invalid block + // time or height. Thus we subtract 1 from the calculated + // time or height. + + // Time-based relative lock-times are measured from the + // smallest allowed timestamp of the block containing the + // txout being spent, which is the median time past of the + // block prior. + nMinTime = std::max(nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); + } else { + nMinHeight = std::max(nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); + } + } + + return std::make_pair(nMinHeight, nMinTime); +} + +bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair) +{ + assert(block.pprev); + int64_t nBlockTime = block.pprev->GetMedianTimePast(); + if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) + return false; + + return true; +} + +bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) +{ + return EvaluateSequenceLocks(block, CalculateSequenceLocks(tx, flags, prevHeights, block)); +} + +unsigned int GetLegacySigOpCount(const CTransaction& tx) +{ + unsigned int nSigOps = 0; + for (const auto& txin : tx.vin) + { + nSigOps += txin.scriptSig.GetSigOpCount(false); + } + for (const auto& txout : tx.vout) + { + nSigOps += txout.scriptPubKey.GetSigOpCount(false); + } + return nSigOps; +} + +unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& inputs) +{ + if (tx.IsCoinBase()) + return 0; + + unsigned int nSigOps = 0; + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; + if (prevout.scriptPubKey.IsPayToScriptHash()) + nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); + } + return nSigOps; +} + +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) +{ + int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; + + if (tx.IsCoinBase()) + return nSigOps; + + if (flags & SCRIPT_VERIFY_P2SH) { + nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; + } + + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; + nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); + } + return nSigOps; +} + +bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) +{ + // Basic checks that don't depend on any context + if (tx.vin.empty()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); + if (tx.vout.empty()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); + // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) + if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); + + // Check for negative or overflow output values + CAmount nValueOut = 0; + for (const auto& txout : tx.vout) + { + if (txout.nValue < 0) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); + if (txout.nValue > MAX_MONEY) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); + nValueOut += txout.nValue; + if (!MoneyRange(nValueOut)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); + } + + // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock + if (fCheckDuplicateInputs) { + std::set<COutPoint> vInOutPoints; + for (const auto& txin : tx.vin) + { + if (!vInOutPoints.insert(txin.prevout).second) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); + } + } + + if (tx.IsCoinBase()) + { + if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) + return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); + } + else + { + for (const auto& txin : tx.vin) + if (txin.prevout.IsNull()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); + } + + return true; +} + +bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) +{ + // This doesn't trigger the DoS code on purpose; if it did, it would make it easier + // for an attacker to attempt to split the network. + if (!inputs.HaveInputs(tx)) + return state.Invalid(false, 0, "", "Inputs unavailable"); + + CAmount nValueIn = 0; + CAmount nFees = 0; + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const COutPoint &prevout = tx.vin[i].prevout; + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // If prev is coinbase, check that it's matured + if (coin.IsCoinBase()) { + if (nSpendHeight - coin.nHeight < COINBASE_MATURITY) + return state.Invalid(false, + REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", + strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); + } + + // Check for negative or overflow input values + nValueIn += coin.out.nValue; + if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); + + } + + if (nValueIn < tx.GetValueOut()) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, + strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); + + // Tally transaction fees + CAmount nTxFee = nValueIn - tx.GetValueOut(); + if (nTxFee < 0) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); + nFees += nTxFee; + if (!MoneyRange(nFees)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); + return true; +} diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h new file mode 100644 index 0000000000..d46d3294ca --- /dev/null +++ b/src/consensus/tx_verify.h @@ -0,0 +1,78 @@ +// Copyright (c) 2017-2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CONSENSUS_TX_VERIFY_H +#define BITCOIN_CONSENSUS_TX_VERIFY_H + +#include <stdint.h> +#include <vector> + +class CBlockIndex; +class CCoinsViewCache; +class CTransaction; +class CValidationState; + +/** Transaction validation functions */ + +/** Context-independent validity checks */ +bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); + +namespace Consensus { +/** + * Check whether all inputs of this transaction are valid (no double spends and amounts) + * This does not modify the UTXO set. This does not check scripts and sigs. + * Preconditions: tx.IsCoinBase() is false. + */ +bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight); +} // namespace Consensus + +/** Auxiliary functions for transaction validation (ideally should not be exposed) */ + +/** + * Count ECDSA signature operations the old-fashioned (pre-0.6) way + * @return number of sigops this transaction's outputs will produce when spent + * @see CTransaction::FetchInputs + */ +unsigned int GetLegacySigOpCount(const CTransaction& tx); + +/** + * Count ECDSA signature operations in pay-to-script-hash inputs. + * + * @param[in] mapInputs Map of previous transactions that have outputs we're spending + * @return maximum number of sigops required to validate this transaction's inputs + * @see CTransaction::FetchInputs + */ +unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& mapInputs); + +/** + * Compute total signature operation cost of a transaction. + * @param[in] tx Transaction for which we are computing the cost + * @param[in] inputs Map of previous transactions that have outputs we're spending + * @param[out] flags Script verification flags + * @return Total signature operation cost of tx + */ +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags); + +/** + * Check if transaction is final and can be included in a block with the + * specified height and time. Consensus critical. + */ +bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime); + +/** + * Calculates the block height and previous block's median time past at + * which the transaction will be considered final in the context of BIP 68. + * Also removes from the vector of input heights any entries which did not + * correspond to sequence locked inputs as they do not affect the calculation. + */ +std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); + +bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair); +/** + * Check if transaction is final per BIP 68 sequence numbers and can be included in a block. + * Consensus critical. Takes as input a list of heights at which tx's inputs (in order) confirmed. + */ +bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); + +#endif // BITCOIN_CONSENSUS_TX_VERIFY_H diff --git a/src/core_memusage.h b/src/core_memusage.h index 5e10182075..e4ccd54c42 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -63,4 +63,9 @@ static inline size_t RecursiveDynamicUsage(const CBlockLocator& locator) { return memusage::DynamicUsage(locator.vHave); } +template<typename X> +static inline size_t RecursiveDynamicUsage(const std::shared_ptr<X>& p) { + return p ? memusage::DynamicUsage(p) + RecursiveDynamicUsage(*p) : 0; +} + #endif // BITCOIN_CORE_MEMUSAGE_H diff --git a/src/core_write.cpp b/src/core_write.cpp index b0993a131f..d116e617ee 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -114,9 +114,9 @@ std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDeco return str; } -std::string EncodeHexTx(const CTransaction& tx, const int serialFlags) +std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serialFlags); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serializeFlags); ssTx << tx; return HexStr(ssTx.begin(), ssTx.end()); } @@ -151,6 +151,8 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry) entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); entry.pushKV("version", tx.nVersion); + entry.pushKV("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); + entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("locktime", (int64_t)tx.nLockTime); UniValue vin(UniValue::VARR); diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp new file mode 100644 index 0000000000..816ae870e1 --- /dev/null +++ b/src/crypto/chacha20.cpp @@ -0,0 +1,180 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on the public domain implementation 'merged' by D. J. Bernstein +// See https://cr.yp.to/chacha.html. + +#include "crypto/common.h" +#include "crypto/chacha20.h" + +#include <string.h> + +constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | (v >> (32 - c)); } + +#define QUARTERROUND(a,b,c,d) \ + a += b; d = rotl32(d ^ a, 16); \ + c += d; b = rotl32(b ^ c, 12); \ + a += b; d = rotl32(d ^ a, 8); \ + c += d; b = rotl32(b ^ c, 7); + +static const unsigned char sigma[] = "expand 32-byte k"; +static const unsigned char tau[] = "expand 16-byte k"; + +void ChaCha20::SetKey(const unsigned char* k, size_t keylen) +{ + const unsigned char *constants; + + input[4] = ReadLE32(k + 0); + input[5] = ReadLE32(k + 4); + input[6] = ReadLE32(k + 8); + input[7] = ReadLE32(k + 12); + if (keylen == 32) { /* recommended */ + k += 16; + constants = sigma; + } else { /* keylen == 16 */ + constants = tau; + } + input[8] = ReadLE32(k + 0); + input[9] = ReadLE32(k + 4); + input[10] = ReadLE32(k + 8); + input[11] = ReadLE32(k + 12); + input[0] = ReadLE32(constants + 0); + input[1] = ReadLE32(constants + 4); + input[2] = ReadLE32(constants + 8); + input[3] = ReadLE32(constants + 12); + input[12] = 0; + input[13] = 0; + input[14] = 0; + input[15] = 0; +} + +ChaCha20::ChaCha20() +{ + memset(input, 0, sizeof(input)); +} + +ChaCha20::ChaCha20(const unsigned char* k, size_t keylen) +{ + SetKey(k, keylen); +} + +void ChaCha20::SetIV(uint64_t iv) +{ + input[14] = iv; + input[15] = iv >> 32; +} + +void ChaCha20::Seek(uint64_t pos) +{ + input[12] = pos; + input[13] = pos >> 32; +} + +void ChaCha20::Output(unsigned char* c, size_t bytes) +{ + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + unsigned char *ctarget = NULL; + unsigned char tmp[64]; + unsigned int i; + + if (!bytes) return; + + j0 = input[0]; + j1 = input[1]; + j2 = input[2]; + j3 = input[3]; + j4 = input[4]; + j5 = input[5]; + j6 = input[6]; + j7 = input[7]; + j8 = input[8]; + j9 = input[9]; + j10 = input[10]; + j11 = input[11]; + j12 = input[12]; + j13 = input[13]; + j14 = input[14]; + j15 = input[15]; + + for (;;) { + if (bytes < 64) { + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20;i > 0;i -= 2) { + QUARTERROUND( x0, x4, x8,x12) + QUARTERROUND( x1, x5, x9,x13) + QUARTERROUND( x2, x6,x10,x14) + QUARTERROUND( x3, x7,x11,x15) + QUARTERROUND( x0, x5,x10,x15) + QUARTERROUND( x1, x6,x11,x12) + QUARTERROUND( x2, x7, x8,x13) + QUARTERROUND( x3, x4, x9,x14) + } + x0 += j0; + x1 += j1; + x2 += j2; + x3 += j3; + x4 += j4; + x5 += j5; + x6 += j6; + x7 += j7; + x8 += j8; + x9 += j9; + x10 += j10; + x11 += j11; + x12 += j12; + x13 += j13; + x14 += j14; + x15 += j15; + + ++j12; + if (!j12) ++j13; + + WriteLE32(c + 0, x0); + WriteLE32(c + 4, x1); + WriteLE32(c + 8, x2); + WriteLE32(c + 12, x3); + WriteLE32(c + 16, x4); + WriteLE32(c + 20, x5); + WriteLE32(c + 24, x6); + WriteLE32(c + 28, x7); + WriteLE32(c + 32, x8); + WriteLE32(c + 36, x9); + WriteLE32(c + 40, x10); + WriteLE32(c + 44, x11); + WriteLE32(c + 48, x12); + WriteLE32(c + 52, x13); + WriteLE32(c + 56, x14); + WriteLE32(c + 60, x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) ctarget[i] = c[i]; + } + input[12] = j12; + input[13] = j13; + return; + } + bytes -= 64; + c += 64; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h new file mode 100644 index 0000000000..a305977bcd --- /dev/null +++ b/src/crypto/chacha20.h @@ -0,0 +1,26 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_CHACHA20_H +#define BITCOIN_CRYPTO_CHACHA20_H + +#include <stdint.h> +#include <stdlib.h> + +/** A PRNG class for ChaCha20. */ +class ChaCha20 +{ +private: + uint32_t input[16]; + +public: + ChaCha20(); + ChaCha20(const unsigned char* key, size_t keylen); + void SetKey(const unsigned char* key, size_t keylen); + void SetIV(uint64_t iv); + void Seek(uint64_t pos); + void Output(unsigned char* output, size_t bytes); +}; + +#endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/crypto/common.h b/src/crypto/common.h index 4a9d1150b6..bcca3d30ea 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -79,4 +79,25 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x) memcpy(ptr, (char*)&v, 8); } +/** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ +uint64_t static inline CountBits(uint64_t x) +{ +#ifdef HAVE_DECL___BUILTIN_CLZL + if (sizeof(unsigned long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; + } +#endif +#ifdef HAVE_DECL___BUILTIN_CLZLL + if (sizeof(unsigned long long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; + } +#endif + int ret = 0; + while (x) { + x >>= 1; + ++ret; + } + return ret; +} + #endif // BITCOIN_CRYPTO_COMMON_H diff --git a/src/cuckoocache.h b/src/cuckoocache.h index ff47e9776b..5837549455 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -154,7 +154,7 @@ public: * @tparam Element should be a movable and copyable type * @tparam Hash should be a function/callable which takes a template parameter * hash_select and an Element and extracts a hash from it. Should return - * high-entropy hashes for `Hash h; h<0>(e) ... h<7>(e)`. + * high-entropy uint32_t hashes for `Hash h; h<0>(e) ... h<7>(e)`. */ template <typename Element, typename Hash> class cache @@ -193,12 +193,6 @@ private: */ uint32_t epoch_size; - /** hash_mask should be set to appropriately mask out a hash such that every - * masked hash is [0,size), eg, if floor(log2(size)) == 20, then hash_mask - * should be (1<<20)-1 - */ - uint32_t hash_mask; - /** depth_limit determines how many elements insert should try to replace. * Should be set to log2(n)*/ uint8_t depth_limit; @@ -217,14 +211,14 @@ private: */ inline std::array<uint32_t, 8> compute_hashes(const Element& e) const { - return {{hash_function.template operator()<0>(e) & hash_mask, - hash_function.template operator()<1>(e) & hash_mask, - hash_function.template operator()<2>(e) & hash_mask, - hash_function.template operator()<3>(e) & hash_mask, - hash_function.template operator()<4>(e) & hash_mask, - hash_function.template operator()<5>(e) & hash_mask, - hash_function.template operator()<6>(e) & hash_mask, - hash_function.template operator()<7>(e) & hash_mask}}; + return {{(uint32_t)((hash_function.template operator()<0>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<1>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<2>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<3>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<4>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<5>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<6>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<7>(e) * (uint64_t)size) >> 32)}}; } /* end @@ -305,7 +299,7 @@ public: } /** setup initializes the container to store no more than new_size - * elements. setup rounds down to a power of two size. + * elements. * * setup should only be called once. * @@ -316,8 +310,7 @@ public: { // depth_limit must be at least one otherwise errors can occur. depth_limit = static_cast<uint8_t>(std::log2(static_cast<float>(std::max((uint32_t)2, new_size)))); - size = 1 << depth_limit; - hash_mask = size-1; + size = std::max<uint32_t>(2, new_size); table.resize(size); collection_flags.setup(size); epoch_flags.resize(size); diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 94ebedeb72..3d2098c059 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -4,16 +4,74 @@ #include "dbwrapper.h" +#include "fs.h" #include "util.h" #include "random.h" -#include <boost/filesystem.hpp> - #include <leveldb/cache.h> #include <leveldb/env.h> #include <leveldb/filter_policy.h> #include <memenv.h> #include <stdint.h> +#include <algorithm> + +class CBitcoinLevelDBLogger : public leveldb::Logger { +public: + // This code is adapted from posix_logger.h, which is why it is using vsprintf. + // Please do not do this in normal code + virtual void Logv(const char * format, va_list ap) override { + if (!LogAcceptCategory(BCLog::LEVELDB)) { + return; + } + char buffer[500]; + for (int iter = 0; iter < 2; iter++) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } + else { + bufsize = 30000; + base = new char[bufsize]; + } + char* p = base; + char* limit = base + bufsize; + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + // Do not use vsnprintf elsewhere in bitcoin source code, see above. + p += vsnprintf(p, limit - p, format, backup_ap); + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) { + continue; // Try again with larger buffer + } + else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + base[std::min(bufsize - 1, (int)(p - base))] = '\0'; + LogPrintStr(base); + if (base != buffer) { + delete[] base; + } + break; + } + } +}; static leveldb::Options GetOptions(size_t nCacheSize) { @@ -23,6 +81,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) options.filter_policy = leveldb::NewBloomFilterPolicy(10); options.compression = leveldb::kNoCompression; options.max_open_files = 64; + options.info_log = new CBitcoinLevelDBLogger(); if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) { // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error // on corruption in later versions. @@ -31,7 +90,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) +CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) { penv = NULL; readoptions.verify_checksums = true; @@ -82,6 +141,8 @@ CDBWrapper::~CDBWrapper() pdb = NULL; delete options.filter_policy; options.filter_policy = NULL; + delete options.info_log; + options.info_log = NULL; delete options.block_cache; options.block_cache = NULL; delete penv; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index dd59cc00ff..24ef71bfbf 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -6,14 +6,13 @@ #define BITCOIN_DBWRAPPER_H #include "clientversion.h" +#include "fs.h" #include "serialize.h" #include "streams.h" #include "util.h" #include "utilstrencodings.h" #include "version.h" -#include <boost/filesystem/path.hpp> - #include <leveldb/db.h> #include <leveldb/write_batch.h> @@ -56,11 +55,19 @@ private: CDataStream ssKey; CDataStream ssValue; + size_t size_estimate; + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { }; + + void Clear() + { + batch.Clear(); + size_estimate = 0; + } template <typename K, typename V> void Write(const K& key, const V& value) @@ -75,6 +82,14 @@ public: leveldb::Slice slValue(ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); + // LevelDB serializes writes as: + // - byte: header + // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) + // - byte[]: key + // - varint: value length + // - byte[]: value + // The formula below assumes the key and value are both less than 16k. + size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); ssKey.clear(); ssValue.clear(); } @@ -87,8 +102,16 @@ public: leveldb::Slice slKey(ssKey.data(), ssKey.size()); batch.Delete(slKey); + // LevelDB serializes erases as: + // - byte: header + // - varint: key length + // - byte[]: key + // The formula below assumes the key is less than 16kB. + size_estimate += 2 + (slKey.size() > 127) + slKey.size(); ssKey.clear(); } + + size_t SizeEstimate() const { return size_estimate; } }; class CDBIterator @@ -132,10 +155,6 @@ public: return true; } - unsigned int GetKeySize() { - return piter->key().size(); - } - template<typename V> bool GetValue(V& value) { leveldb::Slice slValue = piter->value(); try { @@ -199,7 +218,7 @@ public: * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR * with a zero'd byte array. */ - CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); + CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); ~CDBWrapper(); template <typename K, typename V> @@ -286,7 +305,22 @@ public: * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); + + template<typename K> + size_t EstimateSize(const K& key_begin, const K& key_end) const + { + CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); + ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey1 << key_begin; + ssKey2 << key_end; + leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + uint64_t size = 0; + leveldb::Range range(slKey1, slKey2); + pdb->GetApproximateSizes(&range, 1, &size); + return size; + } }; #endif // BITCOIN_DBWRAPPER_H - diff --git a/src/fs.cpp b/src/fs.cpp new file mode 100644 index 0000000000..6f2b768de3 --- /dev/null +++ b/src/fs.cpp @@ -0,0 +1,17 @@ +#include "fs.h" + +#include <boost/filesystem.hpp> + +namespace fsbridge { + +FILE *fopen(const fs::path& p, const char *mode) +{ + return ::fopen(p.string().c_str(), mode); +} + +FILE *freopen(const fs::path& p, const char *mode, FILE *stream) +{ + return ::freopen(p.string().c_str(), mode, stream); +} + +} // fsbridge diff --git a/src/fs.h b/src/fs.h new file mode 100644 index 0000000000..abb4be254b --- /dev/null +++ b/src/fs.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FS_H +#define BITCOIN_FS_H + +#include <stdio.h> +#include <string> + +#include <boost/filesystem.hpp> +#include <boost/filesystem/fstream.hpp> +#include <boost/filesystem/detail/utf8_codecvt_facet.hpp> + +/** Filesystem operations and types */ +namespace fs = boost::filesystem; + +/** Bridge operations to C stdio */ +namespace fsbridge { + FILE *fopen(const fs::path& p, const char *mode); + FILE *freopen(const fs::path& p, const char *mode, FILE *stream); +}; + +#endif // BITCOIN_FS_H diff --git a/src/hash.cpp b/src/hash.cpp index a14a2386a2..b361c90d16 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -208,3 +208,44 @@ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val) SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } + +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra) +{ + /* Specialized implementation for efficiency */ + uint64_t d = val.GetUint64(0); + + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; + + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(1); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(2); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(3); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = (((uint64_t)36) << 56) | extra; + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} diff --git a/src/hash.h b/src/hash.h index eacb8f04fe..b9952d39fc 100644 --- a/src/hash.h +++ b/src/hash.h @@ -160,6 +160,41 @@ public: } }; +/** Reads data from an underlying stream, while hashing the read data. */ +template<typename Source> +class CHashVerifier : public CHashWriter +{ +private: + Source* source; + +public: + CHashVerifier(Source* source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {} + + void read(char* pch, size_t nSize) + { + source->read(pch, nSize); + this->write(pch, nSize); + } + + void ignore(size_t nSize) + { + char data[1024]; + while (nSize > 0) { + size_t now = std::min<size_t>(nSize, 1024); + read(data, now); + nSize -= now; + } + } + + template<typename T> + CHashVerifier<Source>& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template<typename T> uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) @@ -206,5 +241,6 @@ public: * .Finalize() */ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val); +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra); #endif // BITCOIN_HASH_H diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 8ac6925acd..5ab6d8d732 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -30,7 +30,7 @@ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; class HTTPRPCTimer : public RPCTimerBase { public: - HTTPRPCTimer(struct event_base* eventBase, boost::function<void(void)>& func, int64_t millis) : + HTTPRPCTimer(struct event_base* eventBase, std::function<void(void)>& func, int64_t millis) : ev(eventBase, false, func) { struct timeval tv; @@ -52,7 +52,7 @@ public: { return "HTTP"; } - RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) { return new HTTPRPCTimer(base, func, millis); } @@ -93,9 +93,9 @@ static bool multiUserAuthorized(std::string strUserPass) std::string strUser = strUserPass.substr(0, strUserPass.find(":")); std::string strPass = strUserPass.substr(strUserPass.find(":") + 1); - if (mapMultiArgs.count("-rpcauth") > 0) { + if (gArgs.IsArgSet("-rpcauth")) { //Search for multi-user login/pass "rpcauth" from config - BOOST_FOREACH(std::string strRPCAuth, mapMultiArgs.at("-rpcauth")) + BOOST_FOREACH(std::string strRPCAuth, gArgs.GetArgs("-rpcauth")) { std::vector<std::string> vFields; boost::split(vFields, strRPCAuth, boost::is_any_of(":$")); @@ -233,7 +233,7 @@ static bool InitRPCAuthentication() bool StartHTTPRPC() { - LogPrint("rpc", "Starting HTTP RPC server\n"); + LogPrint(BCLog::RPC, "Starting HTTP RPC server\n"); if (!InitRPCAuthentication()) return false; @@ -247,12 +247,12 @@ bool StartHTTPRPC() void InterruptHTTPRPC() { - LogPrint("rpc", "Interrupting HTTP RPC server\n"); + LogPrint(BCLog::RPC, "Interrupting HTTP RPC server\n"); } void StopHTTPRPC() { - LogPrint("rpc", "Stopping HTTP RPC server\n"); + LogPrint(BCLog::RPC, "Stopping HTTP RPC server\n"); UnregisterHTTPHandler("/", true); if (httpRPCTimerInterface) { RPCUnsetTimerInterface(httpRPCTimerInterface); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index e1763c6ad2..0d1cba3fd2 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -146,13 +146,6 @@ public: while (numThreads > 0) cond.wait(lock); } - - /** Return current depth of queue */ - size_t Depth() - { - std::unique_lock<std::mutex> lock(cs); - return queue.size(); - } }; struct HTTPPathHandler @@ -203,9 +196,8 @@ static bool InitHTTPAllowList() LookupHost("::1", localv6, false); rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv4 local subnet rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost - if (mapMultiArgs.count("-rpcallowip")) { - const std::vector<std::string>& vAllow = mapMultiArgs.at("-rpcallowip"); - for (std::string strAllow : vAllow) { + if (gArgs.IsArgSet("-rpcallowip")) { + for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow.c_str(), subnet); if (!subnet.IsValid()) { @@ -220,7 +212,7 @@ static bool InitHTTPAllowList() std::string strAllowed; for (const CSubNet& subnet : rpc_allow_subnets) strAllowed += subnet.ToString() + " "; - LogPrint("http", "Allowing HTTP connections from: %s\n", strAllowed); + LogPrint(BCLog::HTTP, "Allowing HTTP connections from: %s\n", strAllowed); return true; } @@ -250,7 +242,7 @@ static void http_request_cb(struct evhttp_request* req, void* arg) { std::unique_ptr<HTTPRequest> hreq(new HTTPRequest(req)); - LogPrint("http", "Received a %s request for %s from %s\n", + LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n", RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString()); // Early address-based allow check @@ -300,7 +292,7 @@ static void http_request_cb(struct evhttp_request* req, void* arg) /** Callback to reject HTTP requests after shutdown. */ static void http_reject_request_cb(struct evhttp_request* req, void*) { - LogPrint("http", "Rejecting request while shutting down\n"); + LogPrint(BCLog::HTTP, "Rejecting request while shutting down\n"); evhttp_send_error(req, HTTP_SERVUNAVAIL, NULL); } @@ -308,10 +300,10 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) static bool ThreadHTTP(struct event_base* base, struct evhttp* http) { RenameThread("bitcoin-http"); - LogPrint("http", "Entering http event loop\n"); + LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() - LogPrint("http", "Exited http event loop\n"); + LogPrint(BCLog::HTTP, "Exited http event loop\n"); return event_base_got_break(base) == 0; } @@ -328,12 +320,11 @@ static bool HTTPBindAddresses(struct evhttp* http) if (IsArgSet("-rpcbind")) { LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); } - } else if (mapMultiArgs.count("-rpcbind")) { // Specific bind address - const std::vector<std::string>& vbind = mapMultiArgs.at("-rpcbind"); - for (std::vector<std::string>::const_iterator i = vbind.begin(); i != vbind.end(); ++i) { + } else if (gArgs.IsArgSet("-rpcbind")) { // Specific bind address + for (const std::string& strRPCBind : gArgs.GetArgs("-rpcbind")) { int port = defaultPort; std::string host; - SplitHostPort(*i, port, host); + SplitHostPort(strRPCBind, port, host); endpoints.push_back(std::make_pair(host, port)); } } else { // No specific bind address specified, bind to any @@ -343,7 +334,7 @@ static bool HTTPBindAddresses(struct evhttp* http) // Bind addresses for (std::vector<std::pair<std::string, uint16_t> >::iterator i = endpoints.begin(); i != endpoints.end(); ++i) { - LogPrint("http", "Binding RPC on address %s port %i\n", i->first, i->second); + LogPrint(BCLog::HTTP, "Binding RPC on address %s port %i\n", i->first, i->second); evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle(http, i->first.empty() ? NULL : i->first.c_str(), i->second); if (bind_handle) { boundSockets.push_back(bind_handle); @@ -371,7 +362,7 @@ static void libevent_log_cb(int severity, const char *msg) if (severity >= EVENT_LOG_WARN) // Log warn messages and higher without debug category LogPrintf("libevent: %s\n", msg); else - LogPrint("libevent", "libevent: %s\n", msg); + LogPrint(BCLog::LIBEVENT, "libevent: %s\n", msg); } bool InitHTTPServer() @@ -391,14 +382,13 @@ bool InitHTTPServer() // Redirect libevent's logging to our own log event_set_log_callback(&libevent_log_cb); -#if LIBEVENT_VERSION_NUMBER >= 0x02010100 - // If -debug=libevent, set full libevent debugging. - // Otherwise, disable all libevent debugging. - if (LogAcceptCategory("libevent")) - event_enable_debug_logging(EVENT_DBG_ALL); - else - event_enable_debug_logging(EVENT_DBG_NONE); -#endif + // Update libevent's log handling. Returns false if our version of + // libevent doesn't support debug logging, in which case we should + // clear the BCLog::LIBEVENT flag. + if (!UpdateHTTPServerLogging(logCategories & BCLog::LIBEVENT)) { + logCategories &= ~BCLog::LIBEVENT; + } + #ifdef WIN32 evthread_use_windows_threads(); #else @@ -431,7 +421,7 @@ bool InitHTTPServer() return false; } - LogPrint("http", "Initialized HTTP server\n"); + LogPrint(BCLog::HTTP, "Initialized HTTP server\n"); int workQueueDepth = std::max((long)GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); LogPrintf("HTTP: creating work queue of depth %d\n", workQueueDepth); @@ -441,12 +431,26 @@ bool InitHTTPServer() return true; } +bool UpdateHTTPServerLogging(bool enable) { +#if LIBEVENT_VERSION_NUMBER >= 0x02010100 + if (enable) { + event_enable_debug_logging(EVENT_DBG_ALL); + } else { + event_enable_debug_logging(EVENT_DBG_NONE); + } + return true; +#else + // Can't update libevent logging if version < 02010100 + return false; +#endif +} + std::thread threadHTTP; std::future<bool> threadResult; bool StartHTTPServer() { - LogPrint("http", "Starting HTTP server\n"); + LogPrint(BCLog::HTTP, "Starting HTTP server\n"); int rpcThreads = std::max((long)GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); std::packaged_task<bool(event_base*, evhttp*)> task(ThreadHTTP); @@ -462,7 +466,7 @@ bool StartHTTPServer() void InterruptHTTPServer() { - LogPrint("http", "Interrupting HTTP server\n"); + LogPrint(BCLog::HTTP, "Interrupting HTTP server\n"); if (eventHTTP) { // Unlisten sockets for (evhttp_bound_socket *socket : boundSockets) { @@ -477,14 +481,15 @@ void InterruptHTTPServer() void StopHTTPServer() { - LogPrint("http", "Stopping HTTP server\n"); + LogPrint(BCLog::HTTP, "Stopping HTTP server\n"); if (workQueue) { - LogPrint("http", "Waiting for HTTP worker threads to exit\n"); + LogPrint(BCLog::HTTP, "Waiting for HTTP worker threads to exit\n"); workQueue->WaitExit(); delete workQueue; + workQueue = nullptr; } if (eventBase) { - LogPrint("http", "Waiting for HTTP event thread to exit\n"); + LogPrint(BCLog::HTTP, "Waiting for HTTP event thread to exit\n"); // Give event loop a few seconds to exit (to send back last RPC responses), then break it // Before this was solved with event_base_loopexit, but that didn't work as expected in // at least libevent 2.0.21 and always introduced a delay. In libevent @@ -505,7 +510,7 @@ void StopHTTPServer() event_base_free(eventBase); eventBase = 0; } - LogPrint("http", "Stopped HTTP server\n"); + LogPrint(BCLog::HTTP, "Stopped HTTP server\n"); } struct event_base* EventBase() @@ -652,7 +657,7 @@ HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) { - LogPrint("http", "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); + LogPrint(BCLog::HTTP, "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.push_back(HTTPPathHandler(prefix, exactMatch, handler)); } @@ -665,7 +670,7 @@ void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch) break; if (i != iend) { - LogPrint("http", "Unregistering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); + LogPrint(BCLog::HTTP, "Unregistering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.erase(i); } } diff --git a/src/httpserver.h b/src/httpserver.h index b55b253bea..6be9950682 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -32,6 +32,10 @@ void InterruptHTTPServer(); /** Stop HTTP server */ void StopHTTPServer(); +/** Change logging level for libevent. Removes BCLog::LIBEVENT from logCategories if + * libevent doesn't support debug logging.*/ +bool UpdateHTTPServerLogging(bool enable); + /** Handler for requests to a certain HTTP path */ typedef std::function<bool(HTTPRequest* req, const std::string &)> HTTPRequestHandler; /** Register handler for prefix. diff --git a/src/init.cpp b/src/init.cpp index 1c8c3bc620..04d1280c92 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -16,6 +16,7 @@ #include "checkpoints.h" #include "compat/sanity.h" #include "consensus/validation.h" +#include "fs.h" #include "httpserver.h" #include "httprpc.h" #include "key.h" @@ -24,9 +25,12 @@ #include "netbase.h" #include "net.h" #include "net_processing.h" +#include "policy/feerate.h" +#include "policy/fees.h" #include "policy/policy.h" #include "rpc/server.h" #include "rpc/register.h" +#include "rpc/blockchain.h" #include "script/standard.h" #include "script/sigcache.h" #include "scheduler.h" @@ -55,8 +59,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/bind.hpp> -#include <boost/filesystem.hpp> -#include <boost/function.hpp> #include <boost/interprocess/sync/file_lock.hpp> #include <boost/thread.hpp> #include <openssl/crypto.h> @@ -144,9 +146,9 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: CCoinsViewErrorCatcher(CCoinsView* view) : CCoinsViewBacked(view) {} - bool GetCoins(const uint256 &txid, CCoins &coins) const { + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override { try { - return CCoinsViewBacked::GetCoins(txid, coins); + return CCoinsViewBacked::GetCoin(outpoint, coin); } catch(const std::runtime_error& e) { uiInterface.ThreadSafeMessageBox(_("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); @@ -206,15 +208,17 @@ void Shutdown() StopTorControl(); UnregisterNodeSignals(GetNodeSignals()); - if (fDumpMempoolLater) + if (fDumpMempoolLater && GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); + } if (fFeeEstimatesInitialized) { - boost::filesystem::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; - CAutoFile est_fileout(fopen(est_path.string().c_str(), "wb"), SER_DISK, CLIENT_VERSION); + ::feeEstimator.FlushUnconfirmed(::mempool); + fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; + CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); if (!est_fileout.IsNull()) - mempool.WriteFeeEstimates(est_fileout); + ::feeEstimator.Write(est_fileout); else LogPrintf("%s: Failed to write fee estimates to %s\n", __func__, est_path.string()); fFeeEstimatesInitialized = false; @@ -249,8 +253,8 @@ void Shutdown() #ifndef WIN32 try { - boost::filesystem::remove(GetPidFile()); - } catch (const boost::filesystem::filesystem_error& e) { + fs::remove(GetPidFile()); + } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove pidfile: %s\n", __func__, e.what()); } #endif @@ -265,18 +269,31 @@ void Shutdown() } /** - * Signal handlers are very limited in what they are allowed to do, so: + * Signal handlers are very limited in what they are allowed to do. + * The execution context the handler is invoked in is not guaranteed, + * so we restrict handler operations to just touching variables: */ -void HandleSIGTERM(int) +static void HandleSIGTERM(int) { fRequestShutdown = true; } -void HandleSIGHUP(int) +static void HandleSIGHUP(int) { fReopenDebugLog = true; } +#ifndef WIN32 +static void registerSignalHandler(int signal, void(*handler)(int)) +{ + struct sigaction sa; + sa.sa_handler = handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(signal, &sa, NULL); +} +#endif + bool static Bind(CConnman& connman, const CService &addr, unsigned int flags) { if (!(flags & BF_EXPLICIT) && IsLimited(addr)) return false; @@ -298,7 +315,7 @@ void OnRPCStopped() uiInterface.NotifyBlockTip.disconnect(&RPCNotifyBlockChange); RPCNotifyBlockChange(false, nullptr); cvBlockChange.notify_all(); - LogPrint("rpc", "RPC stopped.\n"); + LogPrint(BCLog::RPC, "RPC stopped.\n"); } void OnRPCPreCommand(const CRPCCommand& cmd) @@ -312,6 +329,10 @@ void OnRPCPreCommand(const CRPCCommand& cmd) std::string HelpMessage(HelpMessageMode mode) { + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); + const auto defaultChainParams = CreateChainParams(CBaseChainParams::MAIN); + const auto testnetChainParams = CreateChainParams(CBaseChainParams::TESTNET); const bool showDebug = GetBoolArg("-help-debug", false); // When adding new options to the categories, please keep and ensure alphabetical ordering. @@ -323,7 +344,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-blocknotify=<cmd>", _("Execute command when the best block changes (%s in cmd is replaced by block hash)")); if (showDebug) strUsage += HelpMessageOpt("-blocksonly", strprintf(_("Whether to operate in a blocks only mode (default: %u)"), DEFAULT_BLOCKSONLY)); - strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), Params(CBaseChainParams::MAIN).GetConsensus().defaultAssumeValid.GetHex(), Params(CBaseChainParams::TESTNET).GetConsensus().defaultAssumeValid.GetHex())); + strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex())); strUsage += HelpMessageOpt("-conf=<file>", strprintf(_("Specify configuration file (default: %s)"), BITCOIN_CONF_FILENAME)); if (mode == HMM_BITCOIND) { @@ -339,6 +360,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); + strUsage += HelpMessageOpt("-persistmempool", strprintf(_("Whether to save the mempool on shutdown and load on restart (default: %u)"), DEFAULT_PERSIST_MEMPOOL)); strUsage += HelpMessageOpt("-blockreconstructionextratxn=<n>", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); @@ -360,13 +382,13 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-banscore=<n>", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt("-bantime=<n>", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=<addr>", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -noconnect or -connect=0 alone to disable automatic connections")); + strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -connect=0 disables automatic connections")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); - strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect/-noconnect)")); + strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect used)")); strUsage += HelpMessageOpt("-externalip=<ip>", _("Specify your own public address")); strUsage += HelpMessageOpt("-forcednsseed", strprintf(_("Always query for peer addresses via DNS lookup (default: %u)"), DEFAULT_FORCEDNSSEED)); - strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect/-noconnect)")); + strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect)")); strUsage += HelpMessageOpt("-listenonion", strprintf(_("Automatically create Tor hidden service (default: %d)"), DEFAULT_LISTEN_ONION)); strUsage += HelpMessageOpt("-maxconnections=<n>", strprintf(_("Maintain at most <n> connections to peers (default: %u)"), DEFAULT_MAX_PEER_CONNECTIONS)); strUsage += HelpMessageOpt("-maxreceivebuffer=<n>", strprintf(_("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)"), DEFAULT_MAXRECEIVEBUFFER)); @@ -376,10 +398,9 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-onlynet=<net>", _("Only connect to nodes in network <net> (ipv4, ipv6 or onion)")); strUsage += HelpMessageOpt("-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %u)"), DEFAULT_PERMIT_BAREMULTISIG)); strUsage += HelpMessageOpt("-peerbloomfilters", strprintf(_("Support filtering of blocks and transaction with bloom filters (default: %u)"), DEFAULT_PEERBLOOMFILTERS)); - strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), Params(CBaseChainParams::MAIN).GetDefaultPort(), Params(CBaseChainParams::TESTNET).GetDefaultPort())); + strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort())); strUsage += HelpMessageOpt("-proxy=<ip:port>", _("Connect through SOCKS5 proxy")); strUsage += HelpMessageOpt("-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), DEFAULT_PROXYRANDOMIZE)); - strUsage += HelpMessageOpt("-rpcserialversion", strprintf(_("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(1) (default: %d)"), DEFAULT_RPC_SERIALIZE_VERSION)); strUsage += HelpMessageOpt("-seednode=<ip>", _("Connect to a node to retrieve peer addresses, and disconnect")); strUsage += HelpMessageOpt("-timeout=<n>", strprintf(_("Specify connection timeout in milliseconds (minimum: 1, default: %d)"), DEFAULT_CONNECT_TIMEOUT)); strUsage += HelpMessageOpt("-torcontrol=<ip>:<port>", strprintf(_("Tor control port to use if onion listening enabled (default: %s)"), DEFAULT_TOR_CONTROL)); @@ -394,8 +415,6 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-whitebind=<addr>", _("Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6")); strUsage += HelpMessageOpt("-whitelist=<IP address or network>", _("Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple times.") + " " + _("Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are already in the mempool, useful e.g. for a gateway")); - strUsage += HelpMessageOpt("-whitelistrelay", strprintf(_("Accept relayed transactions received from whitelisted peers even when not relaying transactions (default: %d)"), DEFAULT_WHITELISTRELAY)); - strUsage += HelpMessageOpt("-whitelistforcerelay", strprintf(_("Force relay of transactions from whitelisted peers even if they violate local relay policy (default: %d)"), DEFAULT_WHITELISTFORCERELAY)); strUsage += HelpMessageOpt("-maxuploadtarget=<n>", strprintf(_("Tries to keep outbound traffic under the given target (in MiB per 24h), 0 = no limit (default: %d)"), DEFAULT_MAX_UPLOAD_TARGET)); #ifdef ENABLE_WALLET @@ -416,27 +435,25 @@ std::string HelpMessage(HelpMessageMode mode) { strUsage += HelpMessageOpt("-checkblocks=<n>", strprintf(_("How many blocks to check at startup (default: %u, 0 = all)"), DEFAULT_CHECKBLOCKS)); strUsage += HelpMessageOpt("-checklevel=<n>", strprintf(_("How thorough the block verification of -checkblocks is (0-4, default: %u)"), DEFAULT_CHECKLEVEL)); - strUsage += HelpMessageOpt("-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, setBlockIndexCandidates, chainActive and mapBlocksUnlinked occasionally. Also sets -checkmempool (default: %u)", Params(CBaseChainParams::MAIN).DefaultConsistencyChecks())); - strUsage += HelpMessageOpt("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u)", Params(CBaseChainParams::MAIN).DefaultConsistencyChecks())); + strUsage += HelpMessageOpt("-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, setBlockIndexCandidates, chainActive and mapBlocksUnlinked occasionally. Also sets -checkmempool (default: %u)", defaultChainParams->DefaultConsistencyChecks())); + strUsage += HelpMessageOpt("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u)", defaultChainParams->DefaultConsistencyChecks())); strUsage += HelpMessageOpt("-checkpoints", strprintf("Disable expensive verification for known chain history (default: %u)", DEFAULT_CHECKPOINTS_ENABLED)); strUsage += HelpMessageOpt("-disablesafemode", strprintf("Disable safemode, override a real safe mode event (default: %u)", DEFAULT_DISABLE_SAFEMODE)); strUsage += HelpMessageOpt("-testsafemode", strprintf("Force safe mode (default: %u)", DEFAULT_TESTSAFEMODE)); strUsage += HelpMessageOpt("-dropmessagestest=<n>", "Randomly drop 1 of every <n> network messages"); strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages"); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT)); + strUsage += HelpMessageOpt("-stopatheight", strprintf("Stop running after reaching the given height in the main chain (default: %u)", DEFAULT_STOPATHEIGHT)); + strUsage += HelpMessageOpt("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT)); strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); - strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified BIP9 deployment (regtest-only)"); + strUsage += HelpMessageOpt("-vbparams=deployment:start:end", "Use given start/end times for specified version bits deployment (regtest-only)"); } - std::string debugCategories = "addrman, alert, bench, cmpctblock, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below - if (mode == HMM_BITCOIN_QT) - debugCategories += ", qt"; strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " + - _("If <category> is not supplied or if <category> = 1, output all debugging information.") + _("<category> can be:") + " " + debugCategories + "."); - if (showDebug) - strUsage += HelpMessageOpt("-nodebug", "Turn off debugging messages, same as -debug=0"); + _("If <category> is not supplied or if <category> = 1, output all debugging information.") + " " + _("<category> can be:") + " " + ListLogCategories() + "."); + strUsage += HelpMessageOpt("-debugexclude=<category>", strprintf(_("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."))); strUsage += HelpMessageOpt("-help-debug", _("Show all debugging options (usage: --help -help-debug)")); strUsage += HelpMessageOpt("-logips", strprintf(_("Include IP addresses in debug output (default: %u)"), DEFAULT_LOGIPS)); strUsage += HelpMessageOpt("-logtimestamps", strprintf(_("Prepend debug output with timestamp (default: %u)"), DEFAULT_LOGTIMESTAMPS)); @@ -444,19 +461,15 @@ std::string HelpMessage(HelpMessageMode mode) { strUsage += HelpMessageOpt("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS)); strUsage += HelpMessageOpt("-mocktime=<n>", "Replace actual time with <n> seconds since epoch (default: 0)"); - strUsage += HelpMessageOpt("-limitfreerelay=<n>", strprintf("Continuously rate-limit free transactions to <n>*1000 bytes per minute (default: %u)", DEFAULT_LIMITFREERELAY)); - strUsage += HelpMessageOpt("-relaypriority", strprintf("Require high priority for relaying free or low-fee transactions (default: %u)", DEFAULT_RELAYPRIORITY)); strUsage += HelpMessageOpt("-maxsigcachesize=<n>", strprintf("Limit size of signature cache to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE)); strUsage += HelpMessageOpt("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE)); } - strUsage += HelpMessageOpt("-minrelaytxfee=<amt>", strprintf(_("Fees (in %s/kB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)"), - CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE))); strUsage += HelpMessageOpt("-maxtxfee=<amt>", strprintf(_("Maximum total fees (in %s) to use in a single wallet transaction or raw transaction; setting this too low may abort large transactions (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE))); strUsage += HelpMessageOpt("-printtoconsole", _("Send trace/debug info to console instead of debug.log file")); if (showDebug) { - strUsage += HelpMessageOpt("-printpriority", strprintf("Log transaction priority and fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY)); + strUsage += HelpMessageOpt("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY)); } strUsage += HelpMessageOpt("-shrinkdebugfile", _("Shrink debug.log file on client startup (default: 1 when no -debug)")); @@ -464,7 +477,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("Node relay options:")); if (showDebug) { - strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", !Params(CBaseChainParams::TESTNET).RequireStandard())); + strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", defaultChainParams->RequireStandard())); strUsage += HelpMessageOpt("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to define cost of relay, used for mempool limiting and BIP 125 replacement. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE))); strUsage += HelpMessageOpt("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to defined dust, the value of an output such that it will cost about 1/3 of its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE))); } @@ -472,11 +485,14 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-datacarrier", strprintf(_("Relay and mine data carrier transactions (default: %u)"), DEFAULT_ACCEPT_DATACARRIER)); strUsage += HelpMessageOpt("-datacarriersize", strprintf(_("Maximum size of data in data carrier transactions we relay and mine (default: %u)"), MAX_OP_RETURN_RELAY)); strUsage += HelpMessageOpt("-mempoolreplacement", strprintf(_("Enable transaction replacement in the memory pool (default: %u)"), DEFAULT_ENABLE_REPLACEMENT)); + strUsage += HelpMessageOpt("-minrelaytxfee=<amt>", strprintf(_("Fees (in %s/kB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)"), + CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE))); + strUsage += HelpMessageOpt("-whitelistrelay", strprintf(_("Accept relayed transactions received from whitelisted peers even when not relaying transactions (default: %d)"), DEFAULT_WHITELISTRELAY)); + strUsage += HelpMessageOpt("-whitelistforcerelay", strprintf(_("Force relay of transactions from whitelisted peers even if they violate local relay policy (default: %d)"), DEFAULT_WHITELISTFORCERELAY)); strUsage += HelpMessageGroup(_("Block creation options:")); strUsage += HelpMessageOpt("-blockmaxweight=<n>", strprintf(_("Set maximum BIP141 block weight (default: %d)"), DEFAULT_BLOCK_MAX_WEIGHT)); strUsage += HelpMessageOpt("-blockmaxsize=<n>", strprintf(_("Set maximum block size in bytes (default: %d)"), DEFAULT_BLOCK_MAX_SIZE)); - strUsage += HelpMessageOpt("-blockprioritysize=<n>", strprintf(_("Set maximum size of high-priority/low-fee transactions in bytes (default: %d)"), DEFAULT_BLOCK_PRIORITY_SIZE)); strUsage += HelpMessageOpt("-blockmintxfee=<amt>", strprintf(_("Set lowest fee rate (in %s/kB) for transactions to be included in block creation. (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_BLOCK_MIN_TX_FEE))); if (showDebug) strUsage += HelpMessageOpt("-blockversion=<n>", "Override block version to test forking scenarios"); @@ -484,13 +500,14 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("RPC server options:")); strUsage += HelpMessageOpt("-server", _("Accept command line and JSON-RPC commands")); strUsage += HelpMessageOpt("-rest", strprintf(_("Accept public REST requests (default: %u)"), DEFAULT_REST_ENABLE)); - strUsage += HelpMessageOpt("-rpcbind=<addr>", _("Bind to given address to listen for JSON-RPC connections. Use [host]:port notation for IPv6. This option can be specified multiple times (default: bind to all interfaces)")); + strUsage += HelpMessageOpt("-rpcbind=<addr>[:port]", _("Bind to given address to listen for JSON-RPC connections. This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost, or if -rpcallowip has been specified, 0.0.0.0 and :: i.e., all addresses)")); strUsage += HelpMessageOpt("-rpccookiefile=<loc>", _("Location of the auth cookie (default: data dir)")); strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcauth=<userpw>", _("Username and hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcuser. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times")); - strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); + strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt("-rpcallowip=<ip>", _("Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times")); + strUsage += HelpMessageOpt("-rpcserialversion", strprintf(_("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(1) (default: %d)"), DEFAULT_RPC_SERIALIZE_VERSION)); strUsage += HelpMessageOpt("-rpcthreads=<n>", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); if (showDebug) { strUsage += HelpMessageOpt("-rpcworkqueue=<n>", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE)); @@ -570,14 +587,14 @@ struct CImportingNow // works correctly. void CleanupBlockRevFiles() { - std::map<std::string, boost::filesystem::path> mapBlockFiles; + std::map<std::string, fs::path> mapBlockFiles; // Glob all blk?????.dat and rev?????.dat files from the blocks directory. // Remove the rev files immediately and insert the blk file paths into an // ordered map keyed by block file index. LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - boost::filesystem::path blocksdir = GetDataDir() / "blocks"; - for (boost::filesystem::directory_iterator it(blocksdir); it != boost::filesystem::directory_iterator(); it++) { + fs::path blocksdir = GetDataDir() / "blocks"; + for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { if (is_regular_file(*it) && it->path().filename().string().length() == 12 && it->path().filename().string().substr(8,4) == ".dat") @@ -594,7 +611,7 @@ void CleanupBlockRevFiles() // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) // start removing block files. int nContigCounter = 0; - BOOST_FOREACH(const PAIRTYPE(std::string, boost::filesystem::path)& item, mapBlockFiles) { + BOOST_FOREACH(const PAIRTYPE(std::string, fs::path)& item, mapBlockFiles) { if (atoi(item.first) == nContigCounter) { nContigCounter++; continue; @@ -603,7 +620,7 @@ void CleanupBlockRevFiles() } } -void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) +void ThreadImport(std::vector<fs::path> vImportFiles) { const CChainParams& chainparams = Params(); RenameThread("bitcoin-loadblk"); @@ -616,7 +633,7 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) int nFile = 0; while (true) { CDiskBlockPos pos(nFile, 0); - if (!boost::filesystem::exists(GetBlockPosFilename(pos, "blk"))) + if (!fs::exists(GetBlockPosFilename(pos, "blk"))) break; // No block files left to reindex FILE *file = OpenBlockFile(pos, true); if (!file) @@ -633,11 +650,11 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) } // hardcoded $DATADIR/bootstrap.dat - boost::filesystem::path pathBootstrap = GetDataDir() / "bootstrap.dat"; - if (boost::filesystem::exists(pathBootstrap)) { - FILE *file = fopen(pathBootstrap.string().c_str(), "rb"); + fs::path pathBootstrap = GetDataDir() / "bootstrap.dat"; + if (fs::exists(pathBootstrap)) { + FILE *file = fsbridge::fopen(pathBootstrap, "rb"); if (file) { - boost::filesystem::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; + fs::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; LogPrintf("Importing bootstrap.dat...\n"); LoadExternalBlockFile(chainparams, file); RenameOver(pathBootstrap, pathBootstrapOld); @@ -647,8 +664,8 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) } // -loadblock= - BOOST_FOREACH(const boost::filesystem::path& path, vImportFiles) { - FILE *file = fopen(path.string().c_str(), "rb"); + BOOST_FOREACH(const fs::path& path, vImportFiles) { + FILE *file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); LoadExternalBlockFile(chainparams, file); @@ -669,8 +686,10 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) StartShutdown(); } } // End scope of CImportingNow - LoadMempool(); - fDumpMempoolLater = !fRequestShutdown; + if (GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + LoadMempool(); + fDumpMempoolLater = !fRequestShutdown; + } } /** Sanity checks @@ -683,9 +702,15 @@ bool InitSanityCheck(void) InitError("Elliptic curve cryptography sanity check failure. Aborting."); return false; } + if (!glibc_sanity_test() || !glibcxx_sanity_test()) return false; + if (!Random_SanityCheck()) { + InitError("OS cryptographic RNG sanity check failure. Aborting."); + return false; + } + return true; } @@ -721,7 +746,7 @@ void InitParameterInteraction() LogPrintf("%s: parameter interaction: -whitebind set -> setting -listen=1\n", __func__); } - if (mapMultiArgs.count("-connect") && mapMultiArgs.at("-connect").size() > 0) { + if (gArgs.IsArgSet("-connect")) { // when only connecting to trusted nodes, do not seed via DNS, or listen by default if (SoftSetBoolArg("-dnsseed", false)) LogPrintf("%s: parameter interaction: -connect set -> setting -dnsseed=0\n", __func__); @@ -845,19 +870,11 @@ bool AppInitBasicSetup() } // Clean shutdown on SIGTERM - struct sigaction sa; - sa.sa_handler = HandleSIGTERM; - sigemptyset(&sa.sa_mask); - sa.sa_flags = 0; - sigaction(SIGTERM, &sa, NULL); - sigaction(SIGINT, &sa, NULL); + registerSignalHandler(SIGTERM, HandleSIGTERM); + registerSignalHandler(SIGINT, HandleSIGTERM); // Reopen debug.log on SIGHUP - struct sigaction sa_hup; - sa_hup.sa_handler = HandleSIGHUP; - sigemptyset(&sa_hup.sa_mask); - sa_hup.sa_flags = 0; - sigaction(SIGHUP, &sa_hup, NULL); + registerSignalHandler(SIGHUP, HandleSIGHUP); // Ignore SIGPIPE, otherwise it will bring the daemon down if the client closes unexpectedly signal(SIGPIPE, SIG_IGN); @@ -883,8 +900,8 @@ bool AppInitParameterInteraction() // Make sure enough file descriptors are available int nBind = std::max( - (mapMultiArgs.count("-bind") ? mapMultiArgs.at("-bind").size() : 0) + - (mapMultiArgs.count("-whitebind") ? mapMultiArgs.at("-whitebind").size() : 0), size_t(1)); + (gArgs.IsArgSet("-bind") ? gArgs.GetArgs("-bind").size() : 0) + + (gArgs.IsArgSet("-whitebind") ? gArgs.GetArgs("-whitebind").size() : 0), size_t(1)); nUserMaxConnections = GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); @@ -899,13 +916,32 @@ bool AppInitParameterInteraction() InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags + if (gArgs.IsArgSet("-debug")) { + // Special-case: if -debug=0/-nodebug is set, turn off debugging messages + const std::vector<std::string> categories = gArgs.GetArgs("-debug"); + + if (find(categories.begin(), categories.end(), std::string("0")) == categories.end()) { + for (const auto& cat : categories) { + uint32_t flag = 0; + if (!GetLogCategory(&flag, &cat)) { + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); + continue; + } + logCategories |= flag; + } + } + } - fDebug = mapMultiArgs.count("-debug"); - // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - if (fDebug) { - const std::vector<std::string>& categories = mapMultiArgs.at("-debug"); - if (GetBoolArg("-nodebug", false) || find(categories.begin(), categories.end(), std::string("0")) != categories.end()) - fDebug = false; + // Now remove the logging categories which were explicitly excluded + if (gArgs.IsArgSet("-debugexclude")) { + for (const std::string& cat : gArgs.GetArgs("-debugexclude")) { + uint32_t flag = 0; + if (!GetLogCategory(&flag, &cat)) { + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); + continue; + } + logCategories &= ~flag; + } } // Check for -debugnet @@ -992,17 +1028,18 @@ bool AppInitParameterInteraction() if (nConnectTimeout <= 0) nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; - // Fee-per-kilobyte amount considered the same as "free" + // Fee-per-kilobyte amount required for mempool acceptance and relay // If you are mining, be careful setting this: // if you set it to zero then // a transaction spammer can cheaply fill blocks using - // 1-satoshi-fee transactions. It should be set above the real + // 0-fee transactions. It should be set above the real // cost to you of processing a transaction. if (IsArgSet("-minrelaytxfee")) { CAmount n = 0; - if (!ParseMoney(GetArg("-minrelaytxfee", ""), n) || 0 == n) + if (!ParseMoney(GetArg("-minrelaytxfee", ""), n)) { return InitError(AmountErrMsg("minrelaytxfee", GetArg("-minrelaytxfee", ""))); + } // High fee check is done afterward in CWallet::ParameterInteraction() ::minRelayTxFee = CFeeRate(n); } else if (incrementalRelayFee > ::minRelayTxFee) { @@ -1067,17 +1104,16 @@ bool AppInitParameterInteraction() fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end()); } - if (mapMultiArgs.count("-bip9params")) { - // Allow overriding BIP9 parameters for testing + if (gArgs.IsArgSet("-vbparams")) { + // Allow overriding version bits parameters for testing if (!chainparams.MineBlocksOnDemand()) { - return InitError("BIP9 parameters may only be overridden on regtest."); + return InitError("Version bits parameters may only be overridden on regtest."); } - const std::vector<std::string>& deployments = mapMultiArgs.at("-bip9params"); - for (auto i : deployments) { + for (const std::string& strDeployment : gArgs.GetArgs("-vbparams")) { std::vector<std::string> vDeploymentParams; - boost::split(vDeploymentParams, i, boost::is_any_of(":")); + boost::split(vDeploymentParams, strDeployment, boost::is_any_of(":")); if (vDeploymentParams.size() != 3) { - return InitError("BIP9 parameters malformed, expecting deployment:start:end"); + return InitError("Version bits parameters malformed, expecting deployment:start:end"); } int64_t nStartTime, nTimeout; if (!ParseInt64(vDeploymentParams[1], &nStartTime)) { @@ -1090,9 +1126,9 @@ bool AppInitParameterInteraction() for (int j=0; j<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[j].name) == 0) { - UpdateRegtestBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); + UpdateVersionBitsParameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); found = true; - LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); + LogPrintf("Setting version bits activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); break; } } @@ -1109,8 +1145,8 @@ static bool LockDataDirectory(bool probeOnly) std::string strDataDir = GetDataDir().string(); // Make sure only a single Bitcoin process is using the data directory. - boost::filesystem::path pathLockFile = GetDataDir() / ".lock"; - FILE* file = fopen(pathLockFile.string().c_str(), "a"); // empty lock file; created if it doesn't exist. + fs::path pathLockFile = GetDataDir() / ".lock"; + FILE* file = fsbridge::fopen(pathLockFile, "a"); // empty lock file; created if it doesn't exist. if (file) fclose(file); try { @@ -1158,7 +1194,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) #ifndef WIN32 CreatePidFile(GetPidFile(), getpid()); #endif - if (GetBoolArg("-shrinkdebugfile", !fDebug)) { + if (GetBoolArg("-shrinkdebugfile", logCategories == BCLog::NONE)) { // Do this first since it both loads a bunch of debug.log into memory, // and because this needs to happen before any other debug.log printing ShrinkDebugFile(); @@ -1221,8 +1257,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; - if (mapMultiArgs.count("-uacomment")) { - BOOST_FOREACH(std::string cmt, mapMultiArgs.at("-uacomment")) + if (gArgs.IsArgSet("-uacomment")) { + BOOST_FOREACH(std::string cmt, gArgs.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); @@ -1235,9 +1271,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } - if (mapMultiArgs.count("-onlynet")) { + if (gArgs.IsArgSet("-onlynet")) { std::set<enum Network> nets; - BOOST_FOREACH(const std::string& snet, mapMultiArgs.at("-onlynet")) { + BOOST_FOREACH(const std::string& snet, gArgs.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); @@ -1250,8 +1286,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (mapMultiArgs.count("-whitelist")) { - BOOST_FOREACH(const std::string& net, mapMultiArgs.at("-whitelist")) { + if (gArgs.IsArgSet("-whitelist")) { + BOOST_FOREACH(const std::string& net, gArgs.GetArgs("-whitelist")) { CSubNet subnet; LookupSubNet(net.c_str(), subnet); if (!subnet.IsValid()) @@ -1260,16 +1296,23 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } + // Check for host lookup allowed before parsing any network related parameters + fNameLookup = GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); + bool proxyRandomize = GetBoolArg("-proxyrandomize", DEFAULT_PROXYRANDOMIZE); // -proxy sets a proxy for all outgoing network traffic // -noproxy (or -proxy=0) as well as the empty string can be used to not set a proxy, this is the default std::string proxyArg = GetArg("-proxy", ""); SetLimited(NET_TOR); if (proxyArg != "" && proxyArg != "0") { - CService resolved(LookupNumeric(proxyArg.c_str(), 9050)); - proxyType addrProxy = proxyType(resolved, proxyRandomize); + CService proxyAddr; + if (!Lookup(proxyArg.c_str(), proxyAddr, 9050, fNameLookup)) { + return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); + } + + proxyType addrProxy = proxyType(proxyAddr, proxyRandomize); if (!addrProxy.IsValid()) - return InitError(strprintf(_("Invalid -proxy address: '%s'"), proxyArg)); + return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); SetProxy(NET_IPV4, addrProxy); SetProxy(NET_IPV6, addrProxy); @@ -1286,10 +1329,13 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) if (onionArg == "0") { // Handle -noonion/-onion=0 SetLimited(NET_TOR); // set onions as unreachable } else { - CService resolved(LookupNumeric(onionArg.c_str(), 9050)); - proxyType addrOnion = proxyType(resolved, proxyRandomize); + CService onionProxy; + if (!Lookup(onionArg.c_str(), onionProxy, 9050, fNameLookup)) { + return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); + } + proxyType addrOnion = proxyType(onionProxy, proxyRandomize); if (!addrOnion.IsValid()) - return InitError(strprintf(_("Invalid -onion address: '%s'"), onionArg)); + return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); SetProxy(NET_TOR, addrOnion); SetLimited(NET_TOR, false); } @@ -1298,21 +1344,20 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // see Step 2: parameter interactions for more information about these fListen = GetBoolArg("-listen", DEFAULT_LISTEN); fDiscover = GetBoolArg("-discover", true); - fNameLookup = GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); fRelayTxes = !GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); if (fListen) { bool fBound = false; - if (mapMultiArgs.count("-bind")) { - BOOST_FOREACH(const std::string& strBind, mapMultiArgs.at("-bind")) { + if (gArgs.IsArgSet("-bind")) { + BOOST_FOREACH(const std::string& strBind, gArgs.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) return InitError(ResolveErrMsg("bind", strBind)); fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR)); } } - if (mapMultiArgs.count("-whitebind")) { - BOOST_FOREACH(const std::string& strBind, mapMultiArgs.at("-whitebind")) { + if (gArgs.IsArgSet("-whitebind")) { + BOOST_FOREACH(const std::string& strBind, gArgs.GetArgs("-whitebind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, 0, false)) return InitError(ResolveErrMsg("whitebind", strBind)); @@ -1321,7 +1366,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR | BF_WHITELIST)); } } - if (!mapMultiArgs.count("-bind") && !mapMultiArgs.count("-whitebind")) { + if (!gArgs.IsArgSet("-bind") && !gArgs.IsArgSet("-whitebind")) { struct in_addr inaddr_any; inaddr_any.s_addr = INADDR_ANY; fBound |= Bind(connman, CService(in6addr_any, GetListenPort()), BF_NONE); @@ -1331,8 +1376,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) return InitError(_("Failed to listen on any port. Use -listen=0 if you want this.")); } - if (mapMultiArgs.count("-externalip")) { - BOOST_FOREACH(const std::string& strAddr, mapMultiArgs.at("-externalip")) { + if (gArgs.IsArgSet("-externalip")) { + BOOST_FOREACH(const std::string& strAddr, gArgs.GetArgs("-externalip")) { CService addrLocal; if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); @@ -1341,11 +1386,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (mapMultiArgs.count("-seednode")) { - BOOST_FOREACH(const std::string& strDest, mapMultiArgs.at("-seednode")) - connman.AddOneShot(strDest); - } - #if ENABLE_ZMQ pzmqNotificationInterface = CZMQNotificationInterface::Create(); @@ -1365,7 +1405,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) fReindex = GetBoolArg("-reindex", false); bool fReindexChainState = GetBoolArg("-reindex-chainstate", false); - boost::filesystem::create_directories(GetDataDir() / "blocks"); + fs::create_directories(GetDataDir() / "blocks"); // cache size calculations int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); @@ -1410,6 +1450,12 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (fPruneMode) CleanupBlockRevFiles(); + } else { + // If necessary, upgrade from older database format. + if (!pcoinsdbview->Upgrade()) { + strLoadError = _("Error upgrading chainstate database"); + break; + } } if (!LoadBlockIndex(chainparams)) { @@ -1473,7 +1519,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) break; } } catch (const std::exception& e) { - if (fDebug) LogPrintf("%s\n", e.what()); + LogPrintf("%s\n", e.what()); strLoadError = _("Error opening block database"); break; } @@ -1511,11 +1557,11 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } LogPrintf(" block index %15dms\n", GetTimeMillis() - nStart); - boost::filesystem::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; - CAutoFile est_filein(fopen(est_path.string().c_str(), "rb"), SER_DISK, CLIENT_VERSION); + fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; + CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, CLIENT_VERSION); // Allowed to fail as this file IS missing on first startup. if (!est_filein.IsNull()) - mempool.ReadFeeEstimates(est_filein); + ::feeEstimator.Read(est_filein); fFeeEstimatesInitialized = true; // ********************************************************* Step 8: load wallet @@ -1567,10 +1613,10 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) if (IsArgSet("-blocknotify")) uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); - std::vector<boost::filesystem::path> vImportFiles; - if (mapMultiArgs.count("-loadblock")) + std::vector<fs::path> vImportFiles; + if (gArgs.IsArgSet("-loadblock")) { - BOOST_FOREACH(const std::string& strFile, mapMultiArgs.at("-loadblock")) + BOOST_FOREACH(const std::string& strFile, gArgs.GetArgs("-loadblock")) vImportFiles.push_back(strFile); } @@ -1614,6 +1660,10 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; connOptions.nMaxOutboundLimit = nMaxOutboundLimit; + if (gArgs.IsArgSet("-seednode")) { + connOptions.vSeedNodes = gArgs.GetArgs("-seednode"); + } + if (!connman.Start(scheduler, strNodeError, connOptions)) return InitError(strNodeError); @@ -1624,7 +1674,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) #ifdef ENABLE_WALLET if (pwalletMain) - pwalletMain->postInitProcess(threadGroup); + pwalletMain->postInitProcess(scheduler); #endif return !fRequestShutdown; diff --git a/src/key.cpp b/src/key.cpp index b4f0dc8202..5a75647f1a 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -131,14 +131,6 @@ void CKey::MakeNewKey(bool fCompressedIn) { fCompressed = fCompressedIn; } -bool CKey::SetPrivKey(const CPrivKey &privkey, bool fCompressedIn) { - if (!ec_privkey_import_der(secp256k1_context_sign, (unsigned char*)begin(), &privkey[0], privkey.size())) - return false; - fCompressed = fCompressedIn; - fValid = true; - return true; -} - CPrivKey CKey::GetPrivKey() const { assert(fValid); CPrivKey privkey; @@ -45,7 +45,7 @@ private: //! The actual byte data std::vector<unsigned char, secure_allocator<unsigned char> > keydata; - //! Check whether the 32-byte array pointed to be vch is valid keydata. + //! Check whether the 32-byte array pointed to by vch is valid keydata. bool static Check(const unsigned char* vch); public: @@ -94,9 +94,6 @@ public: //! Check whether the public key corresponding to this private key is (to be) compressed. bool IsCompressed() const { return fCompressed; } - //! Initialize from a CPrivKey (serialized OpenSSL private key data). - bool SetPrivKey(const CPrivKey& vchPrivKey, bool fCompressed); - //! Generate a new private key using a cryptographic PRNG. void MakeNewKey(bool fCompressed); diff --git a/src/memusage.h b/src/memusage.h index 81e8702954..b69acafffd 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -12,6 +12,8 @@ #include <map> #include <set> #include <vector> +#include <unordered_map> +#include <unordered_set> #include <boost/foreach.hpp> #include <boost/unordered_set.hpp> @@ -149,7 +151,7 @@ static inline size_t DynamicUsage(const std::shared_ptr<X>& p) // Boost data structures template<typename X> -struct boost_unordered_node : private X +struct unordered_node : private X { private: void* ptr; @@ -158,13 +160,25 @@ private: template<typename X, typename Y> static inline size_t DynamicUsage(const boost::unordered_set<X, Y>& s) { - return MallocUsage(sizeof(boost_unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); + return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); } template<typename X, typename Y, typename Z> static inline size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& m) { - return MallocUsage(sizeof(boost_unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); + return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); +} + +template<typename X, typename Y> +static inline size_t DynamicUsage(const std::unordered_set<X, Y>& s) +{ + return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); +} + +template<typename X, typename Y, typename Z> +static inline size_t DynamicUsage(const std::unordered_map<X, Y, Z>& m) +{ + return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); } } diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index e3f3e4621a..78d7cd6001 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -65,7 +65,7 @@ uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::ve } else { // calculate left hash uint256 left = CalcHash(height-1, pos*2, vTxid), right; - // calculate right hash if not beyond the end of the array - copy left hash otherwise1 + // calculate right hash if not beyond the end of the array - copy left hash otherwise if (pos*2+1 < CalcTreeWidth(height-1)) right = CalcHash(height-1, pos*2+1, vTxid); else diff --git a/src/merkleblock.h b/src/merkleblock.h index 73cbf670ee..de4c5c8d29 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -23,7 +23,7 @@ * storing a bit for each traversed node, signifying whether the node is the * parent of at least one matched leaf txid (or a matched txid itself). In * case we are at the leaf level, or this bit is 0, its merkle node hash is - * stored, and its children are not explorer further. Otherwise, no hash is + * stored, and its children are not explored further. Otherwise, no hash is * stored, but we recurse into both (or the only) child branch. During * decoding, the same depth-first traversal is performed, consuming bits and * hashes as they written during encoding. diff --git a/src/miner.cpp b/src/miner.cpp index d01edd93b5..28b6f23d56 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -10,11 +10,13 @@ #include "chainparams.h" #include "coins.h" #include "consensus/consensus.h" +#include "consensus/tx_verify.h" #include "consensus/merkle.h" #include "consensus/validation.h" #include "hash.h" #include "validation.h" #include "net.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "pow.h" #include "primitives/transaction.h" @@ -26,8 +28,6 @@ #include "validationinterface.h" #include <algorithm> -#include <boost/thread.hpp> -#include <boost/tuple/tuple.hpp> #include <queue> #include <utility> @@ -39,24 +39,13 @@ // // Unconfirmed transactions in the memory pool often depend on other // transactions in the memory pool. When we select transactions from the -// pool, we select by highest priority or fee rate, so we might consider -// transactions that depend on transactions that aren't yet in the block. +// pool, we select by highest fee rate of a transaction combined with all +// its ancestors. uint64_t nLastBlockTx = 0; uint64_t nLastBlockSize = 0; uint64_t nLastBlockWeight = 0; -class ScoreCompare -{ -public: - ScoreCompare() {} - - bool operator()(const CTxMemPool::txiter a, const CTxMemPool::txiter b) - { - return CompareTxMemPoolEntryByScore()(*b,*a); // Convert to less than - } -}; - int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { int64_t nOldTime = pblock->nTime; @@ -72,43 +61,56 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam return nNewTime - nOldTime; } -BlockAssembler::BlockAssembler(const CChainParams& _chainparams) - : chainparams(_chainparams) +BlockAssembler::Options::Options() { + blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); + nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; + nBlockMaxSize = DEFAULT_BLOCK_MAX_SIZE; +} + +BlockAssembler::BlockAssembler(const CChainParams& params, const Options& options) : chainparams(params) +{ + blockMinFeeRate = options.blockMinFeeRate; + // Limit weight to between 4K and MAX_BLOCK_WEIGHT-4K for sanity: + nBlockMaxWeight = std::max<size_t>(4000, std::min<size_t>(MAX_BLOCK_WEIGHT - 4000, options.nBlockMaxWeight)); + // Limit size to between 1K and MAX_BLOCK_SERIALIZED_SIZE-1K for sanity: + nBlockMaxSize = std::max<size_t>(1000, std::min<size_t>(MAX_BLOCK_SERIALIZED_SIZE - 1000, options.nBlockMaxSize)); + // Whether we need to account for byte usage (in addition to weight usage) + fNeedSizeAccounting = (nBlockMaxSize < MAX_BLOCK_SERIALIZED_SIZE - 1000); +} + +static BlockAssembler::Options DefaultOptions(const CChainParams& params) { // Block resource limits // If neither -blockmaxsize or -blockmaxweight is given, limit to DEFAULT_BLOCK_MAX_* // If only one is given, only restrict the specified resource. // If both are given, restrict both. - nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; - nBlockMaxSize = DEFAULT_BLOCK_MAX_SIZE; + BlockAssembler::Options options; + options.nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; + options.nBlockMaxSize = DEFAULT_BLOCK_MAX_SIZE; bool fWeightSet = false; if (IsArgSet("-blockmaxweight")) { - nBlockMaxWeight = GetArg("-blockmaxweight", DEFAULT_BLOCK_MAX_WEIGHT); - nBlockMaxSize = MAX_BLOCK_SERIALIZED_SIZE; + options.nBlockMaxWeight = GetArg("-blockmaxweight", DEFAULT_BLOCK_MAX_WEIGHT); + options.nBlockMaxSize = MAX_BLOCK_SERIALIZED_SIZE; fWeightSet = true; } if (IsArgSet("-blockmaxsize")) { - nBlockMaxSize = GetArg("-blockmaxsize", DEFAULT_BLOCK_MAX_SIZE); + options.nBlockMaxSize = GetArg("-blockmaxsize", DEFAULT_BLOCK_MAX_SIZE); if (!fWeightSet) { - nBlockMaxWeight = nBlockMaxSize * WITNESS_SCALE_FACTOR; + options.nBlockMaxWeight = options.nBlockMaxSize * WITNESS_SCALE_FACTOR; } } if (IsArgSet("-blockmintxfee")) { CAmount n = 0; ParseMoney(GetArg("-blockmintxfee", ""), n); - blockMinFeeRate = CFeeRate(n); + options.blockMinFeeRate = CFeeRate(n); } else { - blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); + options.blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); } - - // Limit weight to between 4K and MAX_BLOCK_WEIGHT-4K for sanity: - nBlockMaxWeight = std::max((unsigned int)4000, std::min((unsigned int)(MAX_BLOCK_WEIGHT-4000), nBlockMaxWeight)); - // Limit size to between 1K and MAX_BLOCK_SERIALIZED_SIZE-1K for sanity: - nBlockMaxSize = std::max((unsigned int)1000, std::min((unsigned int)(MAX_BLOCK_SERIALIZED_SIZE-1000), nBlockMaxSize)); - // Whether we need to account for byte usage (in addition to weight usage) - fNeedSizeAccounting = (nBlockMaxSize < MAX_BLOCK_SERIALIZED_SIZE-1000); + return options; } +BlockAssembler::BlockAssembler(const CChainParams& params) : BlockAssembler(params, DefaultOptions(params)) {} + void BlockAssembler::resetBlock() { inBlock.clear(); @@ -122,13 +124,12 @@ void BlockAssembler::resetBlock() // These counters do not include coinbase tx nBlockTx = 0; nFees = 0; - - lastFewTxs = 0; - blockFinished = false; } -std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) +std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn, bool fMineWitnessTx) { + int64_t nTimeStart = GetTimeMicros(); + resetBlock(); pblocktemplate.reset(new CBlockTemplate()); @@ -165,10 +166,13 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc // -promiscuousmempoolflags is used. // TODO: replace this with a call to main to assess validity of a mempool // transaction (which in most cases can be a no-op). - fIncludeWitness = IsWitnessEnabled(pindexPrev, chainparams.GetConsensus()); + fIncludeWitness = IsWitnessEnabled(pindexPrev, chainparams.GetConsensus()) && fMineWitnessTx; + + int nPackagesSelected = 0; + int nDescendantsUpdated = 0; + addPackageTxs(nPackagesSelected, nDescendantsUpdated); - addPriorityTxs(); - addPackageTxs(); + int64_t nTime1 = GetTimeMicros(); nLastBlockTx = nBlockTx; nLastBlockSize = nBlockSize; @@ -200,19 +204,11 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state))); } + int64_t nTime2 = GetTimeMicros(); - return std::move(pblocktemplate); -} + LogPrint(BCLog::BENCH, "CreateNewBlock() packages: %.2fms (%d packages, %d updated descendants), validity: %.2fms (total %.2fms)\n", 0.001 * (nTime1 - nTimeStart), nPackagesSelected, nDescendantsUpdated, 0.001 * (nTime2 - nTime1), 0.001 * (nTime2 - nTimeStart)); -bool BlockAssembler::isStillDependent(CTxMemPool::txiter iter) -{ - BOOST_FOREACH(CTxMemPool::txiter parent, mempool.GetMemPoolParents(iter)) - { - if (!inBlock.count(parent)) { - return true; - } - } - return false; + return std::move(pblocktemplate); } void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet) @@ -262,58 +258,6 @@ bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& packa return true; } -bool BlockAssembler::TestForBlock(CTxMemPool::txiter iter) -{ - if (nBlockWeight + iter->GetTxWeight() >= nBlockMaxWeight) { - // If the block is so close to full that no more txs will fit - // or if we've tried more than 50 times to fill remaining space - // then flag that the block is finished - if (nBlockWeight > nBlockMaxWeight - 400 || lastFewTxs > 50) { - blockFinished = true; - return false; - } - // Once we're within 4000 weight of a full block, only look at 50 more txs - // to try to fill the remaining space. - if (nBlockWeight > nBlockMaxWeight - 4000) { - lastFewTxs++; - } - return false; - } - - if (fNeedSizeAccounting) { - if (nBlockSize + ::GetSerializeSize(iter->GetTx(), SER_NETWORK, PROTOCOL_VERSION) >= nBlockMaxSize) { - if (nBlockSize > nBlockMaxSize - 100 || lastFewTxs > 50) { - blockFinished = true; - return false; - } - if (nBlockSize > nBlockMaxSize - 1000) { - lastFewTxs++; - } - return false; - } - } - - if (nBlockSigOpsCost + iter->GetSigOpCost() >= MAX_BLOCK_SIGOPS_COST) { - // If the block has room for no more sig ops then - // flag that the block is finished - if (nBlockSigOpsCost > MAX_BLOCK_SIGOPS_COST - 8) { - blockFinished = true; - return false; - } - // Otherwise attempt to find another tx with fewer sigops - // to put in the block. - return false; - } - - // Must check that lock times are still valid - // This can be removed once MTP is always enforced - // as long as reorgs keep the mempool consistent. - if (!IsFinalTx(iter->GetTx(), nHeight, nLockTimeCutoff)) - return false; - - return true; -} - void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) { pblock->vtx.emplace_back(iter->GetSharedTx()); @@ -330,19 +274,16 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) bool fPrintPriority = GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); if (fPrintPriority) { - double dPriority = iter->GetPriority(nHeight); - CAmount dummy; - mempool.ApplyDeltas(iter->GetTx().GetHash(), dPriority, dummy); - LogPrintf("priority %.1f fee %s txid %s\n", - dPriority, + LogPrintf("fee %s txid %s\n", CFeeRate(iter->GetModifiedFee(), iter->GetTxSize()).ToString(), iter->GetTx().GetHash().ToString()); } } -void BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, +int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx) { + int nDescendantsUpdated = 0; BOOST_FOREACH(const CTxMemPool::txiter it, alreadyAdded) { CTxMemPool::setEntries descendants; mempool.CalculateDescendants(it, descendants); @@ -350,6 +291,7 @@ void BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alread BOOST_FOREACH(CTxMemPool::txiter desc, descendants) { if (alreadyAdded.count(desc)) continue; + ++nDescendantsUpdated; modtxiter mit = mapModifiedTx.find(desc); if (mit == mapModifiedTx.end()) { CTxMemPoolModifiedEntry modEntry(desc); @@ -362,6 +304,7 @@ void BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alread } } } + return nDescendantsUpdated; } // Skip entries in mapTx that are already in a block or are present @@ -376,9 +319,7 @@ void BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alread bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) { assert (it != mempool.mapTx.end()); - if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it)) - return true; - return false; + return mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it); } void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, CTxMemPool::txiter entry, std::vector<CTxMemPool::txiter>& sortedEntries) @@ -402,7 +343,7 @@ void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, CTxMemP // Each time through the loop, we compare the best transaction in // mapModifiedTxs with the next transaction in the mempool to decide what // transaction package to work on next. -void BlockAssembler::addPackageTxs() +void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpdated) { // mapModifiedTx will store sorted packages after they are modified // because some of their txs are already in the block @@ -416,6 +357,13 @@ void BlockAssembler::addPackageTxs() CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin(); CTxMemPool::txiter iter; + + // Limit the number of attempts to add transactions to the block when it is + // close to full; this is just a simple heuristic to finish quickly if the + // mempool has a lot of entries. + const int64_t MAX_CONSECUTIVE_FAILURES = 1000; + int64_t nConsecutiveFailed = 0; + while (mi != mempool.mapTx.get<ancestor_score>().end() || !mapModifiedTx.empty()) { // First try to find a new transaction in mapTx to evaluate. @@ -477,6 +425,14 @@ void BlockAssembler::addPackageTxs() mapModifiedTx.get<ancestor_score>().erase(modit); failedTx.insert(iter); } + + ++nConsecutiveFailed; + + if (nConsecutiveFailed > MAX_CONSECUTIVE_FAILURES && nBlockWeight > + nBlockMaxWeight - 4000) { + // Give up if we're close to full and haven't succeeded in a while + break; + } continue; } @@ -497,6 +453,9 @@ void BlockAssembler::addPackageTxs() continue; } + // This transaction will make it in; reset the failed counter. + nConsecutiveFailed = 0; + // Package can be added. Sort the entries in a valid order. std::vector<CTxMemPool::txiter> sortedEntries; SortForBlock(ancestors, iter, sortedEntries); @@ -507,91 +466,11 @@ void BlockAssembler::addPackageTxs() mapModifiedTx.erase(sortedEntries[i]); } - // Update transactions that depend on each of these - UpdatePackagesForAdded(ancestors, mapModifiedTx); - } -} + ++nPackagesSelected; -void BlockAssembler::addPriorityTxs() -{ - // How much of the block should be dedicated to high-priority transactions, - // included regardless of the fees they pay - unsigned int nBlockPrioritySize = GetArg("-blockprioritysize", DEFAULT_BLOCK_PRIORITY_SIZE); - nBlockPrioritySize = std::min(nBlockMaxSize, nBlockPrioritySize); - - if (nBlockPrioritySize == 0) { - return; - } - - bool fSizeAccounting = fNeedSizeAccounting; - fNeedSizeAccounting = true; - - // This vector will be sorted into a priority queue: - std::vector<TxCoinAgePriority> vecPriority; - TxCoinAgePriorityCompare pricomparer; - std::map<CTxMemPool::txiter, double, CTxMemPool::CompareIteratorByHash> waitPriMap; - typedef std::map<CTxMemPool::txiter, double, CTxMemPool::CompareIteratorByHash>::iterator waitPriIter; - double actualPriority = -1; - - vecPriority.reserve(mempool.mapTx.size()); - for (CTxMemPool::indexed_transaction_set::iterator mi = mempool.mapTx.begin(); - mi != mempool.mapTx.end(); ++mi) - { - double dPriority = mi->GetPriority(nHeight); - CAmount dummy; - mempool.ApplyDeltas(mi->GetTx().GetHash(), dPriority, dummy); - vecPriority.push_back(TxCoinAgePriority(dPriority, mi)); - } - std::make_heap(vecPriority.begin(), vecPriority.end(), pricomparer); - - CTxMemPool::txiter iter; - while (!vecPriority.empty() && !blockFinished) { // add a tx from priority queue to fill the blockprioritysize - iter = vecPriority.front().second; - actualPriority = vecPriority.front().first; - std::pop_heap(vecPriority.begin(), vecPriority.end(), pricomparer); - vecPriority.pop_back(); - - // If tx already in block, skip - if (inBlock.count(iter)) { - assert(false); // shouldn't happen for priority txs - continue; - } - - // cannot accept witness transactions into a non-witness block - if (!fIncludeWitness && iter->GetTx().HasWitness()) - continue; - - // If tx is dependent on other mempool txs which haven't yet been included - // then put it in the waitSet - if (isStillDependent(iter)) { - waitPriMap.insert(std::make_pair(iter, actualPriority)); - continue; - } - - // If this tx fits in the block add it, otherwise keep looping - if (TestForBlock(iter)) { - AddToBlock(iter); - - // If now that this txs is added we've surpassed our desired priority size - // or have dropped below the AllowFreeThreshold, then we're done adding priority txs - if (nBlockSize >= nBlockPrioritySize || !AllowFree(actualPriority)) { - break; - } - - // This tx was successfully added, so - // add transactions that depend on this one to the priority queue to try again - BOOST_FOREACH(CTxMemPool::txiter child, mempool.GetMemPoolChildren(iter)) - { - waitPriIter wpiter = waitPriMap.find(child); - if (wpiter != waitPriMap.end()) { - vecPriority.push_back(TxCoinAgePriority(wpiter->second,child)); - std::push_heap(vecPriority.begin(), vecPriority.end(), pricomparer); - waitPriMap.erase(wpiter); - } - } - } + // Update transactions that depend on each of these + nDescendantsUpdated += UpdatePackagesForAdded(ancestors, mapModifiedTx); } - fNeedSizeAccounting = fSizeAccounting; } void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce) diff --git a/src/miner.h b/src/miner.h index 3ba92b16b8..1f3c9d652f 100644 --- a/src/miner.h +++ b/src/miner.h @@ -158,14 +158,19 @@ private: int64_t nLockTimeCutoff; const CChainParams& chainparams; - // Variables used for addPriorityTxs - int lastFewTxs; - bool blockFinished; - public: - BlockAssembler(const CChainParams& chainparams); + struct Options { + Options(); + size_t nBlockMaxWeight; + size_t nBlockMaxSize; + CFeeRate blockMinFeeRate; + }; + + BlockAssembler(const CChainParams& params); + BlockAssembler(const CChainParams& params, const Options& options); + /** Construct a new block template with coinbase to scriptPubKeyIn */ - std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn); + std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn, bool fMineWitnessTx=true); private: // utility functions @@ -175,16 +180,10 @@ private: void AddToBlock(CTxMemPool::txiter iter); // Methods for how to add transactions to a block. - /** Add transactions based on tx "priority" */ - void addPriorityTxs(); - /** Add transactions based on feerate including unconfirmed ancestors */ - void addPackageTxs(); - - // helper function for addPriorityTxs - /** Test if tx will still "fit" in the block */ - bool TestForBlock(CTxMemPool::txiter iter); - /** Test if tx still has unconfirmed parents not yet in block */ - bool isStillDependent(CTxMemPool::txiter iter); + /** Add transactions based on feerate including unconfirmed ancestors + * Increments nPackagesSelected / nDescendantsUpdated with corresponding + * statistics from the package selection (for logging statistics). */ + void addPackageTxs(int &nPackagesSelected, int &nDescendantsUpdated); // helper functions for addPackageTxs() /** Remove confirmed (inBlock) entries from given set */ @@ -202,8 +201,9 @@ private: /** Sort the package in an order that is valid to appear in a block */ void SortForBlock(const CTxMemPool::setEntries& package, CTxMemPool::txiter entry, std::vector<CTxMemPool::txiter>& sortedEntries); /** Add descendants of given transactions to mapModifiedTx with ancestor - * state updated assuming given transactions are inBlock. */ - void UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx); + * state updated assuming given transactions are inBlock. Returns number + * of updated descendants. */ + int UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx); }; /** Modify the extranonce in a block */ diff --git a/src/net.cpp b/src/net.cpp index de5fc29693..14ac5618eb 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -44,10 +44,15 @@ // We add a random period time (0 to 1 seconds) to feeler connections to prevent synchronization. #define FEELER_SLEEP_WINDOW 1 -#if !defined(HAVE_MSG_NOSIGNAL) && !defined(MSG_NOSIGNAL) +#if !defined(HAVE_MSG_NOSIGNAL) #define MSG_NOSIGNAL 0 #endif +// MSG_DONTWAIT is not available on some platforms, if it doesn't exist define it as 0 +#if !defined(HAVE_MSG_DONTWAIT) +#define MSG_DONTWAIT 0 +#endif + // Fix for ancient MinGW versions, that don't have defined these in ws2tcpip.h. // Todo: Can be removed when our pull-tester is upgraded to a modern MinGW version. #ifdef WIN32 @@ -143,7 +148,7 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn // one by discovery. CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) { - CAddress ret(CService(CNetAddr(),GetListenPort()), NODE_NONE); + CAddress ret(CService(CNetAddr(),GetListenPort()), nLocalServices); CService addr; if (GetLocal(addr, paddrPeer)) { @@ -185,7 +190,7 @@ void AdvertiseLocal(CNode *pnode) } if (addrLocal.IsRoutable()) { - LogPrint("net", "AdvertiseLocal: advertising address %s\n", addrLocal.ToString()); + LogPrint(BCLog::NET, "AdvertiseLocal: advertising address %s\n", addrLocal.ToString()); FastRandomContext insecure_rand; pnode->PushAddress(addrLocal, insecure_rand); } @@ -335,6 +340,22 @@ bool CConnman::CheckIncomingNonce(uint64_t nonce) return true; } +/** Get the bind address for a socket as CAddress */ +static CAddress GetBindAddress(SOCKET sock) +{ + CAddress addr_bind; + struct sockaddr_storage sockaddr_bind; + socklen_t sockaddr_bind_len = sizeof(sockaddr_bind); + if (sock != INVALID_SOCKET) { + if (!getsockname(sock, (struct sockaddr*)&sockaddr_bind, &sockaddr_bind_len)) { + addr_bind.SetSockAddr((const struct sockaddr*)&sockaddr_bind); + } else { + LogPrint(BCLog::NET, "Warning: getsockname failed\n"); + } + } + return addr_bind; +} + CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure) { if (pszDest == NULL) { @@ -351,7 +372,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo } /// debug print - LogPrint("net", "trying connection %s lastseen=%.1fhrs\n", + LogPrint(BCLog::NET, "trying connection %s lastseen=%.1fhrs\n", pszDest ? pszDest : addrConnect.ToString(), pszDest ? 0.0 : (double)(GetAdjustedTime() - addrConnect.nTime)/3600.0); @@ -388,7 +409,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo // Add node NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, pszDest ? pszDest : "", false); + CAddress addr_bind = GetBindAddress(hSocket); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->AddRef(); @@ -413,12 +435,12 @@ void CConnman::DumpBanlist() CBanDB bandb; banmap_t banmap; - SetBannedSetDirty(false); GetBanned(banmap); - if (!bandb.Write(banmap)) - SetBannedSetDirty(true); + if (bandb.Write(banmap)) { + SetBannedSetDirty(false); + } - LogPrint("net", "Flushed %d banned node ips/subnets to banlist.dat %dms\n", + LogPrint(BCLog::NET, "Flushed %d banned node ips/subnets to banlist.dat %dms\n", banmap.size(), GetTimeMillis() - nStart); } @@ -428,7 +450,7 @@ void CNode::CloseSocketDisconnect() LOCK(cs_hSocket); if (hSocket != INVALID_SOCKET) { - LogPrint("net", "disconnecting peer=%d\n", id); + LogPrint(BCLog::NET, "disconnecting peer=%d\n", id); CloseSocket(hSocket); } } @@ -536,6 +558,8 @@ bool CConnman::Unban(const CSubNet &subNet) { void CConnman::GetBanned(banmap_t &banMap) { LOCK(cs_setBanned); + // Sweep the banlist so expired bans are not returned + SweepBanned(); banMap = setBanned; //create a thread safe copy } @@ -560,7 +584,7 @@ void CConnman::SweepBanned() { setBanned.erase(it++); setBannedIsDirty = true; - LogPrint("net", "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); + LogPrint(BCLog::NET, "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); } else ++it; @@ -628,6 +652,7 @@ void CNode::copyStats(CNodeStats &stats) stats.nodeid = this->GetId(); X(nServices); X(addr); + X(addrBind); { LOCK(cs_filter); X(fRelayTxes); @@ -703,10 +728,10 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete handled = msg.readData(pch, nBytes); if (handled < 0) - return false; + return false; if (msg.in_data && msg.hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { - LogPrint("net", "Oversized message from peer=%i, disconnecting\n", GetId()); + LogPrint(BCLog::NET, "Oversized message from peer=%i, disconnecting\n", GetId()); return false; } @@ -781,7 +806,7 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) // reject messages larger than MAX_SIZE if (hdr.nMessageSize > MAX_SIZE) - return -1; + return -1; // switch state to reading message data in_data = true; @@ -941,7 +966,7 @@ bool CConnman::AttemptToEvictConnection() continue; if (node->fDisconnect) continue; - NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, + NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, node->nLastBlockTime, node->nLastTXTime, (node->nServices & nRelevantServices) == nRelevantServices, node->fRelayTxes, node->pfilter != NULL, node->addr, node->nKeyedNetGroup}; @@ -1029,9 +1054,11 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { int nInbound = 0; int nMaxInbound = nMaxConnections - (nMaxOutbound + nMaxFeeler); - if (hSocket != INVALID_SOCKET) - if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) + if (hSocket != INVALID_SOCKET) { + if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) { LogPrintf("Warning: Unknown socket family\n"); + } + } bool whitelisted = hListenSocket.whitelisted || IsWhitelistedRange(addr); { @@ -1064,12 +1091,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { // According to the internet TCP_NODELAY is not carried into accepted sockets // on all platforms. Set it again here just to be sure. - int set = 1; -#ifdef WIN32 - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); -#else - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); -#endif + SetSocketNoDelay(hSocket); if (IsBanned(addr) && !whitelisted) { @@ -1082,7 +1104,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { { if (!AttemptToEvictConnection()) { // No connection to evict, disconnect the new connection - LogPrint("net", "failed to find an eviction candidate - connection dropped (full)\n"); + LogPrint(BCLog::NET, "failed to find an eviction candidate - connection dropped (full)\n"); CloseSocket(hSocket); return; } @@ -1090,13 +1112,14 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, "", true); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", true); pnode->AddRef(); pnode->fWhitelisted = whitelisted; GetNodeSignals().InitializeNode(pnode, *this); - LogPrint("net", "connection from %s accepted\n", addr.ToString()); + LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); { LOCK(cs_vNodes); @@ -1294,58 +1317,55 @@ void CConnman::ThreadSocketHandler() } if (recvSet || errorSet) { + // typical socket buffer is 8K-64K + char pchBuf[0x10000]; + int nBytes = 0; { - { - // typical socket buffer is 8K-64K - char pchBuf[0x10000]; - int nBytes = 0; - { - LOCK(pnode->cs_hSocket); - if (pnode->hSocket == INVALID_SOCKET) - continue; - nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); - } - if (nBytes > 0) - { - bool notify = false; - if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) - pnode->CloseSocketDisconnect(); - RecordBytesRecv(nBytes); - if (notify) { - size_t nSizeAdded = 0; - auto it(pnode->vRecvMsg.begin()); - for (; it != pnode->vRecvMsg.end(); ++it) { - if (!it->complete()) - break; - nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; - } - { - LOCK(pnode->cs_vProcessMsg); - pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); - pnode->nProcessQueueSize += nSizeAdded; - pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; - } - WakeMessageHandler(); - } + LOCK(pnode->cs_hSocket); + if (pnode->hSocket == INVALID_SOCKET) + continue; + nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); + } + if (nBytes > 0) + { + bool notify = false; + if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) + pnode->CloseSocketDisconnect(); + RecordBytesRecv(nBytes); + if (notify) { + size_t nSizeAdded = 0; + auto it(pnode->vRecvMsg.begin()); + for (; it != pnode->vRecvMsg.end(); ++it) { + if (!it->complete()) + break; + nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; } - else if (nBytes == 0) { - // socket closed gracefully - if (!pnode->fDisconnect) - LogPrint("net", "socket closed\n"); - pnode->CloseSocketDisconnect(); - } - else if (nBytes < 0) - { - // error - int nErr = WSAGetLastError(); - if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) - { - if (!pnode->fDisconnect) - LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); - pnode->CloseSocketDisconnect(); - } + LOCK(pnode->cs_vProcessMsg); + pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); + pnode->nProcessQueueSize += nSizeAdded; + pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; } + WakeMessageHandler(); + } + } + else if (nBytes == 0) + { + // socket closed gracefully + if (!pnode->fDisconnect) { + LogPrint(BCLog::NET, "socket closed\n"); + } + pnode->CloseSocketDisconnect(); + } + else if (nBytes < 0) + { + // error + int nErr = WSAGetLastError(); + if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) + { + if (!pnode->fDisconnect) + LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); + pnode->CloseSocketDisconnect(); } } } @@ -1370,7 +1390,7 @@ void CConnman::ThreadSocketHandler() { if (pnode->nLastRecv == 0 || pnode->nLastSend == 0) { - LogPrint("net", "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->id); + LogPrint(BCLog::NET, "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->GetId()); pnode->fDisconnect = true; } else if (nTime - pnode->nLastSend > TIMEOUT_INTERVAL) @@ -1390,7 +1410,7 @@ void CConnman::ThreadSocketHandler() } else if (!pnode->fSuccessfullyConnected) { - LogPrintf("version handshake timeout from %d\n", pnode->id); + LogPrintf("version handshake timeout from %d\n", pnode->GetId()); pnode->fDisconnect = true; } } @@ -1579,6 +1599,9 @@ void CConnman::ThreadDNSAddressSeed() LogPrintf("Loading addresses from DNS seeds (could take a while)\n"); BOOST_FOREACH(const CDNSSeedData &seed, vSeeds) { + if (interruptNet) { + return; + } if (HaveNameProxy()) { AddOneShot(seed.host); } else { @@ -1596,6 +1619,9 @@ void CConnman::ThreadDNSAddressSeed() found++; } } + if (interruptNet) { + return; + } // TODO: The seed name resolve may fail, yielding an IP of [::], which results in // addrman assigning the same source to results from different seeds. // This should switch to a hard-coded stable dummy IP for each seed name, so that the @@ -1629,7 +1655,7 @@ void CConnman::DumpAddresses() CAddrDB adb; adb.Write(addrman); - LogPrint("net", "Flushed %d addresses to peers.dat %dms\n", + LogPrint(BCLog::NET, "Flushed %d addresses to peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); } @@ -1660,12 +1686,12 @@ void CConnman::ProcessOneShot() void CConnman::ThreadOpenConnections() { // Connect to specific addresses - if (mapMultiArgs.count("-connect") && mapMultiArgs.at("-connect").size() > 0) + if (gArgs.IsArgSet("-connect") && gArgs.GetArgs("-connect").size() > 0) { for (int64_t nLoop = 0;; nLoop++) { ProcessOneShot(); - BOOST_FOREACH(const std::string& strAddr, mapMultiArgs.at("-connect")) + BOOST_FOREACH(const std::string& strAddr, gArgs.GetArgs("-connect")) { CAddress addr(CService(), NODE_NONE); OpenNetworkConnection(addr, false, NULL, strAddr.c_str()); @@ -1716,11 +1742,17 @@ void CConnman::ThreadOpenConnections() // Only connect out to one peer per network group (/16 for IPv4). // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. int nOutbound = 0; + int nOutboundRelevant = 0; std::set<std::vector<unsigned char> > setConnected; { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { if (!pnode->fInbound && !pnode->fAddnode) { + + // Count the peers that have all relevant services + if (pnode->fSuccessfullyConnected && !pnode->fFeeler && ((pnode->nServices & nRelevantServices) == nRelevantServices)) { + nOutboundRelevant++; + } // Netgroups for inbound and addnode peers are not excluded because our goal here // is to not use multiple of our limited outbound slots on a single netgroup // but inbound and addnode peers do not use our outbound slots. Inbound peers @@ -1738,9 +1770,9 @@ void CConnman::ThreadOpenConnections() // * Increase the number of connectable addresses in the tried table. // // Method: - // * Choose a random address from new and attempt to connect to it if we can connect + // * Choose a random address from new and attempt to connect to it if we can connect // successfully it is added to tried. - // * Start attempting feeler connections only after node finishes making outbound + // * Start attempting feeler connections only after node finishes making outbound // connections. // * Only make a feeler connection once every few minutes. // @@ -1784,14 +1816,27 @@ void CConnman::ThreadOpenConnections() continue; // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. - if ((addr.nServices & nRelevantServices) != nRelevantServices && (nTries < 40 || nOutbound >= (nMaxOutbound >> 1))) + ServiceFlags nRequiredServices = nRelevantServices; + if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { + nRequiredServices = REQUIRED_SERVICES; + } + + if ((addr.nServices & nRequiredServices) != nRequiredServices) { continue; + } // do not allow non-default ports, unless after 50 invalid addresses selected already if (addr.GetPort() != Params().GetDefaultPort() && nTries < 50) continue; addrConnect = addr; + + // regardless of the services assumed to be available, only require the minimum if half or more outbound have relevant services + if (nOutboundRelevant >= (nMaxOutbound >> 1)) { + addrConnect.nServices = REQUIRED_SERVICES; + } else { + addrConnect.nServices = nRequiredServices; + } break; } @@ -1802,7 +1847,7 @@ void CConnman::ThreadOpenConnections() int randsleep = GetRandInt(FEELER_SLEEP_WINDOW * 1000); if (!interruptNet.sleep_for(std::chrono::milliseconds(randsleep))) return; - LogPrint("net", "Making feeler connection to %s\n", addrConnect.ToString()); + LogPrint(BCLog::NET, "Making feeler connection to %s\n", addrConnect.ToString()); } OpenNetworkConnection(addrConnect, (int)setConnected.size() >= std::min(nMaxConnections - 1, 2), &grant, NULL, false, fFeeler); @@ -1867,8 +1912,8 @@ void CConnman::ThreadOpenAddedConnections() { { LOCK(cs_vAddedNodes); - if (mapMultiArgs.count("-addnode")) - vAddedNodes = mapMultiArgs.at("-addnode"); + if (gArgs.IsArgSet("-addnode")) + vAddedNodes = gArgs.GetArgs("-addnode"); } while (true) @@ -2145,9 +2190,7 @@ void Discover(boost::thread_group& threadGroup) void CConnman::SetNetworkActive(bool active) { - if (fDebug) { - LogPrint("net", "SetNetworkActive: %s\n", active); - } + LogPrint(BCLog::NET, "SetNetworkActive: %s\n", active); if (!active) { fNetworkActive = false; @@ -2209,9 +2252,14 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c SetBestHeight(connOptions.nBestHeight); + for (const auto& strDest : connOptions.vSeedNodes) { + AddOneShot(strDest); + } + clientInterface = connOptions.uiInterface; - if (clientInterface) - clientInterface->InitMessage(_("Loading addresses...")); + if (clientInterface) { + clientInterface->InitMessage(_("Loading P2P addresses...")); + } // Load addresses from peers.dat int64_t nStart = GetTimeMillis(); { @@ -2235,7 +2283,7 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c SetBannedSetDirty(false); // no need to write down, just read data SweepBanned(); // sweep out unused entries - LogPrint("net", "Loaded %d banned node ips/subnets from banlist.dat %dms\n", + LogPrint(BCLog::NET, "Loaded %d banned node ips/subnets from banlist.dat %dms\n", banmap.size(), GetTimeMillis() - nStart); } else { LogPrintf("Invalid or missing banlist.dat; recreating\n"); @@ -2280,14 +2328,14 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c threadOpenAddedConnections = std::thread(&TraceThread<std::function<void()> >, "addcon", std::function<void()>(std::bind(&CConnman::ThreadOpenAddedConnections, this))); // Initiate outbound connections unless connect=0 - if (!mapMultiArgs.count("-connect") || mapMultiArgs.at("-connect").size() != 1 || mapMultiArgs.at("-connect")[0] != "0") + if (!gArgs.IsArgSet("-connect") || gArgs.GetArgs("-connect").size() != 1 || gArgs.GetArgs("-connect")[0] != "0") threadOpenConnections = std::thread(&TraceThread<std::function<void()> >, "opencon", std::function<void()>(std::bind(&CConnman::ThreadOpenConnections, this))); // Process messages threadMessageHandler = std::thread(&TraceThread<std::function<void()> >, "msghand", std::function<void()>(std::bind(&CConnman::ThreadMessageHandler, this))); // Dump network addresses - scheduler.scheduleEvery(boost::bind(&CConnman::DumpData, this), DUMP_ADDRESSES_INTERVAL); + scheduler.scheduleEvery(std::bind(&CConnman::DumpData, this), DUMP_ADDRESSES_INTERVAL * 1000); return true; } @@ -2318,9 +2366,17 @@ void CConnman::Interrupt() interruptNet(); InterruptSocks5(true); - if (semOutbound) - for (int i=0; i<(nMaxOutbound + nMaxFeeler); i++) + if (semOutbound) { + for (int i=0; i<(nMaxOutbound + nMaxFeeler); i++) { semOutbound->post(); + } + } + + if (semAddnode) { + for (int i=0; i<nMaxAddnode; i++) { + semAddnode->post(); + } + } } void CConnman::Stop() @@ -2336,10 +2392,6 @@ void CConnman::Stop() if (threadSocketHandler.joinable()) threadSocketHandler.join(); - if (semAddnode) - for (int i=0; i<nMaxAddnode; i++) - semOutbound->post(); - if (fAddressesInitialized) { DumpData(); @@ -2401,11 +2453,6 @@ void CConnman::MarkAddressGood(const CAddress& addr) addrman.Good(addr); } -void CConnman::AddNewAddress(const CAddress& addr, const CAddress& addrFrom, int64_t nTimePenalty) -{ - addrman.Add(addr, addrFrom, nTimePenalty); -} - void CConnman::AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty) { addrman.Add(vAddr, addrFrom, nTimePenalty); @@ -2479,7 +2526,7 @@ bool CConnman::DisconnectNode(NodeId id) { LOCK(cs_vNodes); for(CNode* pnode : vNodes) { - if (id == pnode->id) { + if (id == pnode->GetId()) { pnode->fDisconnect = true; return true; } @@ -2613,14 +2660,15 @@ int CConnman::GetBestHeight() const unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } unsigned int CConnman::GetSendBufferSize() const{ return nSendBufferMaxSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string& addrNameIn, bool fInboundIn) : +CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string& addrNameIn, bool fInboundIn) : nTimeConnected(GetSystemTimeInSeconds()), addr(addrIn), + addrBind(addrBindIn), fInbound(fInboundIn), - id(idIn), nKeyedNetGroup(nKeyedNetGroupIn), addrKnown(5000, 0.001), filterInventoryKnown(50000, 0.000001), + id(idIn), nLocalHostNonce(nLocalHostNonceIn), nLocalServices(nLocalServicesIn), nMyStartingHeight(nMyStartingHeightIn), @@ -2678,10 +2726,11 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn mapRecvBytesPerMsgCmd[msg] = 0; mapRecvBytesPerMsgCmd[NET_MESSAGE_COMMAND_OTHER] = 0; - if (fLogIPs) - LogPrint("net", "Added connection to %s peer=%d\n", addrName, id); - else - LogPrint("net", "Added connection peer=%d\n", id); + if (fLogIPs) { + LogPrint(BCLog::NET, "Added connection to %s peer=%d\n", addrName, id); + } else { + LogPrint(BCLog::NET, "Added connection peer=%d\n", id); + } } CNode::~CNode() @@ -2708,7 +2757,7 @@ void CNode::AskFor(const CInv& inv) nRequestTime = it->second; else nRequestTime = 0; - LogPrint("net", "askfor %s %d (%s) peer=%d\n", inv.ToString(), nRequestTime, DateTimeStrFormat("%H:%M:%S", nRequestTime/1000000), id); + LogPrint(BCLog::NET, "askfor %s %d (%s) peer=%d\n", inv.ToString(), nRequestTime, DateTimeStrFormat("%H:%M:%S", nRequestTime/1000000), id); // Make sure not to reuse time indexes to keep things in the same order int64_t nNow = GetTimeMicros() - 1000000; @@ -2735,7 +2784,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrint("net", "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->id); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->GetId()); std::vector<unsigned char> serializedHeader; serializedHeader.reserve(CMessageHeader::HEADER_SIZE); @@ -2773,7 +2822,7 @@ bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) CNode* found = nullptr; LOCK(cs_vNodes); for (auto&& pnode : vNodes) { - if(pnode->id == id) { + if(pnode->GetId() == id) { found = pnode; break; } @@ -14,6 +14,7 @@ #include "hash.h" #include "limitedmap.h" #include "netaddress.h" +#include "policy/feerate.h" #include "protocol.h" #include "random.h" #include "streams.h" @@ -32,11 +33,9 @@ #include <arpa/inet.h> #endif -#include <boost/filesystem/path.hpp> #include <boost/foreach.hpp> #include <boost/signals2/signal.hpp> -class CAddrMan; class CScheduler; class CNode; @@ -92,7 +91,7 @@ static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban -typedef int NodeId; +typedef int64_t NodeId; struct AddedNodeInfo { @@ -145,6 +144,7 @@ public: unsigned int nReceiveFloodSize = 0; uint64_t nMaxOutboundTimeframe = 0; uint64_t nMaxOutboundLimit = 0; + std::vector<std::string> vSeedNodes; }; CConnman(uint64_t seed0, uint64_t seed1); ~CConnman(); @@ -207,10 +207,8 @@ public: size_t GetAddressCount() const; void SetServices(const CService &addr, ServiceFlags nServices); void MarkAddressGood(const CAddress& addr); - void AddNewAddress(const CAddress& addr, const CAddress& addrFrom, int64_t nTimePenalty = 0); void AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty = 0); std::vector<CAddress> GetAddresses(); - void AddressCurrentlyConnected(const CService& addr); // Denial-of-service detection/prevention // The idea is to detect peers that are behaving @@ -236,8 +234,6 @@ public: void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); - void AddOneShot(const std::string& strDest); - bool AddNode(const std::string& node); bool RemoveAddedNode(const std::string& node); std::vector<AddedNodeInfo> GetAddedNodeInfo(); @@ -295,6 +291,7 @@ private: }; void ThreadOpenAddedConnections(); + void AddOneShot(const std::string& strDest); void ProcessOneShot(); void ThreadOpenConnections(); void ThreadMessageHandler(); @@ -507,8 +504,12 @@ public: double dPingTime; double dPingWait; double dMinPing; + // Our address, as reported by the peer std::string addrLocal; + // Address of this peer CAddress addr; + // Bind address of our side of the connection + CAddress addrBind; }; @@ -589,7 +590,10 @@ public: std::atomic<int64_t> nLastRecv; const int64_t nTimeConnected; std::atomic<int64_t> nTimeOffset; + // Address of this peer const CAddress addr; + // Bind address of our side of the connection + const CAddress addrBind; std::atomic<int> nVersion; // strSubVer is whatever byte array we read from the wire. However, this field is intended // to be printed out, displayed to humans in various forms and so on. So we sanitize it and @@ -615,7 +619,6 @@ public: CCriticalSection cs_filter; CBloomFilter* pfilter; std::atomic<int> nRefCount; - const NodeId id; const uint64_t nKeyedNetGroup; std::atomic_bool fPauseRecv; @@ -680,12 +683,13 @@ public: CAmount lastSentFeeFilter; int64_t nextSendTimeFeeFilter; - CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string &addrNameIn = "", bool fInboundIn = false); + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); private: CNode(const CNode&); void operator=(const CNode&); + const NodeId id; const uint64_t nLocalHostNonce; @@ -698,23 +702,24 @@ private: mutable CCriticalSection cs_addrName; std::string addrName; + // Our address, as reported by the peer CService addrLocal; mutable CCriticalSection cs_addrLocal; public: NodeId GetId() const { - return id; + return id; } uint64_t GetLocalNonce() const { - return nLocalHostNonce; + return nLocalHostNonce; } int GetMyStartingHeight() const { - return nMyStartingHeight; + return nMyStartingHeight; } - int GetRefCount() + int GetRefCount() const { assert(nRefCount >= 0); return nRefCount; @@ -726,7 +731,7 @@ public: { nRecvVersion = nVersionIn; } - int GetRecvVersion() + int GetRecvVersion() const { return nRecvVersion; } @@ -762,7 +767,7 @@ public: // after addresses were pushed. if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { - vAddrToSend[insecure_rand.rand32() % vAddrToSend.size()] = _addr; + vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { vAddrToSend.push_back(_addr); } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 3ec1a1c27d..4ca02c281d 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -168,6 +168,8 @@ struct CNodeState { int nUnconnectingHeaders; //! Whether we've started headers synchronization with this peer. bool fSyncStarted; + //! When to potentially disconnect peer for stalling headers download + int64_t nHeadersSyncTimeout; //! Since when we're stalling block download progress (in microseconds), or 0. int64_t nStallingSince; std::list<QueuedBlock> vBlocksInFlight; @@ -207,6 +209,7 @@ struct CNodeState { pindexBestHeaderSent = NULL; nUnconnectingHeaders = 0; fSyncStarted = false; + nHeadersSyncTimeout = 0; nStallingSince = 0; nDownloadingSince = 0; nBlocksInFlight = 0; @@ -256,10 +259,11 @@ void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes)); - if (fLogIPs) - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); - else - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); + if (fLogIPs) { + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + } else { + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); + } } void InitializeNode(CNode *pnode, CConnman& connman) { @@ -302,6 +306,7 @@ void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); } + LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } // Requires cs_main. @@ -332,7 +337,7 @@ bool MarkBlockAsReceived(const uint256& hash) { // Requires cs_main. // returns false, still setting pit, if the block was already in flight from the same peer // pit will only be valid as long as the same cs_main lock is being held -bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex* pindex = NULL, std::list<QueuedBlock>::iterator** pit = NULL) { +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const CBlockIndex* pindex = NULL, std::list<QueuedBlock>::iterator** pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); @@ -479,7 +484,7 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<con // Make sure pindexBestKnownBlock is up to date, we'll need it. ProcessBlockAvailability(nodeid); - if (state->pindexBestKnownBlock == NULL || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork) { + if (state->pindexBestKnownBlock == NULL || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < UintToArith256(consensusParams.nMinimumChainWork)) { // This peer has nothing interesting. return; } @@ -619,7 +624,7 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE unsigned int sz = GetTransactionWeight(*tx); if (sz >= MAX_STANDARD_TX_WEIGHT) { - LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); + LogPrint(BCLog::MEMPOOL, "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); return false; } @@ -631,7 +636,7 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE AddToCompactExtraTransactions(tx); - LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), + LogPrint(BCLog::MEMPOOL, "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; } @@ -666,7 +671,7 @@ void EraseOrphansFor(NodeId peer) nErased += EraseOrphanTx(maybeErase->second.tx->GetHash()); } } - if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer=%d\n", nErased, peer); + if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx from peer=%d\n", nErased, peer); } @@ -691,7 +696,7 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRE } // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; - if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx due to expiration\n", nErased); + if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx due to expiration\n", nErased); } while (mapOrphanTransactions.size() > nMaxOrphans) { @@ -743,21 +748,23 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanI recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); } -void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) { - if (nPosInBlock == CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK) - return; - +void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) { LOCK(cs_main); std::vector<uint256> vOrphanErase; - // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); - if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const CTransaction& orphanTx = *(*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); - vOrphanErase.push_back(orphanHash); + + for (const CTransactionRef& ptx : pblock->vtx) { + const CTransaction& tx = *ptx; + + // Which orphan pool entries must we evict? + for (const auto& txin : tx.vin) { + auto itByPrev = mapOrphanTransactionsByPrev.find(txin.prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = *(*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } } } @@ -767,14 +774,16 @@ void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIn BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { nErased += EraseOrphanTx(orphanHash); } - LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); + LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } } +// All of the following cache a recent block, and are protected by cs_most_recent_block static CCriticalSection cs_most_recent_block; static std::shared_ptr<const CBlock> most_recent_block; static std::shared_ptr<const CBlockHeaderAndShortTxIDs> most_recent_compact_block; static uint256 most_recent_block_hash; +static bool fWitnessesPresentInMostRecentCompactBlock; void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { std::shared_ptr<const CBlockHeaderAndShortTxIDs> pcmpctblock = std::make_shared<const CBlockHeaderAndShortTxIDs> (*pblock, true); @@ -795,6 +804,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: most_recent_block_hash = hashBlock; most_recent_block = pblock; most_recent_compact_block = pcmpctblock; + fWitnessesPresentInMostRecentCompactBlock = fWitnessEnabled; } connman->ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) { @@ -808,8 +818,8 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: if (state.fPreferHeaderAndIDs && (!fWitnessEnabled || state.fWantsCmpctWitness) && !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { - LogPrint("net", "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", - hashBlock.ToString(), pnode->id); + LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", + hashBlock.ToString(), pnode->GetId()); connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); state.pindexBestHeaderSent = pindex; } @@ -855,8 +865,8 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta int nDoS = 0; if (state.IsInvalid(nDoS)) { - if (it != mapBlockSource.end() && State(it->second.first)) { - assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + // Don't send reject message with code 0 or an internal reject code. + if (it != mapBlockSource.end() && State(it->second.first) && state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) { CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; State(it->second.first)->rejects.push_back(reject); if (nDoS > 0 && it->second.second) @@ -904,12 +914,11 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } - // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude - // requesting or processing some txs which have already been included in a block return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoinsInCache(inv.hash); + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: @@ -944,7 +953,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { if (pnode->nVersion >= CADDR_TIME_VERSION) { - uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); + uint64_t hashKey = CSipHasher(hasher).Write(pnode->GetId()).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { std::copy(best.begin() + i, best.begin() + nRelayNodes - 1, best.begin() + i + 1); @@ -987,6 +996,15 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam { bool send = false; BlockMap::iterator mi = mapBlockIndex.find(inv.hash); + std::shared_ptr<const CBlock> a_recent_block; + std::shared_ptr<const CBlockHeaderAndShortTxIDs> a_recent_compact_block; + bool fWitnessesPresentInARecentCompactBlock; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + a_recent_compact_block = most_recent_compact_block; + fWitnessesPresentInARecentCompactBlock = fWitnessesPresentInMostRecentCompactBlock; + } if (mi != mapBlockIndex.end()) { if (mi->second->nChainTx && !mi->second->IsValid(BLOCK_VALID_SCRIPTS) && @@ -996,11 +1014,6 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // before ActivateBestChain but after AcceptBlock). // In this case, we need to run ActivateBestChain prior to checking the relay // conditions below. - std::shared_ptr<const CBlock> a_recent_block; - { - LOCK(cs_most_recent_block); - a_recent_block = most_recent_block; - } CValidationState dummy; ActivateBestChain(dummy, Params(), a_recent_block); } @@ -1024,7 +1037,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam static const int nOneWeek = 7 * 24 * 60 * 60; // assume > 1 week = historical if (send && connman.OutboundTargetReached(true) && ( ((pindexBestHeader != NULL) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > nOneWeek)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) { - LogPrint("net", "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); //disconnect node pfrom->fDisconnect = true; @@ -1034,14 +1047,20 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // it's available before trying to send. if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) { - // Send block from disk - CBlock block; - if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) - assert(!"cannot load block from disk"); + std::shared_ptr<const CBlock> pblock; + if (a_recent_block && a_recent_block->GetHash() == (*mi).second->GetBlockHash()) { + pblock = a_recent_block; + } else { + // Send block from disk + std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); + if (!ReadBlockFromDisk(*pblockRead, (*mi).second, consensusParams)) + assert(!"cannot load block from disk"); + pblock = pblockRead; + } if (inv.type == MSG_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_WITNESS_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -1050,7 +1069,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam LOCK(pfrom->cs_filter); if (pfrom->pfilter) { sendMerkleBlock = true; - merkleBlock = CMerkleBlock(block, *pfrom->pfilter); + merkleBlock = CMerkleBlock(*pblock, *pfrom->pfilter); } } if (sendMerkleBlock) { @@ -1063,7 +1082,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *block.vtx[pair.first])); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1077,10 +1096,15 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { - CBlockHeaderAndShortTxIDs cmpctblock(block, fPeerWantsWitness); - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); - } else - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, block)); + if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == mi->second->GetBlockHash()) { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + } else { + CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + } + } else { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); + } } // Trigger the peer node to send a getblocks request for the next batch of inventory @@ -1141,7 +1165,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } -uint32_t GetFetchFlags(CNode* pfrom, const CBlockIndex* pprev, const Consensus::Params& chainparams) { +uint32_t GetFetchFlags(CNode* pfrom) { uint32_t nFetchFlags = 0; if ((pfrom->GetLocalServices() & NODE_WITNESS) && State(pfrom->GetId())->fHaveWitness) { nFetchFlags |= MSG_WITNESS_FLAG; @@ -1155,7 +1179,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac if (req.indexes[i] >= block.vtx.size()) { LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->GetId()); return; } resp.txn[i] = block.vtx[req.indexes[i]]; @@ -1168,7 +1192,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic<bool>& interruptMsgProc) { - LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); if (IsArgSet("-dropmessagestest") && GetRand(GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1192,7 +1216,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (strCommand == NetMsgType::REJECT) { - if (fDebug) { + if (LogAcceptCategory(BCLog::NET)) { try { std::string strMsg; unsigned char ccode; std::string strReason; vRecv >> LIMITED_STRING(strMsg, CMessageHeader::COMMAND_SIZE) >> ccode >> LIMITED_STRING(strReason, MAX_REJECT_MESSAGE_LENGTH); @@ -1206,10 +1230,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr vRecv >> hash; ss << ": hash " << hash.ToString(); } - LogPrint("net", "Reject %s\n", SanitizeString(ss.str())); + LogPrint(BCLog::NET, "Reject %s\n", SanitizeString(ss.str())); } catch (const std::ios_base::failure&) { // Avoid feedback loops by preventing reject messages from triggering a new reject message. - LogPrint("net", "Unparseable reject message received\n"); + LogPrint(BCLog::NET, "Unparseable reject message received\n"); } } } @@ -1247,7 +1271,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (pfrom->nServicesExpected & ~nServices) { - LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, nServices, pfrom->nServicesExpected); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, pfrom->nServicesExpected); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); pfrom->fDisconnect = true; @@ -1257,7 +1281,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, nVersion); + LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; @@ -1335,11 +1359,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr FastRandomContext insecure_rand; if (addr.IsRoutable()) { - LogPrint("net", "ProcessMessages: advertising address %s\n", addr.ToString()); + LogPrint(BCLog::NET, "ProcessMessages: advertising address %s\n", addr.ToString()); pfrom->PushAddress(addr, insecure_rand); } else if (IsPeerAddrLocalGood(pfrom)) { addr.SetIP(addrMe); - LogPrint("net", "ProcessMessages: advertising address %s\n", addr.ToString()); + LogPrint(BCLog::NET, "ProcessMessages: advertising address %s\n", addr.ToString()); pfrom->PushAddress(addr, insecure_rand); } } @@ -1359,7 +1383,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", cleanSubVer, pfrom->nVersion, - pfrom->nStartingHeight, addrMe.ToString(), pfrom->id, + pfrom->nStartingHeight, addrMe.ToString(), pfrom->GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); @@ -1529,19 +1553,15 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); - std::vector<CInv> vToFetch; - - for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) + for (CInv &inv : vInv) { - CInv &inv = vInv[nInv]; - if (interruptMsgProc) return true; bool fAlreadyHave = AlreadyHave(inv); - LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); + LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->GetId()); if (inv.type == MSG_TX) { inv.type |= nFetchFlags; @@ -1556,24 +1576,22 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we now only provide a getheaders response here. When we receive the headers, we will // then ask for the blocks we need. connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); - LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId()); } } else { pfrom->AddInventoryKnown(inv); - if (fBlocksOnly) - LogPrint("net", "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); - else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) + if (fBlocksOnly) { + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->GetId()); + } else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) { pfrom->AskFor(inv); + } } // Track requests for our stuff GetMainSignals().Inventory(inv.hash); } - - if (!vToFetch.empty()) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vToFetch)); } @@ -1588,11 +1606,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return error("message getdata size() = %u", vInv.size()); } - if (fDebug || (vInv.size() != 1)) - LogPrint("net", "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); + LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->GetId()); - if ((fDebug && vInv.size() > 0) || (vInv.size() == 1)) - LogPrint("net", "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); + if (vInv.size() > 0) { + LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->GetId()); + } pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); ProcessGetData(pfrom, chainparams.GetConsensus(), connman, interruptMsgProc); @@ -1631,12 +1649,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (pindex) pindex = chainActive.Next(pindex); int nLimit = 500; - LogPrint("net", "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); + LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->GetId()); for (; pindex; pindex = chainActive.Next(pindex)) { if (pindex->GetBlockHash() == hashStop) { - LogPrint("net", " getblocks stopping at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + LogPrint(BCLog::NET, " getblocks stopping at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } // If pruning, don't inv blocks unless we have on disk and are likely to still have @@ -1644,7 +1662,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / chainparams.GetConsensus().nPowTargetSpacing; if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= chainActive.Tip()->nHeight - nPrunedBlocksLikelyToHave)) { - LogPrint("net", " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); @@ -1652,7 +1670,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { // When this block is requested, we'll send an inv that'll // trigger the peer to getblocks the next batch of inventory. - LogPrint("net", " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + LogPrint(BCLog::NET, " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); pfrom->hashContinue = pindex->GetBlockHash(); break; } @@ -1681,7 +1699,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->GetId()); return true; } @@ -1693,7 +1711,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // might maliciously send lots of getblocktxn requests to trigger // expensive disk reads, because it will require the peer to // actually receive all the data read from disk over the network. - LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->GetId(), MAX_BLOCKTXN_DEPTH); CInv inv; inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; inv.hash = req.blockhash; @@ -1718,7 +1736,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); if (IsInitialBlockDownload() && !pfrom->fWhitelisted) { - LogPrint("net", "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->GetId()); return true; } @@ -1743,7 +1761,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end std::vector<CBlock> vHeaders; int nLimit = MAX_HEADERS_RESULTS; - LogPrint("net", "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->GetId()); for (; pindex; pindex = chainActive.Next(pindex)) { vHeaders.push_back(pindex->GetBlockHeader()); @@ -1773,7 +1791,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) { - LogPrint("net", "transaction sent in violation of protocol peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->GetId()); return true; } @@ -1805,8 +1823,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->nLastTXTime = GetTime(); - LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", - pfrom->id, + LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", + pfrom->GetId(), tx.GetHash().ToString(), mempool.size(), mempool.DynamicMemoryUsage() / 1000); @@ -1835,7 +1853,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (setMisbehaving.count(fromPeer)) continue; if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2, &lRemovedTxn)) { - LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); + LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanTx, connman); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { vWorkQueue.emplace_back(orphanHash, i); @@ -1850,11 +1868,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Punish peer that gave us an invalid orphan tx Misbehaving(fromPeer, nDos); setMisbehaving.insert(fromPeer); - LogPrint("mempool", " invalid orphan tx %s\n", orphanHash.ToString()); + LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); } // Has inputs but not accepted to mempool - // Probably non-standard or insufficient fee/priority - LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); + // Probably non-standard or insufficient fee + LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); vEraseQueue.push_back(orphanHash); if (!orphanTx.HasWitness() && !stateDummy.CorruptionPossible()) { // Do not use rejection cache for witness transactions or @@ -1881,7 +1899,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } if (!fRejectedParents) { - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); BOOST_FOREACH(const CTxIn& txin, tx.vin) { CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); pfrom->AddInventoryKnown(_inv); @@ -1892,10 +1910,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // DoS prevention: do not allow mapOrphanTransactions to grow unbounded unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); - if (nEvicted > 0) - LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + if (nEvicted > 0) { + LogPrint(BCLog::MEMPOOL, "mapOrphan overflow, removed %u tx\n", nEvicted); + } } else { - LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); + LogPrint(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); // We will continue to reject this tx since it has rejected // parents so avoid re-requesting it from other peers. recentRejects->insert(tx.GetHash()); @@ -1925,10 +1944,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // case. int nDoS = 0; if (!state.IsInvalid(nDoS) || nDoS == 0) { - LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); + LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); RelayTransaction(tx, connman); } else { - LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); + LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); } } } @@ -1939,10 +1958,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr int nDoS = 0; if (state.IsInvalid(nDoS)) { - LogPrint("mempoolrej", "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), - pfrom->id, + LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), + pfrom->GetId(), FormatStateMessage(state)); - if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P + if (state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); if (nDoS > 0) { @@ -1977,7 +1996,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); } - LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); return true; } } @@ -2017,7 +2036,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return true; @@ -2041,12 +2060,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { std::list<QueuedBlock>::iterator* queuedBlockIt = NULL; - if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); else { // The block was already in flight using compact blocks from the same peer - LogPrint("net", "Peer sent us compact block we were already syncing!\n"); + LogPrint(BCLog::NET, "Peer sent us compact block we were already syncing!\n"); return true; } } @@ -2056,12 +2075,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->GetId()); return true; } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } @@ -2104,7 +2123,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } else { @@ -2161,7 +2180,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || it->second.first != pfrom->GetId()) { - LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->GetId()); return true; } @@ -2170,12 +2189,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->GetId()); return true; } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received @@ -2254,11 +2273,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); - LogPrint("net", "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), pindexBestHeader->nHeight, - pfrom->id, nodestate->nUnconnectingHeaders); + pfrom->GetId(), nodestate->nUnconnectingHeaders); // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. @@ -2296,7 +2315,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); CNodeState *nodestate = State(pfrom->GetId()); if (nodestate->nUnconnectingHeaders > 0) { - LogPrint("net", "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); } nodestate->nUnconnectingHeaders = 0; @@ -2307,7 +2326,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. - LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); } @@ -2332,7 +2351,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // the main chain -- this shouldn't really happen. Bail out on the // direct fetch and rely on parallel download instead. if (!chainActive.Contains(pindexWalk)) { - LogPrint("net", "Large reorg, won't direct fetch to %s (%d)\n", + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } else { @@ -2343,14 +2362,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Can't download any more from this peer break; } - uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); - LogPrint("net", "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->id); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom->GetId()); } if (vGetData.size() > 1) { - LogPrint("net", "Downloading blocks toward %s (%d) via headers direct fetch\n", + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } if (vGetData.size() > 0) { @@ -2370,7 +2389,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); vRecv >> *pblock; - LogPrint("net", "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->id); + LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); // Process all blocks from whitelisted peers, even if not requested, // unless we're still syncing with the network. @@ -2402,14 +2421,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Making nodes which are behind NAT and can only make outgoing connections ignore // the getaddr message mitigates the attack. if (!pfrom->fInbound) { - LogPrint("net", "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->GetId()); return true; } // Only send one GetAddr response per connection to reduce resource waste // and discourage addr stamping of INV announcements. if (pfrom->fSentAddr) { - LogPrint("net", "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->GetId()); return true; } pfrom->fSentAddr = true; @@ -2426,14 +2445,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { if (!(pfrom->GetLocalServices() & NODE_BLOOM) && !pfrom->fWhitelisted) { - LogPrint("net", "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); pfrom->fDisconnect = true; return true; } if (connman.OutboundTargetReached(false) && !pfrom->fWhitelisted) { - LogPrint("net", "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); pfrom->fDisconnect = true; return true; } @@ -2509,8 +2528,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (!(sProblem.empty())) { - LogPrint("net", "pong peer=%d: %s, %x expected, %x received, %u bytes\n", - pfrom->id, + LogPrint(BCLog::NET, "pong peer=%d: %s, %x expected, %x received, %u bytes\n", + pfrom->GetId(), sProblem, pfrom->nPingNonceSent, nonce, @@ -2587,7 +2606,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(pfrom->cs_feeFilter); pfrom->minFeeFilter = newFeeFilter; } - LogPrint("net", "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); + LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->GetId()); } } @@ -2598,7 +2617,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr else { // Ignore unknown commands for extensibility - LogPrint("net", "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->GetId()); } @@ -2658,100 +2677,100 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic<bool>& i // this maintains the order of responses if (!pfrom->vRecvGetData.empty()) return true; - // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - return false; + // Don't bother if send buffer is too full to respond anyway + if (pfrom->fPauseSend) + return false; - std::list<CNetMessage> msgs; - { - LOCK(pfrom->cs_vProcessMsg); - if (pfrom->vProcessMsg.empty()) - return false; - // Just take one message - msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); - pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; - pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); - fMoreWork = !pfrom->vProcessMsg.empty(); - } - CNetMessage& msg(msgs.front()); - - msg.SetVersion(pfrom->GetRecvVersion()); - // Scan for message start - if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); - pfrom->fDisconnect = true; + std::list<CNetMessage> msgs; + { + LOCK(pfrom->cs_vProcessMsg); + if (pfrom->vProcessMsg.empty()) return false; - } + // Just take one message + msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); + pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; + pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); + fMoreWork = !pfrom->vProcessMsg.empty(); + } + CNetMessage& msg(msgs.front()); + + msg.SetVersion(pfrom->GetRecvVersion()); + // Scan for message start + if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { + LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); + pfrom->fDisconnect = true; + return false; + } - // Read header - CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid(chainparams.MessageStart())) - { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); - return fMoreWork; - } - std::string strCommand = hdr.GetCommand(); + // Read header + CMessageHeader& hdr = msg.hdr; + if (!hdr.IsValid(chainparams.MessageStart())) + { + LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); + return fMoreWork; + } + std::string strCommand = hdr.GetCommand(); - // Message size - unsigned int nMessageSize = hdr.nMessageSize; + // Message size + unsigned int nMessageSize = hdr.nMessageSize; - // Checksum - CDataStream& vRecv = msg.vRecv; - const uint256& hash = msg.GetMessageHash(); - if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + // Checksum + CDataStream& vRecv = msg.vRecv; + const uint256& hash = msg.GetMessageHash(); + if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + { + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, + SanitizeString(strCommand), nMessageSize, + HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), + HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + return fMoreWork; + } + + // Process message + bool fRet = false; + try + { + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); + if (interruptMsgProc) + return false; + if (!pfrom->vRecvGetData.empty()) + fMoreWork = true; + } + catch (const std::ios_base::failure& e) + { + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); + if (strstr(e.what(), "end of data")) { - LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, - SanitizeString(strCommand), nMessageSize, - HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), - HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); - return fMoreWork; + // Allow exceptions from under-length message on vRecv + LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - - // Process message - bool fRet = false; - try + else if (strstr(e.what(), "size too large")) { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); - if (interruptMsgProc) - return false; - if (!pfrom->vRecvGetData.empty()) - fMoreWork = true; + // Allow exceptions from over-long size + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - catch (const std::ios_base::failure& e) + else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); - if (strstr(e.what(), "end of data")) - { - // Allow exceptions from under-length message on vRecv - LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "size too large")) - { - // Allow exceptions from over-long size - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "non-canonical ReadCompactSize()")) - { - // Allow exceptions from non-canonical encoding - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else - { - PrintExceptionContinue(&e, "ProcessMessages()"); - } + // Allow exceptions from non-canonical encoding + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - catch (const std::exception& e) { + else + { PrintExceptionContinue(&e, "ProcessMessages()"); - } catch (...) { - PrintExceptionContinue(NULL, "ProcessMessages()"); } + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "ProcessMessages()"); + } catch (...) { + PrintExceptionContinue(NULL, "ProcessMessages()"); + } - if (!fRet) { - LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); - } + if (!fRet) { + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + } - LOCK(cs_main); - SendRejectsAndCheckIfBanned(pfrom, connman); + LOCK(cs_main); + SendRejectsAndCheckIfBanned(pfrom, connman); return fMoreWork; } @@ -2865,6 +2884,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { state.fSyncStarted = true; + state.nHeadersSyncTimeout = GetTimeMicros() + HEADERS_DOWNLOAD_TIMEOUT_BASE + HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER * (GetAdjustedTime() - pindexBestHeader->GetBlockTime())/(consensusParams.nPowTargetSpacing); nSyncStarted++; const CBlockIndex *pindexStart = pindexBestHeader; /* If possible, start at the block preceding the currently @@ -2876,7 +2896,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); } } @@ -2906,7 +2926,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); const CBlockIndex *pBestIndex = NULL; // last header queued for delivery - ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date + ProcessBlockAvailability(pto->GetId()); // ensure pindexBestKnownBlock is up-to-date if (!fRevertToInv) { bool fFoundStartingHeader = false; @@ -2960,8 +2980,8 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { // We only send up to 1 block as header-and-ids, as otherwise // probably means we're doing an initial-ish-sync or they're slow - LogPrint("net", "%s sending header-and-ids %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); + LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->GetId()); int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; @@ -2969,7 +2989,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr { LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { - if (state.fWantsCmpctWitness) + if (state.fWantsCmpctWitness || !fWitnessesPresentInMostRecentCompactBlock) connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); else { CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); @@ -2988,13 +3008,13 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { if (vHeaders.size() > 1) { - LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, + LogPrint(BCLog::NET, "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, vHeaders.size(), vHeaders.front().GetHash().ToString(), - vHeaders.back().GetHash().ToString(), pto->id); + vHeaders.back().GetHash().ToString(), pto->GetId()); } else { - LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); + LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->GetId()); } connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; @@ -3015,15 +3035,15 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // This should be very rare and could be optimized out. // Just log for now. if (chainActive[pindex->nHeight] != pindex) { - LogPrint("net", "Announcing block %s not on main chain (tip=%s)\n", + LogPrint(BCLog::NET, "Announcing block %s not on main chain (tip=%s)\n", hashToAnnounce.ToString(), chainActive.Tip()->GetBlockHash().ToString()); } // If the peer's chain has this block, don't inv it back. if (!PeerHasHeader(&state, pindex)) { pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); - LogPrint("net", "%s: sending inv peer=%d hash=%s\n", __func__, - pto->id, hashToAnnounce.ToString()); + LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, + pto->GetId(), hashToAnnounce.ToString()); } } } @@ -3170,7 +3190,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); + LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); pto->fDisconnect = true; return true; } @@ -3183,11 +3203,44 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); if (nNow > state.nDownloadingSince + consensusParams.nPowTargetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); + LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->GetId()); pto->fDisconnect = true; return true; } } + // Check for headers sync timeouts + if (state.fSyncStarted && state.nHeadersSyncTimeout < std::numeric_limits<int64_t>::max()) { + // Detect whether this is a stalling initial-headers-sync peer + if (pindexBestHeader->GetBlockTime() <= GetAdjustedTime() - 24*60*60) { + if (nNow > state.nHeadersSyncTimeout && nSyncStarted == 1 && (nPreferredDownload - state.fPreferredDownload >= 1)) { + // Disconnect a (non-whitelisted) peer if it is our only sync peer, + // and we have others we could be using instead. + // Note: If all our peers are inbound, then we won't + // disconnect our sync peer for stalling; we have bigger + // problems if we can't get any outbound peers. + if (!pto->fWhitelisted) { + LogPrintf("Timeout downloading headers from peer=%d, disconnecting\n", pto->GetId()); + pto->fDisconnect = true; + return true; + } else { + LogPrintf("Timeout downloading headers from whitelisted peer=%d, not disconnecting\n", pto->GetId()); + // Reset the headers sync state so that we have a + // chance to try downloading from a different peer. + // Note: this will also result in at least one more + // getheaders message to be sent to + // this peer (eventually). + state.fSyncStarted = false; + nSyncStarted--; + state.nHeadersSyncTimeout = 0; + } + } + } else { + // After we've caught up once, reset the timeout so we can't trigger + // disconnect later. + state.nHeadersSyncTimeout = std::numeric_limits<int64_t>::max(); + } + } + // // Message: getdata (blocks) @@ -3198,16 +3251,16 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); BOOST_FOREACH(const CBlockIndex *pindex, vToDownload) { - uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); + uint32_t nFetchFlags = GetFetchFlags(pto); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); - LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), - pindex->nHeight, pto->id); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), pindex); + LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), + pindex->nHeight, pto->GetId()); } if (state.nBlocksInFlight == 0 && staller != -1) { if (State(staller)->nStallingSince == 0) { State(staller)->nStallingSince = nNow; - LogPrint("net", "Stall started peer=%d\n", staller); + LogPrint(BCLog::NET, "Stall started peer=%d\n", staller); } } } @@ -3220,8 +3273,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr const CInv& inv = (*pto->mapAskFor.begin()).second; if (!AlreadyHave(inv)) { - if (fDebug) - LogPrint("net", "Requesting %s peer=%d\n", inv.ToString(), pto->id); + LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); vGetData.push_back(inv); if (vGetData.size() >= 1000) { @@ -3249,9 +3301,8 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr static CFeeRate default_feerate(DEFAULT_MIN_RELAY_TX_FEE); static FeeFilterRounder filterRounder(default_feerate); CAmount filterToSend = filterRounder.round(currentFilter); - // If we don't allow free transactions, then we always have a fee filter of at least minRelayTxFee - if (GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) <= 0) - filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); + // We always have a fee filter of at least minRelayTxFee + filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); if (filterToSend != pto->lastSentFeeFilter) { connman.PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); pto->lastSentFeeFilter = filterToSend; diff --git a/src/net_processing.h b/src/net_processing.h index 9e3f1b7156..db6d81e6b6 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -17,6 +17,10 @@ static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; /** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; +/** Headers download timeout expressed in microseconds + * Timeout = base + per_header * (expected number of headers) */ +static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes +static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/header /** Register with a network node to receive its signals */ void RegisterNodeSignals(CNodeSignals& nodeSignals); @@ -30,10 +34,10 @@ private: public: PeerLogicValidation(CConnman* connmanIn); - virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock); - virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); - virtual void BlockChecked(const CBlock& block, const CValidationState& state); - virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock); + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; + void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; + void BlockChecked(const CBlock& block, const CValidationState& state) override; + void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) override; }; struct CNodeStateStats { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index ab07270f3f..34a7029862 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -179,12 +179,6 @@ bool CNetAddr::IsLocal() const return false; } -bool CNetAddr::IsMulticast() const -{ - return (IsIPv4() && (GetByte(3) & 0xF0) == 0xE0) - || (GetByte(15) == 0xFF); -} - bool CNetAddr::IsValid() const { // Cleanup 3-byte shifted addresses caused by garbage in size field diff --git a/src/netaddress.h b/src/netaddress.h index a85c2b7452..fbc4d1a65f 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -65,7 +65,6 @@ class CNetAddr bool IsLocal() const; bool IsRoutable() const; bool IsValid() const; - bool IsMulticast() const; enum Network GetNetwork() const; std::string ToString() const; std::string ToStringIP() const; diff --git a/src/netbase.cpp b/src/netbase.cpp index fc9a6ed0be..32557dd179 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -25,7 +25,7 @@ #include <boost/algorithm/string/case_conv.hpp> // for to_lower() #include <boost/algorithm/string/predicate.hpp> // for startswith() and endswith() -#if !defined(HAVE_MSG_NOSIGNAL) && !defined(MSG_NOSIGNAL) +#if !defined(HAVE_MSG_NOSIGNAL) #define MSG_NOSIGNAL 0 #endif @@ -281,7 +281,7 @@ std::string Socks5ErrorString(int err) static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, SOCKET& hSocket) { IntrRecvError recvr; - LogPrint("net", "SOCKS5 connecting %s\n", strDest); + LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); if (strDest.size() > 255) { CloseSocket(hSocket); return error("Hostname too long"); @@ -327,7 +327,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials CloseSocket(hSocket); return error("Error sending authentication to proxy"); } - LogPrint("proxy", "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); + LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); char pchRetA[2]; if ((recvr = InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { CloseSocket(hSocket); @@ -409,7 +409,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials CloseSocket(hSocket); return error("Error reading from proxy"); } - LogPrint("net", "SOCKS5 connected %s\n", strDest); + LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); return true; } @@ -428,18 +428,14 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe if (hSocket == INVALID_SOCKET) return false; - int set = 1; #ifdef SO_NOSIGPIPE + int set = 1; // Different way of disabling SIGPIPE on BSD setsockopt(hSocket, SOL_SOCKET, SO_NOSIGPIPE, (void*)&set, sizeof(int)); #endif //Disable Nagle's algorithm -#ifdef WIN32 - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); -#else - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); -#endif + SetSocketNoDelay(hSocket); // Set to non-blocking if (!SetSocketNonBlocking(hSocket, true)) @@ -458,7 +454,7 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe int nRet = select(hSocket + 1, NULL, &fdset, NULL, &timeout); if (nRet == 0) { - LogPrint("net", "connection to %s timeout\n", addrConnect.ToString()); + LogPrint(BCLog::NET, "connection to %s timeout\n", addrConnect.ToString()); CloseSocket(hSocket); return false; } @@ -672,13 +668,14 @@ std::string NetworkErrorString(int err) std::string NetworkErrorString(int err) { char buf[256]; - const char *s = buf; buf[0] = 0; /* Too bad there are two incompatible implementations of the * thread-safe strerror. */ + const char *s; #ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ s = strerror_r(err, buf, sizeof(buf)); #else /* POSIX variant always returns message in buffer */ + s = buf; if (strerror_r(err, buf, sizeof(buf))) buf[0] = 0; #endif @@ -728,6 +725,13 @@ bool SetSocketNonBlocking(SOCKET& hSocket, bool fNonBlocking) return true; } +bool SetSocketNoDelay(SOCKET& hSocket) +{ + int set = 1; + int rc = setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); + return rc == 0; +} + void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index dd33b6e47e..c9d108aadd 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -59,6 +59,8 @@ std::string NetworkErrorString(int err); bool CloseSocket(SOCKET& hSocket); /** Disable or enable blocking-mode for a socket */ bool SetSocketNonBlocking(SOCKET& hSocket, bool fNonBlocking); +/** Set the TCP_NODELAY flag on a socket */ +bool SetSocketNoDelay(SOCKET& hSocket); /** * Convert milliseconds to a struct timeval for e.g. select. */ diff --git a/src/amount.cpp b/src/policy/feerate.cpp index a5f6bc3cd9..a089c02284 100644 --- a/src/amount.cpp +++ b/src/policy/feerate.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "amount.h" +#include "feerate.h" #include "tinyformat.h" diff --git a/src/policy/feerate.h b/src/policy/feerate.h new file mode 100644 index 0000000000..e82268b095 --- /dev/null +++ b/src/policy/feerate.h @@ -0,0 +1,54 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POLICY_FEERATE_H +#define BITCOIN_POLICY_FEERATE_H + +#include "amount.h" +#include "serialize.h" + +#include <string> + +extern const std::string CURRENCY_UNIT; + +/** + * Fee rate in satoshis per kilobyte: CAmount / kB + */ +class CFeeRate +{ +private: + CAmount nSatoshisPerK; // unit is satoshis-per-1,000-bytes +public: + /** Fee rate of 0 satoshis per kB */ + CFeeRate() : nSatoshisPerK(0) { } + explicit CFeeRate(const CAmount& _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } + /** Constructor for a fee rate in satoshis per kB. The size in bytes must not exceed (2^63 - 1)*/ + CFeeRate(const CAmount& nFeePaid, size_t nBytes); + CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } + /** + * Return the fee in satoshis for the given size in bytes. + */ + CAmount GetFee(size_t nBytes) const; + /** + * Return the fee in satoshis for a size of 1000 bytes + */ + CAmount GetFeePerK() const { return GetFee(1000); } + friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } + friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } + friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } + friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } + friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } + CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } + std::string ToString() const; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(nSatoshisPerK); + } +}; + +#endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 8f6a1e60f4..7a9af5edc2 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -7,46 +7,163 @@ #include "policy/policy.h" #include "amount.h" +#include "clientversion.h" #include "primitives/transaction.h" #include "random.h" #include "streams.h" #include "txmempool.h" #include "util.h" -void TxConfirmStats::Initialize(std::vector<double>& defaultBuckets, - unsigned int maxConfirms, double _decay) +static constexpr double INF_FEERATE = 1e99; + +/** + * We will instantiate an instance of this class to track transactions that were + * included in a block. We will lump transactions into a bucket according to their + * approximate feerate and then track how long it took for those txs to be included in a block + * + * The tracking of unconfirmed (mempool) transactions is completely independent of the + * historical tracking of transactions that have been confirmed in a block. + */ +class TxConfirmStats +{ +private: + //Define the buckets we will group transactions into + const std::vector<double>& buckets; // The upper-bound of the range for the bucket (inclusive) + const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + // For each bucket X: + // Count the total # of txs in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> txCtAvg; + + // Count the total # of txs confirmed within Y blocks in each bucket + // Track the historical moving average of theses totals over blocks + std::vector<std::vector<double>> confAvg; // confAvg[Y][X] + + // Track moving avg of txs which have been evicted from the mempool + // after failing to be confirmed within Y blocks + std::vector<std::vector<double>> failAvg; // failAvg[Y][X] + + // Sum the total feerate of all tx's in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> avg; + + // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X + // Combine the total value with the tx counts to calculate the avg feerate per bucket + + double decay; + + // Resolution (# of blocks) with which confirmations are tracked + unsigned int scale; + + // Mempool counts of outstanding transactions + // For each bucket X, track the number of transactions in the mempool + // that are unconfirmed for each possible confirmation value Y + std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] + // transactions still unconfirmed after GetMaxConfirms for each bucket + std::vector<int> oldUnconfTxs; + + void resizeInMemoryCounters(size_t newbuckets); + +public: + /** + * Create new TxConfirmStats. This is called by BlockPolicyEstimator's + * constructor with default values. + * @param defaultBuckets contains the upper limits for the bucket boundaries + * @param maxConfirms max number of confirms to track + * @param decay how much to decay the historical moving average per block + */ + TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap, + unsigned int maxPeriods, double decay, unsigned int scale); + + /** Roll the circular buffer for unconfirmed txs*/ + void ClearCurrent(unsigned int nBlockHeight); + + /** + * Record a new transaction data point in the current block stats + * @param blocksToConfirm the number of blocks it took this transaction to confirm + * @param val the feerate of the transaction + * @warning blocksToConfirm is 1-based and has to be >= 1 + */ + void Record(int blocksToConfirm, double val); + + /** Record a new transaction entering the mempool*/ + unsigned int NewTx(unsigned int nBlockHeight, double val); + + /** Remove a transaction from mempool tracking stats*/ + void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, + unsigned int bucketIndex, bool inBlock); + + /** Update our estimates by decaying our historical moving average and updating + with the data gathered from the current block */ + void UpdateMovingAverages(); + + /** + * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets + * to make sure we have enough data points) whose transactions still have sufficient likelihood + * of being confirmed within the target number of confirmations + * @param confTarget target number of confirmations + * @param sufficientTxVal required average number of transactions per block in a bucket range + * @param minSuccess the success probability we require + * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR + * return the highest feerate such that all lower values fail minSuccess + * @param nBlockHeight the current block height + */ + double EstimateMedianVal(int confTarget, double sufficientTxVal, + double minSuccess, bool requireGreater, unsigned int nBlockHeight, + EstimationResult *result = nullptr) const; + + /** Return the max number of confirms we're tracking */ + unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } + + /** Write state of estimation data to a file*/ + void Write(CAutoFile& fileout) const; + + /** + * Read saved state of estimation data from a file and replace all internal data structures and + * variables with this state. + */ + void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); +}; + + +TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets, + const std::map<double, unsigned int>& defaultBucketMap, + unsigned int maxPeriods, double _decay, unsigned int _scale) + : buckets(defaultBuckets), bucketMap(defaultBucketMap) { decay = _decay; - for (unsigned int i = 0; i < defaultBuckets.size(); i++) { - buckets.push_back(defaultBuckets[i]); - bucketMap[defaultBuckets[i]] = i; - } - confAvg.resize(maxConfirms); - curBlockConf.resize(maxConfirms); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { + scale = _scale; + confAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { confAvg[i].resize(buckets.size()); - curBlockConf[i].resize(buckets.size()); - unconfTxs[i].resize(buckets.size()); + } + failAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { + failAvg[i].resize(buckets.size()); } - oldUnconfTxs.resize(buckets.size()); - curBlockTxCt.resize(buckets.size()); txCtAvg.resize(buckets.size()); - curBlockVal.resize(buckets.size()); avg.resize(buckets.size()); + + resizeInMemoryCounters(buckets.size()); } -// Zero out the data for the current block +void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) { + // newbuckets must be passed in because the buckets referred to during Read have not been updated yet. + unconfTxs.resize(GetMaxConfirms()); + for (unsigned int i = 0; i < unconfTxs.size(); i++) { + unconfTxs[i].resize(newbuckets); + } + oldUnconfTxs.resize(newbuckets); +} + +// Roll the unconfirmed txs circular buffer void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) { for (unsigned int j = 0; j < buckets.size(); j++) { oldUnconfTxs[j] += unconfTxs[nBlockHeight%unconfTxs.size()][j]; unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0; - for (unsigned int i = 0; i < curBlockConf.size(); i++) - curBlockConf[i][j] = 0; - curBlockTxCt[j] = 0; - curBlockVal[j] = 0; } } @@ -56,33 +173,38 @@ void TxConfirmStats::Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; + int periodsToConfirm = (blocksToConfirm + scale - 1)/scale; unsigned int bucketindex = bucketMap.lower_bound(val)->second; - for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { - curBlockConf[i - 1][bucketindex]++; + for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) { + confAvg[i - 1][bucketindex]++; } - curBlockTxCt[bucketindex]++; - curBlockVal[bucketindex] += val; + txCtAvg[bucketindex]++; + avg[bucketindex] += val; } void TxConfirmStats::UpdateMovingAverages() { for (unsigned int j = 0; j < buckets.size(); j++) { for (unsigned int i = 0; i < confAvg.size(); i++) - confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; - avg[j] = avg[j] * decay + curBlockVal[j]; - txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; + confAvg[i][j] = confAvg[i][j] * decay; + for (unsigned int i = 0; i < failAvg.size(); i++) + failAvg[i][j] = failAvg[i][j] * decay; + avg[j] = avg[j] * decay; + txCtAvg[j] = txCtAvg[j] * decay; } } // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, - unsigned int nBlockHeight) + unsigned int nBlockHeight, EstimationResult *result) const { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget double totalNum = 0; // Total number of tx's that were ever confirmed int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget + int periodTarget = (confTarget + scale - 1)/scale; int maxbucketindex = buckets.size() - 1; @@ -105,12 +227,21 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, bool foundAnswer = false; unsigned int bins = unconfTxs.size(); + bool newBucketRange = true; + bool passing = true; + EstimatorBucket passBucket; + EstimatorBucket failBucket; // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + if (newBucketRange) { + curNearBucket = bucket; + newBucketRange = false; + } curFarBucket = bucket; - nConf += confAvg[confTarget - 1][bucket]; + nConf += confAvg[periodTarget - 1][bucket]; totalNum += txCtAvg[bucket]; + failNum += failAvg[periodTarget - 1][bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; extraNum += oldUnconfTxs[bucket]; @@ -119,24 +250,41 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - decay)) { - double curPct = nConf / (totalNum + extraNum); + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; - + if ((requireGreater && curPct < successBreakPoint) || (!requireGreater && curPct > successBreakPoint)) { + if (passing == true) { + // First time we hit a failure record the failed bucket + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + passing = false; + } + continue; + } // Otherwise update the cumulative stats, and the bucket variables // and reset the counters else { + failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing foundAnswer = true; + passing = true; + passBucket.withinTarget = nConf; nConf = 0; + passBucket.totalConfirmed = totalNum; totalNum = 0; + passBucket.inMempool = extraNum; + passBucket.leftMempool = failNum; + failNum = 0; extraNum = 0; bestNearBucket = curNearBucket; bestFarBucket = curFarBucket; - curNearBucket = bucket + step; + newBucketRange = true; } } } @@ -148,8 +296,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate - unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + unsigned int minBucket = std::min(bestNearBucket, bestFarBucket); + unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket); for (unsigned int j = minBucket; j <= maxBucket; j++) { txSum += txCtAvg[j]; } @@ -163,85 +311,111 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, break; } } + + passBucket.start = minBucket ? buckets[minBucket-1] : 0; + passBucket.end = buckets[maxBucket]; } - LogPrint("estimatefee", "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, - requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], - 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); + // If we were passing until we reached last few buckets with insufficient data, then report those as failed + if (passing && !newBucketRange) { + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + } + LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d %s%.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", + confTarget, requireGreater ? ">" : "<", 100.0 * successBreakPoint, decay, + median, passBucket.start, passBucket.end, + 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool), + passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool, + failBucket.start, failBucket.end, + 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool), + failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool); + + + if (result) { + result->pass = passBucket; + result->fail = failBucket; + result->decay = decay; + result->scale = scale; + } return median; } -void TxConfirmStats::Write(CAutoFile& fileout) +void TxConfirmStats::Write(CAutoFile& fileout) const { fileout << decay; - fileout << buckets; + fileout << scale; fileout << avg; fileout << txCtAvg; fileout << confAvg; + fileout << failAvg; } -void TxConfirmStats::Read(CAutoFile& filein) +void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) { - // Read data file into temporary variables and do some very basic sanity checking - std::vector<double> fileBuckets; - std::vector<double> fileAvg; - std::vector<std::vector<double> > fileConfAvg; - std::vector<double> fileTxCtAvg; - double fileDecay; - size_t maxConfirms; - size_t numBuckets; - - filein >> fileDecay; - if (fileDecay <= 0 || fileDecay >= 1) - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - filein >> fileBuckets; - numBuckets = fileBuckets.size(); - if (numBuckets <= 1 || numBuckets > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - filein >> fileAvg; - if (fileAvg.size() != numBuckets) + // Read data file and do some very basic sanity checking + // buckets and bucketMap are not updated yet, so don't access them + // If there is a read failure, we'll just discard this entire object anyway + size_t maxConfirms, maxPeriods; + + // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor + if (nFileVersion >= 149900) { + filein >> decay; + if (decay <= 0 || decay >= 1) { + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + filein >> scale; + } + + filein >> avg; + if (avg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); - filein >> fileTxCtAvg; - if (fileTxCtAvg.size() != numBuckets) + } + filein >> txCtAvg; + if (txCtAvg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); - filein >> fileConfAvg; - maxConfirms = fileConfAvg.size(); - if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) // one week - throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); - for (unsigned int i = 0; i < maxConfirms; i++) { - if (fileConfAvg[i].size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); } - // Now that we've processed the entire feerate estimate data file and not - // thrown any errors, we can copy it to our data structures - decay = fileDecay; - buckets = fileBuckets; - avg = fileAvg; - confAvg = fileConfAvg; - txCtAvg = fileTxCtAvg; - bucketMap.clear(); + filein >> confAvg; + maxPeriods = confAvg.size(); + maxConfirms = scale * maxPeriods; - // Resize the current block variables which aren't stored in the data file - // to match the number of confirms and buckets - curBlockConf.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - curBlockConf[i].resize(buckets.size()); + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week + throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (confAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); + } } - curBlockTxCt.resize(buckets.size()); - curBlockVal.resize(buckets.size()); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - unconfTxs[i].resize(buckets.size()); + if (nFileVersion >= 149900) { + filein >> failAvg; + if (maxPeriods != failAvg.size()) { + throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (failAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); + } + } + } else { + failAvg.resize(confAvg.size()); + for (unsigned int i = 0; i < failAvg.size(); i++) { + failAvg[i].resize(numBuckets); + } } - oldUnconfTxs.resize(buckets.size()); - for (unsigned int i = 0; i < buckets.size(); i++) - bucketMap[buckets[i]] = i; + // Resize the current block variables which aren't stored in the data file + // to match the number of confirms and buckets + resizeInMemoryCounters(numBuckets); - LogPrint("estimatefee", "Reading estimates: %u buckets counting confirms up to %u blocks\n", + LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n", numBuckets, maxConfirms); } @@ -253,31 +427,39 @@ unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) return bucketindex; } -void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) +void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock) { //nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet blocksAgo = 0; if (blocksAgo < 0) { - LogPrint("estimatefee", "Blockpolicy error, blocks ago is negative for mempool tx\n"); + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n"); return; //This can't happen because we call this with our best seen height, no entries can have higher } if (blocksAgo >= (int)unconfTxs.size()) { - if (oldUnconfTxs[bucketindex] > 0) + if (oldUnconfTxs[bucketindex] > 0) { oldUnconfTxs[bucketindex]--; - else - LogPrint("estimatefee", "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n", + } else { + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n", bucketindex); + } } else { unsigned int blockIndex = entryHeight % unconfTxs.size(); - if (unconfTxs[blockIndex][bucketindex] > 0) + if (unconfTxs[blockIndex][bucketindex] > 0) { unconfTxs[blockIndex][bucketindex]--; - else - LogPrint("estimatefee", "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n", + } else { + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n", blockIndex, bucketindex); + } + } + if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period + unsigned int periodsAgo = blocksAgo / scale; + for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) { + failAvg[i][bucketindex]++; + } } } @@ -286,11 +468,14 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe // tracked. Txs that were part of a block have already been removed in // processBlockTx to ensure they are never double tracked, but it is // of no harm to try to remove them again. -bool CBlockPolicyEstimator::removeTx(uint256 hash) +bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock) { + LOCK(cs_feeEstimator); std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { - feeStats.removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); + feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); mapMemPoolTxs.erase(hash); return true; } else { @@ -298,25 +483,38 @@ bool CBlockPolicyEstimator::removeTx(uint256 hash) } } -CBlockPolicyEstimator::CBlockPolicyEstimator(const CFeeRate& _minRelayFee) - : nBestSeenHeight(0), trackedTxs(0), untrackedTxs(0) +CBlockPolicyEstimator::CBlockPolicyEstimator() + : nBestSeenHeight(0), firstRecordedHeight(0), historicalFirst(0), historicalBest(0), trackedTxs(0), untrackedTxs(0) +{ + static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); + size_t bucketIndex = 0; + for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) { + buckets.push_back(bucketBoundary); + bucketMap[bucketBoundary] = bucketIndex; + } + buckets.push_back(INF_FEERATE); + bucketMap[INF_FEERATE] = bucketIndex; + assert(bucketMap.size() == buckets.size()); + + feeStats = new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE); + shortStats = new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE); + longStats = new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE); +} + +CBlockPolicyEstimator::~CBlockPolicyEstimator() { - static_assert(MIN_FEERATE > 0, "Min feerate must be nonzero"); - minTrackedFee = _minRelayFee < CFeeRate(MIN_FEERATE) ? CFeeRate(MIN_FEERATE) : _minRelayFee; - std::vector<double> vfeelist; - for (double bucketBoundary = minTrackedFee.GetFeePerK(); bucketBoundary <= MAX_FEERATE; bucketBoundary *= FEE_SPACING) { - vfeelist.push_back(bucketBoundary); - } - vfeelist.push_back(INF_FEERATE); - feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); + delete feeStats; + delete shortStats; + delete longStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) { + LOCK(cs_feeEstimator); unsigned int txHeight = entry.GetHeight(); uint256 hash = entry.GetTx().GetHash(); if (mapMemPoolTxs.count(hash)) { - LogPrint("estimatefee", "Blockpolicy error mempool tx %s already being tracked\n", + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", hash.ToString().c_str()); return; } @@ -341,12 +539,17 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[hash].blockHeight = txHeight; - mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + mapMemPoolTxs[hash].bucketIndex = bucketIndex; + unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex2); + unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex3); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) { - if (!removeTx(entry->GetTx().GetHash())) { + if (!removeTx(entry->GetTx().GetHash(), true)) { // This transaction wasn't being tracked for fee estimation return false; } @@ -358,20 +561,23 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM if (blocksToConfirm <= 0) { // This can't happen because we don't process transactions from a block with a height // lower than our greatest seen height - LogPrint("estimatefee", "Blockpolicy error Transaction had negative blocksToConfirm\n"); + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n"); return false; } // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); - feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); return true; } void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, std::vector<const CTxMemPoolEntry*>& entries) { + LOCK(cs_feeEstimator); if (nBlockHeight <= nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random // they don't affect the estimate. @@ -386,34 +592,77 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, // of unconfirmed txs to remove from tracking. nBestSeenHeight = nBlockHeight; - // Clear the current block state and update unconfirmed circular buffer - feeStats.ClearCurrent(nBlockHeight); + // Update unconfirmed circular buffer + feeStats->ClearCurrent(nBlockHeight); + shortStats->ClearCurrent(nBlockHeight); + longStats->ClearCurrent(nBlockHeight); + + // Decay all exponential averages + feeStats->UpdateMovingAverages(); + shortStats->UpdateMovingAverages(); + longStats->UpdateMovingAverages(); unsigned int countedTxs = 0; - // Repopulate the current block states - for (unsigned int i = 0; i < entries.size(); i++) { - if (processBlockTx(nBlockHeight, entries[i])) + // Update averages with data points from current block + for (const auto& entry : entries) { + if (processBlockTx(nBlockHeight, entry)) countedTxs++; } - // Update all exponential averages with the current block state - feeStats.UpdateMovingAverages(); + if (firstRecordedHeight == 0 && countedTxs > 0) { + firstRecordedHeight = nBestSeenHeight; + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight); + } + - LogPrint("estimatefee", "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", - countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n", + countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(), + MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current"); trackedTxs = 0; untrackedTxs = 0; } -CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) +CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const { - // Return failure if trying to analyze a target we're not tracking // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) + if (confTarget <= 1) + return CFeeRate(0); + + return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE); +} + +CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const +{ + TxConfirmStats* stats; + double sufficientTxs = SUFFICIENT_FEETXS; + switch (horizon) { + case FeeEstimateHorizon::SHORT_HALFLIFE: { + stats = shortStats; + sufficientTxs = SUFFICIENT_TXS_SHORT; + break; + } + case FeeEstimateHorizon::MED_HALFLIFE: { + stats = feeStats; + break; + } + case FeeEstimateHorizon::LONG_HALFLIFE: { + stats = longStats; + break; + } + default: { + return CFeeRate(0); + } + } + + LOCK(cs_feeEstimator); + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms()) + return CFeeRate(0); + if (successThreshold > 1) return CFeeRate(0); - double median = feeStats.EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, nBestSeenHeight, result); if (median < 0) return CFeeRate(0); @@ -421,25 +670,148 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) return CFeeRate(median); } -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) +unsigned int CBlockPolicyEstimator::BlockSpan() const +{ + if (firstRecordedHeight == 0) return 0; + assert(nBestSeenHeight >= firstRecordedHeight); + + return nBestSeenHeight - firstRecordedHeight; +} + +unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const +{ + if (historicalFirst == 0) return 0; + assert(historicalBest >= historicalFirst); + + if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0; + + return historicalBest - historicalFirst; +} + +unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const +{ + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2); +} + +/** Return a fee estimate at the required successThreshold from the shortest + * time horizon which tracks confirmations up to the desired target. If + * checkShorterHorizon is requested, also allow short time horizon estimates + * for a lower target to reduce the given answer */ +double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const +{ + double estimate = -1; + if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) { + // Find estimate from shortest time horizon possible + if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon + estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + } + else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon + estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + else { // long horizon + estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + if (checkShorterHorizon) { + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > feeStats->GetMaxConfirms()) { + double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + estimate = medMax; + } + if (confTarget > shortStats->GetMaxConfirms()) { + double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + estimate = shortMax; + } + } + } + return estimate; +} + +/** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + * at 2 * target for any longer time horizons. + */ +double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget) const +{ + double estimate = -1; + if (doubleTarget <= shortStats->GetMaxConfirms()) { + estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + } + if (doubleTarget <= feeStats->GetMaxConfirms()) { + double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + if (longEstimate > estimate) { + estimate = longEstimate; + } + } + return estimate; +} + +/** estimateSmartFee returns the max of the feerates calculated with a 60% + * threshold required at target / 2, an 85% threshold required at target and a + * 95% threshold required at 2 * target. Each calculation is performed at the + * shortest time horizon which tracks the required target. Conservative + * estimates, however, required the 95% threshold at 2 * target be met for any + * longer time horizons also. + */ +CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative) const { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) - return CFeeRate(0); - - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) - confTarget = 2; double median = -1; - while (median < 0 && (unsigned int)confTarget <= feeStats.GetMaxConfirms()) { - median = feeStats.EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); - } + { + LOCK(cs_feeEstimator); + + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) + return CFeeRate(0); + + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + confTarget = 2; + + unsigned int maxUsableEstimate = MaxUsableEstimate(); + if (maxUsableEstimate <= 1) + return CFeeRate(0); + + if ((unsigned int)confTarget > maxUsableEstimate) { + confTarget = maxUsableEstimate; + } + + assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints + + /** true is passed to estimateCombined fee for target/2 and target so + * that we check the max confirms for shorter time horizons as well. + * This is necessary to preserve monotonically increasing estimates. + * For non-conservative estimates we do the same thing for 2*target, but + * for conservative estimates we want to skip these shorter horizons + * checks for 2*target because we are taking the max over all time + * horizons so we already have monotonically increasing estimates and + * the purpose of conservative estimates is not to let short term + * fluctuations lower our estimates by too much. + */ + double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true); + double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true); + double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative); + median = halfEst; + if (actualEst > median) { + median = actualEst; + } + if (doubleEst > median) { + median = doubleEst; + } + + if (conservative || median == -1) { + double consEst = estimateConservativeFee(2 * confTarget); + if (consEst > median) { + median = consEst; + } + } + } // Must unlock cs_feeEstimator before taking mempool locks if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget - 1; + *answerFoundAtTarget = confTarget; // If mempool is limiting txs , return at least the min feerate from the mempool CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); @@ -452,44 +824,134 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun return CFeeRate(median); } -double CBlockPolicyEstimator::estimatePriority(int confTarget) -{ - return -1; -} -double CBlockPolicyEstimator::estimateSmartPriority(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) +bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { - if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget; - - // If mempool is limiting txs, no priority txs are allowed - CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); - if (minPoolFee > 0) - return INF_PRIORITY; - - return -1; + try { + LOCK(cs_feeEstimator); + fileout << 149900; // version required to read: 0.14.99 or later + fileout << CLIENT_VERSION; // version that wrote the file + fileout << nBestSeenHeight; + if (BlockSpan() > HistoricalBlockSpan()/2) { + fileout << firstRecordedHeight << nBestSeenHeight; + } + else { + fileout << historicalFirst << historicalBest; + } + fileout << buckets; + feeStats->Write(fileout); + shortStats->Write(fileout); + longStats->Write(fileout); + } + catch (const std::exception&) { + LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n"); + return false; + } + return true; } -void CBlockPolicyEstimator::Write(CAutoFile& fileout) +bool CBlockPolicyEstimator::Read(CAutoFile& filein) { - fileout << nBestSeenHeight; - feeStats.Write(fileout); + try { + LOCK(cs_feeEstimator); + int nVersionRequired, nVersionThatWrote; + unsigned int nFileBestSeenHeight, nFileHistoricalFirst, nFileHistoricalBest; + filein >> nVersionRequired >> nVersionThatWrote; + if (nVersionRequired > CLIENT_VERSION) + return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + + // Read fee estimates file into temporary variables so existing data + // structures aren't corrupted if there is an exception. + filein >> nFileBestSeenHeight; + + if (nVersionThatWrote < 149900) { + // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. + // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor + double tempDecay; + filein >> tempDecay; + if (tempDecay <= 0 || tempDecay >= 1) + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + + std::vector<double> tempBuckets; + filein >> tempBuckets; + size_t tempNum = tempBuckets.size(); + if (tempNum <= 1 || tempNum > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::map<double, unsigned int> tempMap; + + std::unique_ptr<TxConfirmStats> tempFeeStats(new TxConfirmStats(tempBuckets, tempMap, MED_BLOCK_PERIODS, tempDecay, 1)); + tempFeeStats->Read(filein, nVersionThatWrote, tempNum); + // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + + tempMap.clear(); + for (unsigned int i = 0; i < tempBuckets.size(); i++) { + tempMap[tempBuckets[i]] = i; + } + } + else { // nVersionThatWrote >= 149900 + filein >> nFileHistoricalFirst >> nFileHistoricalBest; + if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { + throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid"); + } + std::vector<double> fileBuckets; + filein >> fileBuckets; + size_t numBuckets = fileBuckets.size(); + if (numBuckets <= 1 || numBuckets > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); + std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); + std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); + fileFeeStats->Read(filein, nVersionThatWrote, numBuckets); + fileShortStats->Read(filein, nVersionThatWrote, numBuckets); + fileLongStats->Read(filein, nVersionThatWrote, numBuckets); + + // Fee estimates file parsed correctly + // Copy buckets from file and refresh our bucketmap + buckets = fileBuckets; + bucketMap.clear(); + for (unsigned int i = 0; i < buckets.size(); i++) { + bucketMap[buckets[i]] = i; + } + + // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap + delete feeStats; + delete shortStats; + delete longStats; + feeStats = fileFeeStats.release(); + shortStats = fileShortStats.release(); + longStats = fileLongStats.release(); + + nBestSeenHeight = nFileBestSeenHeight; + historicalFirst = nFileHistoricalFirst; + historicalBest = nFileHistoricalBest; + } + } + catch (const std::exception& e) { + LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what()); + return false; + } + return true; } -void CBlockPolicyEstimator::Read(CAutoFile& filein, int nFileVersion) -{ - int nFileBestSeenHeight; - filein >> nFileBestSeenHeight; - feeStats.Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. +void CBlockPolicyEstimator::FlushUnconfirmed(CTxMemPool& pool) { + int64_t startclear = GetTimeMicros(); + std::vector<uint256> txids; + pool.queryHashes(txids); + LOCK(cs_feeEstimator); + for (auto& txid : txids) { + removeTx(txid, false); + } + int64_t endclear = GetTimeMicros(); + LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n",txids.size(), (endclear - startclear)*0.000001); } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) { CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FEERATE; bucketBoundary *= FEE_SPACING) { + for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { feeset.insert(bucketBoundary); } } diff --git a/src/policy/fees.h b/src/policy/fees.h index 064466afe4..e99fec2c39 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -6,8 +6,10 @@ #define BITCOIN_POLICYESTIMATOR_H #include "amount.h" +#include "feerate.h" #include "uint256.h" #include "random.h" +#include "sync.h" #include <map> #include <string> @@ -17,6 +19,7 @@ class CAutoFile; class CFeeRate; class CTxMemPoolEntry; class CTxMemPool; +class TxConfirmStats; /** \class CBlockPolicyEstimator * The BlockPolicyEstimator is used for estimating the feerate needed @@ -39,157 +42,58 @@ class CTxMemPool; * within your desired 5 blocks. * * Here is a brief description of the implementation: - * When a transaction enters the mempool, we - * track the height of the block chain at entry. Whenever a block comes in, - * we count the number of transactions in each bucket and the total amount of feerate - * paid in each bucket. Then we calculate how many blocks Y it took each - * transaction to be mined and we track an array of counters in each bucket - * for how long it to took transactions to get confirmed from 1 to a max of 25 - * and we increment all the counters from Y up to 25. This is because for any - * number Z>=Y the transaction was successfully mined within Z blocks. We - * want to save a history of this information, so at any time we have a - * counter of the total number of transactions that happened in a given feerate - * bucket and the total number that were confirmed in each number 1-25 blocks - * or less for any bucket. We save this history by keeping an exponentially - * decaying moving average of each one of these stats. Furthermore we also - * keep track of the number unmined (in mempool) transactions in each bucket - * and for how many blocks they have been outstanding and use that to increase - * the number of transactions we've seen in that feerate bucket when calculating - * an estimate for any number of confirmations below the number of blocks - * they've been outstanding. + * When a transaction enters the mempool, we track the height of the block chain + * at entry. All further calculations are conducted only on this set of "seen" + * transactions. Whenever a block comes in, we count the number of transactions + * in each bucket and the total amount of feerate paid in each bucket. Then we + * calculate how many blocks Y it took each transaction to be mined. We convert + * from a number of blocks to a number of periods Y' each encompassing "scale" + * blocks. This is is tracked in 3 different data sets each up to a maximum + * number of periods. Within each data set we have an array of counters in each + * feerate bucket and we increment all the counters from Y' up to max periods + * representing that a tx was successfully confirmed in less than or equal to + * that many periods. We want to save a history of this information, so at any + * time we have a counter of the total number of transactions that happened in a + * given feerate bucket and the total number that were confirmed in each of the + * periods or less for any bucket. We save this history by keeping an + * exponentially decaying moving average of each one of these stats. This is + * done for a different decay in each of the 3 data sets to keep relevant data + * from different time horizons. Furthermore we also keep track of the number + * unmined (in mempool or left mempool without being included in a block) + * transactions in each bucket and for how many blocks they have been + * outstanding and use both of these numbers to increase the number of transactions + * we've seen in that feerate bucket when calculating an estimate for any number + * of confirmations below the number of blocks they've been outstanding. */ -/** - * We will instantiate an instance of this class to track transactions that were - * included in a block. We will lump transactions into a bucket according to their - * approximate feerate and then track how long it took for those txs to be included in a block - * - * The tracking of unconfirmed (mempool) transactions is completely independent of the - * historical tracking of transactions that have been confirmed in a block. - */ -class TxConfirmStats -{ -private: - //Define the buckets we will group transactions into - std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) - std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket - - // For each bucket X: - // Count the total # of txs in each bucket - // Track the historical moving average of this total over blocks - std::vector<double> txCtAvg; - // and calculate the total for the current block to update the moving average - std::vector<int> curBlockTxCt; - - // Count the total # of txs confirmed within Y blocks in each bucket - // Track the historical moving average of theses totals over blocks - std::vector<std::vector<double> > confAvg; // confAvg[Y][X] - // and calculate the totals for the current block to update the moving averages - std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] - - // Sum the total feerate of all tx's in each bucket - // Track the historical moving average of this total over blocks - std::vector<double> avg; - // and calculate the total for the current block to update the moving average - std::vector<double> curBlockVal; +/* Identifier for each of the 3 different TxConfirmStats which will track + * history over different time horizons. */ +enum FeeEstimateHorizon { + SHORT_HALFLIFE = 0, + MED_HALFLIFE = 1, + LONG_HALFLIFE = 2 +}; - // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X - // Combine the total value with the tx counts to calculate the avg feerate per bucket +/* Used to return detailed information about a feerate bucket */ +struct EstimatorBucket +{ + double start = -1; + double end = -1; + double withinTarget = 0; + double totalConfirmed = 0; + double inMempool = 0; + double leftMempool = 0; +}; +/* Used to return detailed information about a fee estimate calculation */ +struct EstimationResult +{ + EstimatorBucket pass; + EstimatorBucket fail; double decay; - - // Mempool counts of outstanding transactions - // For each bucket X, track the number of transactions in the mempool - // that are unconfirmed for each possible confirmation value Y - std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] - // transactions still unconfirmed after MAX_CONFIRMS for each bucket - std::vector<int> oldUnconfTxs; - -public: - /** - * Initialize the data structures. This is called by BlockPolicyEstimator's - * constructor with default values. - * @param defaultBuckets contains the upper limits for the bucket boundaries - * @param maxConfirms max number of confirms to track - * @param decay how much to decay the historical moving average per block - */ - void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay); - - /** Clear the state of the curBlock variables to start counting for the new block */ - void ClearCurrent(unsigned int nBlockHeight); - - /** - * Record a new transaction data point in the current block stats - * @param blocksToConfirm the number of blocks it took this transaction to confirm - * @param val the feerate of the transaction - * @warning blocksToConfirm is 1-based and has to be >= 1 - */ - void Record(int blocksToConfirm, double val); - - /** Record a new transaction entering the mempool*/ - unsigned int NewTx(unsigned int nBlockHeight, double val); - - /** Remove a transaction from mempool tracking stats*/ - void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); - - /** Update our estimates by decaying our historical moving average and updating - with the data gathered from the current block */ - void UpdateMovingAverages(); - - /** - * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets - * to make sure we have enough data points) whose transactions still have sufficient likelihood - * of being confirmed within the target number of confirmations - * @param confTarget target number of confirmations - * @param sufficientTxVal required average number of transactions per block in a bucket range - * @param minSuccess the success probability we require - * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR - * return the highest feerate such that all lower values fail minSuccess - * @param nBlockHeight the current block height - */ - double EstimateMedianVal(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, unsigned int nBlockHeight); - - /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() { return confAvg.size(); } - - /** Write state of estimation data to a file*/ - void Write(CAutoFile& fileout); - - /** - * Read saved state of estimation data from a file and replace all internal data structures and - * variables with this state. - */ - void Read(CAutoFile& filein); + unsigned int scale; }; - - -/** Track confirm delays up to 25 blocks, can't estimate beyond that */ -static const unsigned int MAX_BLOCK_CONFIRMS = 25; - -/** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ -static const double DEFAULT_DECAY = .998; - -/** Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough */ -static const double MIN_SUCCESS_PCT = .95; - -/** Require an avg of 1 tx in the combined feerate bucket per block to have stat significance */ -static const double SUFFICIENT_FEETXS = 1; - -// Minimum and Maximum values for tracking feerates -static constexpr double MIN_FEERATE = 10; -static const double MAX_FEERATE = 1e7; -static const double INF_FEERATE = MAX_MONEY; -static const double INF_PRIORITY = 1e9 * MAX_MONEY; - -// We have to lump transactions into buckets based on feerate, but we want to be able -// to give accurate estimates over a large range of potential feerates -// Therefore it makes sense to exponentially space the buckets -/** Spacing of FeeRate buckets */ -static const double FEE_SPACING = 1.1; - /** * We want to be able to estimate feerates that are needed on tx's to be included in * a certain number of blocks. Every time a block is added to the best chain, this class records @@ -197,55 +101,101 @@ static const double FEE_SPACING = 1.1; */ class CBlockPolicyEstimator { +private: + /** Track confirm delays up to 12 blocks for short horizon */ + static constexpr unsigned int SHORT_BLOCK_PERIODS = 12; + static constexpr unsigned int SHORT_SCALE = 1; + /** Track confirm delays up to 48 blocks for medium horizon */ + static constexpr unsigned int MED_BLOCK_PERIODS = 24; + static constexpr unsigned int MED_SCALE = 2; + /** Track confirm delays up to 1008 blocks for long horizon */ + static constexpr unsigned int LONG_BLOCK_PERIODS = 42; + static constexpr unsigned int LONG_SCALE = 24; + /** Historical estimates that are older than this aren't valid */ + static const unsigned int OLDEST_ESTIMATE_HISTORY = 6 * 1008; + + /** Decay of .962 is a half-life of 18 blocks or about 3 hours */ + static constexpr double SHORT_DECAY = .962; + /** Decay of .998 is a half-life of 144 blocks or about 1 day */ + static constexpr double MED_DECAY = .9952; + /** Decay of .9995 is a half-life of 1008 blocks or about 1 week */ + static constexpr double LONG_DECAY = .99931; + + /** Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks*/ + static constexpr double HALF_SUCCESS_PCT = .6; + /** Require greater than 85% of X feerate transactions to be confirmed within Y blocks*/ + static constexpr double SUCCESS_PCT = .85; + /** Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks*/ + static constexpr double DOUBLE_SUCCESS_PCT = .95; + + /** Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance */ + static constexpr double SUFFICIENT_FEETXS = 0.1; + /** Require an avg of 0.5 tx when using short decay since there are fewer blocks considered*/ + static constexpr double SUFFICIENT_TXS_SHORT = 0.5; + + /** Minimum and Maximum values for tracking feerates + * The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we + * might ever want to track. Historically this has been 1000 since it was + * inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it + * invalidates old estimates files. So leave it at 1000 unless it becomes + * necessary to lower it, and then lower it substantially. + */ + static constexpr double MIN_BUCKET_FEERATE = 1000; + static constexpr double MAX_BUCKET_FEERATE = 1e7; + + /** Spacing of FeeRate buckets + * We have to lump transactions into buckets based on feerate, but we want to be able + * to give accurate estimates over a large range of potential feerates + * Therefore it makes sense to exponentially space the buckets + */ + static constexpr double FEE_SPACING = 1.05; + public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ - CBlockPolicyEstimator(const CFeeRate& minRelayFee); + CBlockPolicyEstimator(); + ~CBlockPolicyEstimator(); /** Process all the transactions that have been included in a block */ void processBlock(unsigned int nBlockHeight, std::vector<const CTxMemPoolEntry*>& entries); - /** Process a transaction confirmed in a block*/ - bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); - /** Process a transaction accepted to the mempool*/ void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash); - - /** Return a feerate estimate */ - CFeeRate estimateFee(int confTarget); + bool removeTx(uint256 hash, bool inBlock); - /** Estimate feerate needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. - */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); + /** DEPRECATED. Return a feerate estimate */ + CFeeRate estimateFee(int confTarget) const; - /** Return a priority estimate. - * DEPRECATED - * Returns -1 + /** Estimate feerate needed to get be included in a block within confTarget + * blocks. If no answer can be given at confTarget, return an estimate at + * the closest target where one can be given. 'conservative' estimates are + * valid over longer time horizons also. */ - double estimatePriority(int confTarget); + CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative = true) const; - /** Estimate priority needed to get be included in a block within - * confTarget blocks. - * DEPRECATED - * Returns -1 unless mempool is currently limited then returns INF_PRIORITY - * answerFoundAtTarget is set to confTarget + /** Return a specific fee estimate calculation with a given success + * threshold and time horizon, and optionally return detailed data about + * calculation */ - double estimateSmartPriority(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); + CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult *result = nullptr) const; /** Write estimation data to a file */ - void Write(CAutoFile& fileout); + bool Write(CAutoFile& fileout) const; /** Read estimation data from a file */ - void Read(CAutoFile& filein, int nFileVersion); + bool Read(CAutoFile& filein); + + /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ + void FlushUnconfirmed(CTxMemPool& pool); private: - CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main unsigned int nBestSeenHeight; + unsigned int firstRecordedHeight; + unsigned int historicalFirst; + unsigned int historicalBest; + struct TxStatsInfo { unsigned int blockHeight; @@ -257,14 +207,43 @@ private: std::map<uint256, TxStatsInfo> mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats feeStats; + TxConfirmStats* feeStats; + TxConfirmStats* shortStats; + TxConfirmStats* longStats; unsigned int trackedTxs; unsigned int untrackedTxs; + + std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) + std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + mutable CCriticalSection cs_feeEstimator; + + /** Process a transaction confirmed in a block*/ + bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); + + /** Helper for estimateSmartFee */ + double estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const; + /** Helper for estimateSmartFee */ + double estimateConservativeFee(unsigned int doubleTarget) const; + /** Number of blocks of data recorded while fee estimates have been running */ + unsigned int BlockSpan() const; + /** Number of blocks of recorded fee estimate data represented in saved data file */ + unsigned int HistoricalBlockSpan() const; + /** Calculation of highest target that reasonable estimate can be provided for */ + unsigned int MaxUsableEstimate() const; }; class FeeFilterRounder { +private: + static constexpr double MAX_FILTER_FEERATE = 1e7; + /** FEE_FILTER_SPACING is just used to provide some quantization of fee + * filter results. Historically it reused FEE_SPACING, but it is completely + * unrelated, and was made a separate constant so the two concepts are not + * tied together */ + static constexpr double FEE_FILTER_SPACING = 1.1; + public: /** Create new FeeFilterRounder */ FeeFilterRounder(const CFeeRate& minIncrementalFee); @@ -276,4 +255,5 @@ private: std::set<double> feeset; FastRandomContext insecure_rand; }; + #endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index ec398f6627..14d58e7442 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -8,12 +8,50 @@ #include "policy/policy.h" #include "validation.h" +#include "coins.h" #include "tinyformat.h" #include "util.h" #include "utilstrencodings.h" #include <boost/foreach.hpp> +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) +{ + // "Dust" is defined in terms of dustRelayFee, + // which has units satoshis-per-kilobyte. + // If you'd pay more than 1/3 in fees + // to spend something, then we consider it dust. + // A typical spendable non-segwit txout is 34 bytes big, and will + // need a CTxIn of at least 148 bytes to spend: + // so dust is a spendable txout less than + // 546*dustRelayFee/1000 (in satoshis). + // A typical spendable segwit txout is 31 bytes big, and will + // need a CTxIn of at least 67 bytes to spend: + // so dust is a spendable txout less than + // 294*dustRelayFee/1000 (in satoshis). + if (txout.scriptPubKey.IsUnspendable()) + return 0; + + size_t nSize = GetSerializeSize(txout, SER_DISK, 0); + int witnessversion = 0; + std::vector<unsigned char> witnessprogram; + + if (txout.scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { + // sum the sizes of the parts of a transaction input + // with 75% segwit discount applied to the script size. + nSize += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); + } else { + nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above + } + + return 3 * dustRelayFeeIn.GetFee(nSize); +} + +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) +{ + return (txout.nValue < GetDustThreshold(txout, dustRelayFeeIn)); +} + /** * Check transaction inputs to mitigate two * potential denial-of-service attacks: @@ -105,7 +143,7 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnes else if ((whichType == TX_MULTISIG) && (!fIsBareMultisigStd)) { reason = "bare-multisig"; return false; - } else if (txout.IsDust(dustRelayFee)) { + } else if (IsDust(txout, ::dustRelayFee)) { reason = "dust"; return false; } @@ -127,7 +165,7 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut& prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector<std::vector<unsigned char> > vSolutions; txnouttype whichType; @@ -166,7 +204,7 @@ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) if (tx.vin[i].scriptWitness.IsNull()) continue; - const CTxOut &prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut &prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; // get the scriptPubKey corresponding to this input: CScript prevScript = prev.scriptPubKey; diff --git a/src/policy/policy.h b/src/policy/policy.h index 9b1323ac26..2c2ea9d5b8 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -7,17 +7,17 @@ #define BITCOIN_POLICY_POLICY_H #include "consensus/consensus.h" +#include "feerate.h" #include "script/interpreter.h" #include "script/standard.h" #include <string> class CCoinsViewCache; +class CTxOut; /** Default for -blockmaxsize, which controls the maximum size of block the mining code will create **/ static const unsigned int DEFAULT_BLOCK_MAX_SIZE = 750000; -/** Default for -blockprioritysize, maximum space for zero/low-fee transactions **/ -static const unsigned int DEFAULT_BLOCK_PRIORITY_SIZE = 0; /** Default for -blockmaxweight, which controls the range of block weights the mining code will create **/ static const unsigned int DEFAULT_BLOCK_MAX_WEIGHT = 3000000; /** Default for -blockmintxfee, which sets the minimum feerate for a transaction in blocks created by mining code **/ @@ -74,6 +74,10 @@ static const unsigned int STANDARD_NOT_MANDATORY_VERIFY_FLAGS = STANDARD_SCRIPT_ static const unsigned int STANDARD_LOCKTIME_VERIFY_FLAGS = LOCKTIME_VERIFY_SEQUENCE | LOCKTIME_MEDIAN_TIME_PAST; +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); + +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); + bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool witnessEnabled = false); /** * Check for standard transaction types diff --git a/src/pow.cpp b/src/pow.cpp index e57fd866f8..e06d9662e6 100644 --- a/src/pow.cpp +++ b/src/pow.cpp @@ -12,12 +12,9 @@ unsigned int GetNextWorkRequired(const CBlockIndex* pindexLast, const CBlockHeader *pblock, const Consensus::Params& params) { + assert(pindexLast != NULL); unsigned int nProofOfWorkLimit = UintToArith256(params.powLimit).GetCompact(); - // Genesis block - if (pindexLast == NULL) - return nProofOfWorkLimit; - // Only change once per difficulty adjustment interval if ((pindexLast->nHeight+1) % params.DifficultyAdjustmentInterval() != 0) { diff --git a/src/prevector.h b/src/prevector.h index cba2e30057..177d81383e 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -11,6 +11,7 @@ #include <string.h> #include <iterator> +#include <type_traits> #pragma pack(push, 1) /** Implements a drop-in replacement for std::vector<T> which stores up to N @@ -388,10 +389,14 @@ public: iterator erase(iterator first, iterator last) { iterator p = first; char* endp = (char*)&(*end()); - while (p != last) { - (*p).~T(); - _size--; - ++p; + if (!std::is_trivially_destructible<T>::value) { + while (p != last) { + (*p).~T(); + _size--; + ++p; + } + } else { + _size -= last - p; } memmove(&(*first), &(*last), endp - ((char*)(&(*last)))); return first; @@ -432,7 +437,9 @@ public: } ~prevector() { - clear(); + if (!std::is_trivially_destructible<T>::value) { + clear(); + } if (!is_direct()) { free(_union.indirect); _union.indirect = NULL; diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 790bc71d14..a0d7793f97 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -69,6 +69,9 @@ uint256 CTransaction::ComputeHash() const uint256 CTransaction::GetWitnessHash() const { + if (!HasWitness()) { + return GetHash(); + } return SerializeHash(*this, SER_GETHASH, 0); } @@ -89,32 +92,6 @@ CAmount CTransaction::GetValueOut() const return nValueOut; } -double CTransaction::ComputePriority(double dPriorityInputs, unsigned int nTxSize) const -{ - nTxSize = CalculateModifiedSize(nTxSize); - if (nTxSize == 0) return 0.0; - - return dPriorityInputs / nTxSize; -} - -unsigned int CTransaction::CalculateModifiedSize(unsigned int nTxSize) const -{ - // In order to avoid disincentivizing cleaning up the UTXO set we don't count - // the constant overhead for each txin and up to 110 bytes of scriptSig (which - // is enough to cover a compressed pubkey p2sh redemption) for priority. - // Providing any more cleanup incentive than making additional inputs free would - // risk encouraging people to create junk outputs to redeem later. - if (nTxSize == 0) - nTxSize = (GetTransactionWeight(*this) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR; - for (std::vector<CTxIn>::const_iterator it(vin.begin()); it != vin.end(); ++it) - { - unsigned int offset = 41U + std::min(110U, (unsigned int)it->scriptSig.size()); - if (nTxSize > offset) - nTxSize -= offset; - } - return nTxSize; -} - unsigned int CTransaction::GetTotalSize() const { return ::GetSerializeSize(*this, SER_NETWORK, PROTOCOL_VERSION); diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index af2986a41b..00ac0b92b5 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -22,8 +22,8 @@ public: uint256 hash; uint32_t n; - COutPoint() { SetNull(); } - COutPoint(uint256 hashIn, uint32_t nIn) { hash = hashIn; n = nIn; } + COutPoint(): n((uint32_t) -1) { } + COutPoint(const uint256& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } ADD_SERIALIZE_METHODS; @@ -161,43 +161,6 @@ public: return (nValue == -1); } - CAmount GetDustThreshold(const CFeeRate &minRelayTxFee) const - { - // "Dust" is defined in terms of CTransaction::minRelayTxFee, - // which has units satoshis-per-kilobyte. - // If you'd pay more than 1/3 in fees - // to spend something, then we consider it dust. - // A typical spendable non-segwit txout is 34 bytes big, and will - // need a CTxIn of at least 148 bytes to spend: - // so dust is a spendable txout less than - // 546*minRelayTxFee/1000 (in satoshis). - // A typical spendable segwit txout is 31 bytes big, and will - // need a CTxIn of at least 67 bytes to spend: - // so dust is a spendable txout less than - // 294*minRelayTxFee/1000 (in satoshis). - if (scriptPubKey.IsUnspendable()) - return 0; - - size_t nSize = GetSerializeSize(*this, SER_DISK, 0); - int witnessversion = 0; - std::vector<unsigned char> witnessprogram; - - if (scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { - // sum the sizes of the parts of a transaction input - // with 75% segwit discount applied to the script size. - nSize += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); - } else { - nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above - } - - return 3 * minRelayTxFee.GetFee(nSize); - } - - bool IsDust(const CFeeRate &minRelayTxFee) const - { - return (nValue < GetDustThreshold(minRelayTxFee)); - } - friend bool operator==(const CTxOut& a, const CTxOut& b) { return (a.nValue == b.nValue && @@ -361,12 +324,6 @@ public: // GetValueIn() is a method on CCoinsViewCache, because // inputs must be known to compute value in. - // Compute priority, given priority of inputs and (optionally) tx size - double ComputePriority(double dPriorityInputs, unsigned int nTxSize=0) const; - - // Compute modified tx size for priority calculation (optionally given tx size) - unsigned int CalculateModifiedSize(unsigned int nTxSize=0) const; - /** * Get the total transaction size in bytes, including witness data. * "Total Size" defined in BIP141 and BIP144. diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 4b34e73eb7..f8a99506c1 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -181,7 +181,5 @@ void BanTableModel::sort(int column, Qt::SortOrder order) bool BanTableModel::shouldShow() { - if (priv->size() > 0) - return true; - return false; + return priv->size() > 0; } diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index c31dea2067..23ec3ab434 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -10,6 +10,7 @@ #include "chainparams.h" #include "clientmodel.h" +#include "fs.h" #include "guiconstants.h" #include "guiutil.h" #include "intro.h" @@ -38,7 +39,6 @@ #include <stdint.h> -#include <boost/filesystem/operations.hpp> #include <boost/thread.hpp> #include <QApplication> @@ -152,15 +152,21 @@ static void initTranslations(QTranslator &qtTranslatorBase, QTranslator &qtTrans #if QT_VERSION < 0x050000 void DebugMessageHandler(QtMsgType type, const char *msg) { - const char *category = (type == QtDebugMsg) ? "qt" : NULL; - LogPrint(category, "GUI: %s\n", msg); + if (type == QtDebugMsg) { + LogPrint(BCLog::QT, "GUI: %s\n", msg); + } else { + LogPrintf("GUI: %s\n", msg); + } } #else void DebugMessageHandler(QtMsgType type, const QMessageLogContext& context, const QString &msg) { Q_UNUSED(context); - const char *category = (type == QtDebugMsg) ? "qt" : NULL; - LogPrint(category, "GUI: %s\n", msg.toStdString()); + if (type == QtDebugMsg) { + LogPrint(BCLog::QT, "GUI: %s\n", msg.toStdString()); + } else { + LogPrintf("GUI: %s\n", msg.toStdString()); + } } #endif @@ -178,8 +184,8 @@ public Q_SLOTS: void shutdown(); Q_SIGNALS: - void initializeResult(int retval); - void shutdownResult(int retval); + void initializeResult(bool success); + void shutdownResult(); void runawayException(const QString &message); private: @@ -223,8 +229,8 @@ public: WId getMainWinId() const; public Q_SLOTS: - void initializeResult(int retval); - void shutdownResult(int retval); + void initializeResult(bool success); + void shutdownResult(); /// Handle runaway exceptions. Shows a message box with the problem and quits the program. void handleRunawayException(const QString &message); @@ -284,7 +290,7 @@ void BitcoinCore::initialize() Q_EMIT initializeResult(false); return; } - int rv = AppInitMain(threadGroup, scheduler); + bool rv = AppInitMain(threadGroup, scheduler); Q_EMIT initializeResult(rv); } catch (const std::exception& e) { handleRunawayException(&e); @@ -302,7 +308,7 @@ void BitcoinCore::shutdown() threadGroup.join_all(); Shutdown(); qDebug() << __func__ << ": Shutdown finished"; - Q_EMIT shutdownResult(1); + Q_EMIT shutdownResult(); } catch (const std::exception& e) { handleRunawayException(&e); } catch (...) { @@ -398,8 +404,8 @@ void BitcoinApplication::startThread() executor->moveToThread(coreThread); /* communication to and from thread */ - connect(executor, SIGNAL(initializeResult(int)), this, SLOT(initializeResult(int))); - connect(executor, SIGNAL(shutdownResult(int)), this, SLOT(shutdownResult(int))); + connect(executor, SIGNAL(initializeResult(bool)), this, SLOT(initializeResult(bool))); + connect(executor, SIGNAL(shutdownResult()), this, SLOT(shutdownResult())); connect(executor, SIGNAL(runawayException(QString)), this, SLOT(handleRunawayException(QString))); connect(this, SIGNAL(requestedInitialize()), executor, SLOT(initialize())); connect(this, SIGNAL(requestedShutdown()), executor, SLOT(shutdown())); @@ -450,12 +456,12 @@ void BitcoinApplication::requestShutdown() Q_EMIT requestedShutdown(); } -void BitcoinApplication::initializeResult(int retval) +void BitcoinApplication::initializeResult(bool success) { - qDebug() << __func__ << ": Initialization result: " << retval; - // Set exit result: 0 if successful, 1 if failure - returnValue = retval ? 0 : 1; - if(retval) + qDebug() << __func__ << ": Initialization result: " << success; + // Set exit result. + returnValue = success ? EXIT_SUCCESS : EXIT_FAILURE; + if(success) { // Log this only after AppInitMain finishes, as then logging setup is guaranteed complete qWarning() << "Platform customization:" << platformStyle->getName(); @@ -507,9 +513,8 @@ void BitcoinApplication::initializeResult(int retval) } } -void BitcoinApplication::shutdownResult(int retval) +void BitcoinApplication::shutdownResult() { - qDebug() << __func__ << ": Shutdown result: " << retval; quit(); // Exit main loop after shutdown finished } @@ -603,7 +608,7 @@ int main(int argc, char *argv[]) /// 6. Determine availability of data directory and parse bitcoin.conf /// - Do not call GetDataDir(true) before this step finishes - if (!boost::filesystem::is_directory(GetDataDir(false))) + if (!fs::is_directory(GetDataDir(false))) { QMessageBox::critical(0, QObject::tr(PACKAGE_NAME), QObject::tr("Error: Specified data directory \"%1\" does not exist.").arg(QString::fromStdString(GetArg("-datadir", "")))); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index be79a67990..5c26baef9e 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -478,6 +478,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) connect(_clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); connect(_clientModel, SIGNAL(networkActiveChanged(bool)), this, SLOT(setNetworkActive(bool))); + modalOverlay->setKnownBestHeight(_clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(_clientModel->getHeaderTipTime())); setNumBlocks(_clientModel->getNumBlocks(), _clientModel->getLastBlockDate(), _clientModel->getVerificationProgress(NULL), false); connect(_clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); @@ -505,8 +506,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) // initialize the disable state of the tray icon with the current value in the model. setTrayIconVisible(optionsModel->getHideTrayIcon()); } - - modalOverlay->setKnownBestHeight(clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(clientModel->getHeaderTipTime())); } else { // Disable possibility to show main window via action toggleHideAction->setEnabled(false); diff --git a/src/qt/callback.h b/src/qt/callback.h new file mode 100644 index 0000000000..a8b593a652 --- /dev/null +++ b/src/qt/callback.h @@ -0,0 +1,30 @@ +#ifndef BITCOIN_QT_CALLBACK_H +#define BITCOIN_QT_CALLBACK_H + +#include <QObject> + +class Callback : public QObject +{ + Q_OBJECT +public Q_SLOTS: + virtual void call() = 0; +}; + +template <typename F> +class FunctionCallback : public Callback +{ + F f; + +public: + FunctionCallback(F f_) : f(std::move(f_)) {} + ~FunctionCallback() override {} + void call() override { f(this); } +}; + +template <typename F> +FunctionCallback<F>* makeCallback(F f) +{ + return new FunctionCallback<F>(std::move(f)); +} + +#endif // BITCOIN_QT_CALLBACK_H diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index bb10e49422..de00eacdb9 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -9,6 +9,7 @@ #include "guiutil.h" #include "peertablemodel.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "clientversion.h" @@ -36,6 +37,8 @@ ClientModel::ClientModel(OptionsModel *_optionsModel, QObject *parent) : banTableModel(0), pollTimer(0) { + cachedBestHeaderHeight = -1; + cachedBestHeaderTime = -1; peerTableModel = new PeerTableModel(this); banTableModel = new BanTableModel(this); pollTimer = new QTimer(this); @@ -74,18 +77,28 @@ int ClientModel::getNumBlocks() const int ClientModel::getHeaderTipHeight() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->nHeight; + if (cachedBestHeaderHeight == -1) { + // make sure we initially populate the cache via a cs_main lock + // otherwise we need to wait for a tip update + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderHeight; } int64_t ClientModel::getHeaderTipTime() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->GetBlockTime(); + if (cachedBestHeaderTime == -1) { + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderTime; } quint64 ClientModel::getTotalBytesRecv() const @@ -283,6 +296,11 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, const CB int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; + if (fHeader) { + // cache best headers time and height to reduce future cs_main locks + clientmodel->cachedBestHeaderHeight = pIndex->nHeight; + clientmodel->cachedBestHeaderTime = pIndex->GetBlockTime(); + } // if we are in-sync, update the UI regardless of last update time if (!initialSync || now - nLastUpdateNotification > MODEL_UPDATE_DELAY) { //pass a async signal to the UI thread diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 2c10e633b8..4c92e2144e 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -8,6 +8,8 @@ #include <QObject> #include <QDateTime> +#include <atomic> + class AddressTableModel; class BanTableModel; class OptionsModel; @@ -81,6 +83,10 @@ public: QString formatClientStartupTime() const; QString dataDir() const; + // caches for the best header + mutable std::atomic<int> cachedBestHeaderHeight; + mutable std::atomic<int64_t> cachedBestHeaderTime; + private: OptionsModel *optionsModel; PeerTableModel *peerTableModel; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index d4fd8bd372..135cf6f701 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -15,6 +15,7 @@ #include "wallet/coincontrol.h" #include "init.h" +#include "policy/fees.h" #include "policy/policy.h" #include "validation.h" // For mempool #include "wallet/wallet.h" @@ -433,8 +434,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) { CTxOut txout(amount, (CScript)std::vector<unsigned char>(24, 0)); txDummy.vout.push_back(txout); - if (txout.IsDust(dustRelayFee)) - fDust = true; + fDust |= IsDust(txout, ::dustRelayFee); } } @@ -444,11 +444,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) CAmount nChange = 0; unsigned int nBytes = 0; unsigned int nBytesInputs = 0; - double dPriority = 0; - double dPriorityInputs = 0; unsigned int nQuantity = 0; - int nQuantityUncompressed = 0; - bool fAllowFree = false; bool fWitness = false; std::vector<COutPoint> vCoinControl; @@ -473,9 +469,6 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) // Amount nAmount += out.tx->tx->vout[out.i].nValue; - // Priority - dPriorityInputs += (double)out.tx->tx->vout[out.i].nValue * (out.nDepth+1); - // Bytes CTxDestination address; int witnessversion = 0; @@ -492,8 +485,6 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (keyid && model->getPubKey(*keyid, pubkey)) { nBytesInputs += (pubkey.IsCompressed() ? 148 : 180); - if (!pubkey.IsCompressed()) - nQuantityUncompressed++; } else nBytesInputs += 148; // in all error cases, simply assume 148 here @@ -521,20 +512,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nBytes -= 34; // Fee - nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, mempool); - if (nPayFee > 0 && coinControl->nMinimumTotalFee > nPayFee) - nPayFee = coinControl->nMinimumTotalFee; - - - // Allow free? (require at least hard-coded threshold and default to that if no estimate) - double mempoolEstimatePriority = mempool.estimateSmartPriority(nTxConfirmTarget); - dPriority = dPriorityInputs / (nBytes - nBytesInputs + (nQuantityUncompressed * 29)); // 29 = 180 - 151 (uncompressed public keys are over the limit. max 151 bytes of the input are ignored for priority) - double dPriorityNeeded = std::max(mempoolEstimatePriority, AllowFreeThreshold()); - fAllowFree = (dPriority >= dPriorityNeeded); - - if (fSendFreeTransactions) - if (fAllowFree && nBytes <= MAX_FREE_TRANSACTION_CREATE_SIZE) - nPayFee = 0; + nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, ::mempool, ::feeEstimator); if (nPayAmount > 0) { @@ -546,10 +524,10 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (nChange > 0 && nChange < MIN_CHANGE) { CTxOut txout(nChange, (CScript)std::vector<unsigned char>(24, 0)); - if (txout.IsDust(dustRelayFee)) + if (IsDust(txout, ::dustRelayFee)) { if (CoinControlDialog::fSubtractFeeFromAmount) // dust-change will be raised until no dust - nChange = txout.GetDustThreshold(dustRelayFee); + nChange = GetDustThreshold(txout, ::dustRelayFee); else { nPayFee += nChange; @@ -593,7 +571,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) l5->setText(((nBytes > 0) ? ASYMP_UTF8 : "") + QString::number(nBytes)); // Bytes l7->setText(fDust ? tr("yes") : tr("no")); // Dust l8->setText(BitcoinUnits::formatWithUnit(nDisplayUnit, nChange)); // Change - if (nPayFee > 0 && (coinControl->nMinimumTotalFee < nPayFee)) + if (nPayFee > 0) { l3->setText(ASYMP_UTF8 + l3->text()); l4->setText(ASYMP_UTF8 + l4->text()); @@ -612,7 +590,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (payTxFee.GetFeePerK() > 0) dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), payTxFee.GetFeePerK()) / 1000; else { - dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), mempool.estimateSmartFee(nTxConfirmTarget).GetFeePerK()) / 1000; + dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), ::feeEstimator.estimateSmartFee(nTxConfirmTarget, NULL, ::mempool).GetFeePerK()) / 1000; } QString toolTip4 = tr("Can vary +/- %1 satoshi(s) per input.").arg(dFeeVary); diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 8be4a955b3..093e644bdc 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -645,7 +645,7 @@ <item> <widget class="QPushButton" name="btnClearTrafficGraph"> <property name="text"> - <string>&Clear</string> + <string>&Reset</string> </property> <property name="autoDefault"> <bool>false</bool> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 0b29201872..0f1b3f4a73 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -692,17 +692,34 @@ <item> <layout class="QHBoxLayout" name="horizontalLayout_Buttons"> <item> - <widget class="QPushButton" name="resetButton"> - <property name="toolTip"> - <string>Reset all client options to default.</string> - </property> - <property name="text"> - <string>&Reset Options</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - </widget> + <layout class="QVBoxLayout" name="verticalLayout_Buttons"> + <item> + <widget class="QPushButton" name="openBitcoinConfButton"> + <property name="toolTip"> + <string>Open the %1 configuration file from the working directory.</string> + </property> + <property name="text"> + <string>Open Configuration File</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="resetButton"> + <property name="toolTip"> + <string>Reset all client options to default.</string> + </property> + <property name="text"> + <string>&Reset Options</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + </layout> </item> <item> <spacer name="horizontalSpacer_1"> @@ -756,27 +773,48 @@ </spacer> </item> <item> - <widget class="QPushButton" name="okButton"> - <property name="text"> - <string>&OK</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - <property name="default"> - <bool>true</bool> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="cancelButton"> - <property name="text"> - <string>&Cancel</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - </widget> + <layout class="QVBoxLayout" name="verticalLayout_4"> + <item> + <spacer name="verticalSpacer"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>40</height> + </size> + </property> + </spacer> + </item> + <item> + <layout class="QHBoxLayout" name="horizontalLayout"> + <item> + <widget class="QPushButton" name="okButton"> + <property name="text"> + <string>&OK</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + <property name="default"> + <bool>true</bool> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="cancelButton"> + <property name="text"> + <string>&Cancel</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + </layout> + </item> + </layout> </item> </layout> </item> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index ca8ecffafe..89f9c25d14 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -760,10 +760,32 @@ </layout> </item> <item> + <widget class="QLabel" name="fallbackFeeWarningLabel"> + <property name="toolTip"> + <string>Using the fallbackfee can result in sending a transaction that will take several hours or days (or never) to confirm. Consider choosing your fee manually or wait until your have validated the complete chain.</string> + </property> + <property name="font"> + <font> + <weight>75</weight> + <bold>true</bold> + </font> + </property> + <property name="text"> + <string>Warning: Fee estimation is currently not possible.</string> + </property> + <property name="wordWrap"> + <bool>false</bool> + </property> + </widget> + </item> + <item> <spacer name="horizontalSpacer_4"> <property name="orientation"> <enum>Qt::Horizontal</enum> </property> + <property name="sizeType"> + <enum>QSizePolicy::MinimumExpanding</enum> + </property> <property name="sizeHint" stdset="0"> <size> <width>40</width> @@ -840,19 +862,6 @@ </widget> </item> <item> - <widget class="QRadioButton" name="radioCustomAtLeast"> - <property name="toolTip"> - <string>If the custom fee is set to 1000 satoshis and the transaction is only 250 bytes, then "per kilobyte" only pays 250 satoshis in fee, while "total at least" pays 1000 satoshis. For transactions bigger than a kilobyte both pay by kilobyte.</string> - </property> - <property name="text"> - <string>total at least</string> - </property> - <attribute name="buttonGroup"> - <string notr="true">groupCustomFee</string> - </attribute> - </widget> - </item> - <item> <widget class="BitcoinAmountField" name="customFee"/> </item> <item> @@ -1158,6 +1167,16 @@ </item> </layout> </item> + <item> + <widget class="QCheckBox" name="optInRBF"> + <property name="text"> + <string>Request Replace-By-Fee</string> + </property> + <property name="toolTip"> + <string>Indicates that the sender may wish to replace this transaction with a new one paying higher fees (prior to being confirmed).</string> + </property> + </widget> + </item> </layout> </widget> </item> @@ -1168,8 +1187,8 @@ </property> <property name="sizeHint" stdset="0"> <size> - <width>800</width> - <height>1</height> + <width>40</width> + <height>5</height> </size> </property> </spacer> @@ -1203,7 +1222,7 @@ <bool>false</bool> </property> <property name="default"> - <bool>true</bool> + <bool>false</bool> </property> </widget> </item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index fd3dcac424..bffa81137b 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -9,6 +9,7 @@ #include "qvalidatedlineedit.h" #include "walletmodel.h" +#include "fs.h" #include "primitives/transaction.h" #include "init.h" #include "policy/policy.h" @@ -35,9 +36,6 @@ #include "shlwapi.h" #endif -#include <boost/filesystem.hpp> -#include <boost/filesystem/fstream.hpp> -#include <boost/filesystem/detail/utf8_codecvt_facet.hpp> #include <boost/scoped_array.hpp> #include <QAbstractItemView> @@ -65,7 +63,7 @@ #include <QFontDatabase> #endif -static boost::filesystem::detail::utf8_codecvt_facet utf8; +static fs::detail::utf8_codecvt_facet utf8; #if defined(Q_OS_MAC) extern double NSAppKitVersionNumber; @@ -253,7 +251,7 @@ bool isDust(const QString& address, const CAmount& amount) CTxDestination dest = CBitcoinAddress(address.toStdString()).Get(); CScript script = GetScriptForDestination(dest); CTxOut txOut(amount, script); - return txOut.IsDust(dustRelayFee); + return IsDust(txOut, ::dustRelayFee); } QString HtmlEscape(const QString& str, bool fMultiLine) @@ -410,13 +408,29 @@ bool isObscured(QWidget *w) void openDebugLogfile() { - boost::filesystem::path pathDebug = GetDataDir() / "debug.log"; + fs::path pathDebug = GetDataDir() / "debug.log"; /* Open debug.log with the associated application */ - if (boost::filesystem::exists(pathDebug)) + if (fs::exists(pathDebug)) QDesktopServices::openUrl(QUrl::fromLocalFile(boostPathToQString(pathDebug))); } +bool openBitcoinConf() +{ + boost::filesystem::path pathConfig = GetConfigFile(BITCOIN_CONF_FILENAME); + + /* Create the file */ + boost::filesystem::ofstream configFile(pathConfig, std::ios_base::app); + + if (!configFile.good()) + return false; + + configFile.close(); + + /* Open bitcoin.conf with the associated application */ + return QDesktopServices::openUrl(QUrl::fromLocalFile(boostPathToQString(pathConfig))); +} + void SubstituteFonts(const QString& language) { #if defined(Q_OS_MAC) @@ -443,7 +457,7 @@ void SubstituteFonts(const QString& language) /* 10.10 or later system */ if (language == "zh_CN" || language == "zh_TW" || language == "zh_HK") // traditional or simplified Chinese QFont::insertSubstitution(".Helvetica Neue DeskInterface", "Heiti SC"); - else if (language == "ja") // Japanesee + else if (language == "ja") // Japanese QFont::insertSubstitution(".Helvetica Neue DeskInterface", "Songti SC"); else QFont::insertSubstitution(".Helvetica Neue DeskInterface", "Lucida Grande"); @@ -597,7 +611,7 @@ TableViewLastColumnResizingFixer::TableViewLastColumnResizingFixer(QTableView* t } #ifdef WIN32 -boost::filesystem::path static StartupShortcutPath() +fs::path static StartupShortcutPath() { std::string chain = ChainNameFromCommandLine(); if (chain == CBaseChainParams::MAIN) @@ -610,13 +624,13 @@ boost::filesystem::path static StartupShortcutPath() bool GetStartOnSystemStartup() { // check for Bitcoin*.lnk - return boost::filesystem::exists(StartupShortcutPath()); + return fs::exists(StartupShortcutPath()); } bool SetStartOnSystemStartup(bool fAutoStart) { // If the shortcut exists already, remove it for updating - boost::filesystem::remove(StartupShortcutPath()); + fs::remove(StartupShortcutPath()); if (fAutoStart) { @@ -686,10 +700,8 @@ bool SetStartOnSystemStartup(bool fAutoStart) // Follow the Desktop Application Autostart Spec: // http://standards.freedesktop.org/autostart-spec/autostart-spec-latest.html -boost::filesystem::path static GetAutostartDir() +fs::path static GetAutostartDir() { - namespace fs = boost::filesystem; - char* pszConfigHome = getenv("XDG_CONFIG_HOME"); if (pszConfigHome) return fs::path(pszConfigHome) / "autostart"; char* pszHome = getenv("HOME"); @@ -697,7 +709,7 @@ boost::filesystem::path static GetAutostartDir() return fs::path(); } -boost::filesystem::path static GetAutostartFilePath() +fs::path static GetAutostartFilePath() { std::string chain = ChainNameFromCommandLine(); if (chain == CBaseChainParams::MAIN) @@ -707,7 +719,7 @@ boost::filesystem::path static GetAutostartFilePath() bool GetStartOnSystemStartup() { - boost::filesystem::ifstream optionFile(GetAutostartFilePath()); + fs::ifstream optionFile(GetAutostartFilePath()); if (!optionFile.good()) return false; // Scan through file for "Hidden=true": @@ -727,7 +739,7 @@ bool GetStartOnSystemStartup() bool SetStartOnSystemStartup(bool fAutoStart) { if (!fAutoStart) - boost::filesystem::remove(GetAutostartFilePath()); + fs::remove(GetAutostartFilePath()); else { char pszExePath[MAX_PATH+1]; @@ -735,9 +747,9 @@ bool SetStartOnSystemStartup(bool fAutoStart) if (readlink("/proc/self/exe", pszExePath, sizeof(pszExePath)-1) == -1) return false; - boost::filesystem::create_directories(GetAutostartDir()); + fs::create_directories(GetAutostartDir()); - boost::filesystem::ofstream optionFile(GetAutostartFilePath(), std::ios_base::out|std::ios_base::trunc); + fs::ofstream optionFile(GetAutostartFilePath(), std::ios_base::out|std::ios_base::trunc); if (!optionFile.good()) return false; std::string chain = ChainNameFromCommandLine(); @@ -843,14 +855,17 @@ void restoreWindowGeometry(const QString& strSetting, const QSize& defaultSize, QPoint pos = settings.value(strSetting + "Pos").toPoint(); QSize size = settings.value(strSetting + "Size", defaultSize).toSize(); - if (!pos.x() && !pos.y()) { - QRect screen = QApplication::desktop()->screenGeometry(); - pos.setX((screen.width() - size.width()) / 2); - pos.setY((screen.height() - size.height()) / 2); - } - parent->resize(size); parent->move(pos); + + if ((!pos.x() && !pos.y()) || (QApplication::desktop()->screenNumber(parent) == -1)) + { + QRect screen = QApplication::desktop()->screenGeometry(); + QPoint defaultPos((screen.width() - defaultSize.width()) / 2, + (screen.height() - defaultSize.height()) / 2); + parent->resize(defaultSize); + parent->move(defaultPos); + } } void setClipboard(const QString& str) @@ -859,12 +874,12 @@ void setClipboard(const QString& str) QApplication::clipboard()->setText(str, QClipboard::Selection); } -boost::filesystem::path qstringToBoostPath(const QString &path) +fs::path qstringToBoostPath(const QString &path) { - return boost::filesystem::path(path.toStdString(), utf8); + return fs::path(path.toStdString(), utf8); } -QString boostPathToQString(const boost::filesystem::path &path) +QString boostPathToQString(const fs::path &path) { return QString::fromStdString(path.string(utf8)); } diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 913aa5e24b..d6aa8c4ea6 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_GUIUTIL_H #include "amount.h" +#include "fs.h" #include <QEvent> #include <QHeaderView> @@ -16,8 +17,6 @@ #include <QTableView> #include <QLabel> -#include <boost/filesystem.hpp> - class QValidatedLineEdit; class SendCoinsRecipient; @@ -114,6 +113,9 @@ namespace GUIUtil // Open debug.log void openDebugLogfile(); + // Open the config file + bool openBitcoinConf(); + // Replace invalid default fonts with known good ones void SubstituteFonts(const QString& language); @@ -183,10 +185,10 @@ namespace GUIUtil void restoreWindowGeometry(const QString& strSetting, const QSize &defaultSizeIn, QWidget *parent); /* Convert QString to OS specific boost path through UTF-8 */ - boost::filesystem::path qstringToBoostPath(const QString &path); + fs::path qstringToBoostPath(const QString &path); /* Convert OS specific boost path to QString through UTF-8 */ - QString boostPathToQString(const boost::filesystem::path &path); + QString boostPathToQString(const fs::path &path); /* Convert seconds into a QString with days, hours, mins, secs */ QString formatDurationStr(int secs); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 4939648ff0..2460a59109 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -6,6 +6,7 @@ #include "config/bitcoin-config.h" #endif +#include "fs.h" #include "intro.h" #include "ui_intro.h" @@ -13,8 +14,6 @@ #include "util.h" -#include <boost/filesystem.hpp> - #include <QFileDialog> #include <QSettings> #include <QMessageBox> @@ -70,7 +69,6 @@ FreespaceChecker::FreespaceChecker(Intro *_intro) void FreespaceChecker::check() { - namespace fs = boost::filesystem; QString dataDirStr = intro->getPathToCheck(); fs::path dataDir = GUIUtil::qstringToBoostPath(dataDirStr); uint64_t freeBytesAvailable = 0; @@ -190,7 +188,6 @@ QString Intro::getDefaultDataDirectory() bool Intro::pickDataDirectory() { - namespace fs = boost::filesystem; QSettings settings; /* If data directory provided on command line, no need to look at settings or show a picking dialog */ diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 7ff00b1e9e..efb25aaf18 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -232,6 +232,18 @@ void OptionsDialog::on_resetButton_clicked() } } +void OptionsDialog::on_openBitcoinConfButton_clicked() +{ + /* explain the purpose of the config file */ + QMessageBox::information(this, tr("Configuration options"), + tr("The configuration file is used to specify advanced user options which override GUI settings. " + "Additionally, any command-line options will override this configuration file.")); + + /* show an error if there was some problem opening the file */ + if (!GUIUtil::openBitcoinConf()) + QMessageBox::critical(this, tr("Error"), tr("The configuration file could not be opened.")); +} + void OptionsDialog::on_okButton_clicked() { mapper->submit(); diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index d98c1dc193..f9f5823c05 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -47,6 +47,7 @@ private Q_SLOTS: /* set OK button state (enabled / disabled) */ void setOkButtonState(bool fState); void on_resetButton_clicked(); + void on_openBitcoinConfButton_clicked(); void on_okButton_clicked(); void on_cancelButton_clicked(); diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index dd75f12076..c31a7a478d 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -219,14 +219,16 @@ void PaymentServer::ipcParseCommandLine(int argc, char* argv[]) if (GUIUtil::parseBitcoinURI(arg, &r) && !r.address.isEmpty()) { CBitcoinAddress address(r.address.toStdString()); + auto tempChainParams = CreateChainParams(CBaseChainParams::MAIN); - if (address.IsValid(Params(CBaseChainParams::MAIN))) + if (address.IsValid(*tempChainParams)) { SelectParams(CBaseChainParams::MAIN); } - else if (address.IsValid(Params(CBaseChainParams::TESTNET))) - { - SelectParams(CBaseChainParams::TESTNET); + else { + tempChainParams = CreateChainParams(CBaseChainParams::TESTNET); + if (address.IsValid(*tempChainParams)) + SelectParams(CBaseChainParams::TESTNET); } } } @@ -580,7 +582,7 @@ bool PaymentServer::processPaymentRequest(const PaymentRequestPlus& request, Sen // Extract and check amounts CTxOut txOut(sendingTo.second, sendingTo.first); - if (txOut.IsDust(dustRelayFee)) { + if (IsDust(txOut, ::dustRelayFee)) { Q_EMIT message(tr("Payment request error"), tr("Requested payment amount of %1 is too small (considered dust).") .arg(BitcoinUnits::formatWithUnit(optionsModel->getDisplayUnit(), sendingTo.second)), CClientUIInterface::MSG_ERROR); diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 974a71ddca..fff072fd4c 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -166,7 +166,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const switch(index.column()) { case NetNodeId: - return rec->nodeStats.nodeid; + return (qint64)rec->nodeStats.nodeid; case Address: return QString::fromStdString(rec->nodeStats.addrName); case Subversion: diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 60406c2059..b200cb1127 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -98,7 +98,7 @@ class QtRPCTimerBase: public QObject, public RPCTimerBase { Q_OBJECT public: - QtRPCTimerBase(boost::function<void(void)>& _func, int64_t millis): + QtRPCTimerBase(std::function<void(void)>& _func, int64_t millis): func(_func) { timer.setSingleShot(true); @@ -110,7 +110,7 @@ private Q_SLOTS: void timeout() { func(); } private: QTimer timer; - boost::function<void(void)> func; + std::function<void(void)> func; }; class QtRPCTimerInterface: public RPCTimerInterface @@ -118,7 +118,7 @@ class QtRPCTimerInterface: public RPCTimerInterface public: ~QtRPCTimerInterface() {} const char *Name() { return "Qt"; } - RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) { return new QtRPCTimerBase(func, millis); } @@ -175,6 +175,10 @@ bool RPCConsole::RPCParseCommandLine(std::string &strResult, const std::string & nDepthInsideSensitive = 1; filter_begin_pos = chpos; } + // Make sure stack is not empty before adding something + if (stack.empty()) { + stack.push_back(std::vector<std::string>()); + } stack.back().push_back(strArg); }; @@ -626,9 +630,12 @@ void RPCConsole::setClientModel(ClientModel *model) for (size_t i = 0; i < commandList.size(); ++i) { wordList << commandList[i].c_str(); + wordList << ("help " + commandList[i]).c_str(); } + wordList.sort(); autoCompleter = new QCompleter(wordList, this); + autoCompleter->setModelSorting(QCompleter::CaseSensitivelySortedModel); ui->lineEdit->setCompleter(autoCompleter); autoCompleter->popup()->installEventFilter(this); // Start thread to execute RPC commands. @@ -723,8 +730,14 @@ void RPCConsole::clear(bool clearHistory) ).arg(fixedFontInfo.family(), QString("%1pt").arg(consoleFontSize)) ); +#ifdef Q_OS_MAC + QString clsKey = "(⌘)-L"; +#else + QString clsKey = "Ctrl-L"; +#endif + message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + - tr("Use up and down arrows to navigate history, and <b>Ctrl-L</b> to clear screen.") + "<br>" + + tr("Use up and down arrows to navigate history, and %1 to clear screen.").arg("<b>"+clsKey+"</b>") + "<br>" + tr("Type <b>help</b> for an overview of available commands.")) + "<br><span class=\"secwarning\">" + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramification of a command.") + @@ -822,7 +835,7 @@ void RPCConsole::on_lineEdit_returnPressed() cmdBeforeBrowsing = QString(); - message(CMD_REQUEST, cmd); + message(CMD_REQUEST, QString::fromStdString(strFilteredCmd)); Q_EMIT cmdRequest(cmd); cmd = QString::fromStdString(strFilteredCmd); @@ -1111,7 +1124,7 @@ void RPCConsole::disconnectSelectedNode() for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data().toInt(); + NodeId id = nodes.at(i).data().toLongLong(); // Find the node, disconnect it and clear the selected node if(g_connman->DisconnectNode(id)) clearSelectedNode(); @@ -1128,7 +1141,7 @@ void RPCConsole::banSelectedNode(int bantime) for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data().toInt(); + NodeId id = nodes.at(i).data().toLongLong(); // Get currently selected peer address int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 1c0ed663c1..272ab9486a 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -21,16 +21,16 @@ #include "validation.h" // mempool and minRelayTxFee #include "ui_interface.h" #include "txmempool.h" +#include "policy/fees.h" #include "wallet/wallet.h" +#include <QFontMetrics> #include <QMessageBox> #include <QScrollBar> #include <QSettings> #include <QTextDocument> #include <QTimer> -#define SEND_CONFIRM_DELAY 3 - SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent), ui(new Ui::SendCoinsDialog), @@ -109,7 +109,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupFee->setId(ui->radioCustomFee, 1); ui->groupFee->button((int)std::max(0, std::min(1, settings.value("nFeeRadio").toInt())))->setChecked(true); ui->groupCustomFee->setId(ui->radioCustomPerKilobyte, 0); - ui->groupCustomFee->setId(ui->radioCustomAtLeast, 1); ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); @@ -173,6 +172,9 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateSmartFeeLabel(); updateGlobalFeeVariables(); + // set default rbf checkbox state + ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); + // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; if (settings.value("nSmartFeeSliderPosition").toInt() == 0) @@ -246,6 +248,8 @@ void SendCoinsDialog::on_sendButton_clicked() else ctrl.nConfirmTarget = 0; + ctrl.signalRbf = ui->optInRBF->isChecked(); + prepareStatus = model->prepareTransaction(currentTransaction, &ctrl); // process prepareStatus and on error generate message shown to user @@ -325,6 +329,13 @@ void SendCoinsDialog::on_sendButton_clicked() questionString.append(QString("<span style='font-size:10pt;font-weight:normal;'><br />(=%2)</span>") .arg(alternativeUnits.join(" " + tr("or") + "<br />"))); + if (ui->optInRBF->isChecked()) + { + questionString.append("<hr /><span>"); + questionString.append(tr("This transaction signals replaceability (optin-RBF).")); + questionString.append("</span>"); + } + SendConfirmationDialog confirmationDialog(tr("Confirm send coins"), questionString.arg(formatted.join("<br />")), SEND_CONFIRM_DELAY, this); confirmationDialog.exec(); @@ -594,7 +605,6 @@ void SendCoinsDialog::updateFeeSectionControls() ui->checkBoxMinimumFee ->setEnabled(ui->radioCustomFee->isChecked()); ui->labelMinFeeWarning ->setEnabled(ui->radioCustomFee->isChecked()); ui->radioCustomPerKilobyte ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); - ui->radioCustomAtLeast ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked() && CoinControlDialog::coinControl->HasSelected()); ui->customFee ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); } @@ -605,19 +615,12 @@ void SendCoinsDialog::updateGlobalFeeVariables() int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; payTxFee = CFeeRate(0); - // set nMinimumTotalFee to 0 to not accidentally pay a custom fee - CoinControlDialog::coinControl->nMinimumTotalFee = 0; - // show the estimated required time for confirmation ui->confirmationTargetLabel->setText(GUIUtil::formatDurationStr(nConfirmTarget * Params().GetConsensus().nPowTargetSpacing) + " / " + tr("%n block(s)", "", nConfirmTarget)); } else { payTxFee = CFeeRate(ui->customFee->value()); - - // if user has selected to set a minimum absolute fee, pass the value to coincontrol - // set nMinimumTotalFee to 0 in case of user has selected that the fee is per KB - CoinControlDialog::coinControl->nMinimumTotalFee = ui->radioCustomAtLeast->isChecked() ? ui->customFee->value() : 0; } } @@ -649,13 +652,18 @@ void SendCoinsDialog::updateSmartFeeLabel() int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; int estimateFoundAtBlocks = nBlocksToConfirm; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks, ::mempool); if (feeRate <= CFeeRate(0)) // not enough data => minfee { ui->labelSmartFee->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), std::max(CWallet::fallbackFee.GetFeePerK(), CWallet::GetRequiredFee(1000))) + "/kB"); ui->labelSmartFee2->show(); // (Smart fee not initialized yet. This usually takes a few blocks...) ui->labelFeeEstimation->setText(""); + ui->fallbackFeeWarningLabel->setVisible(true); + int lightness = ui->fallbackFeeWarningLabel->palette().color(QPalette::WindowText).lightness(); + QColor warning_colour(255 - (lightness / 5), 176 - (lightness / 3), 48 - (lightness / 14)); + ui->fallbackFeeWarningLabel->setStyleSheet("QLabel { color: " + warning_colour.name() + "; }"); + ui->fallbackFeeWarningLabel->setIndent(QFontMetrics(ui->fallbackFeeWarningLabel->font()).width("x")); } else { @@ -663,6 +671,7 @@ void SendCoinsDialog::updateSmartFeeLabel() std::max(feeRate.GetFeePerK(), CWallet::GetRequiredFee(1000))) + "/kB"); ui->labelSmartFee2->hide(); ui->labelFeeEstimation->setText(tr("Estimated to begin confirmation within %n block(s).", "", estimateFoundAtBlocks)); + ui->fallbackFeeWarningLabel->setVisible(false); } updateFeeMinimizedLabel(); @@ -810,21 +819,6 @@ void SendCoinsDialog::coinControlUpdateLabels() if (!model || !model->getOptionsModel()) return; - if (model->getOptionsModel()->getCoinControlFeatures()) - { - // enable minimum absolute fee UI controls - ui->radioCustomAtLeast->setVisible(true); - - // only enable the feature if inputs are selected - ui->radioCustomAtLeast->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked() &&CoinControlDialog::coinControl->HasSelected()); - } - else - { - // in case coin control is disabled (=default), hide minimum absolute fee UI controls - ui->radioCustomAtLeast->setVisible(false); - return; - } - // set pay amounts CoinControlDialog::payAmounts.clear(); CoinControlDialog::fSubtractFeeFromAmount = false; diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index a402edc961..a932f129be 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -100,13 +100,14 @@ Q_SIGNALS: }; +#define SEND_CONFIRM_DELAY 3 class SendConfirmationDialog : public QMessageBox { Q_OBJECT public: - SendConfirmationDialog(const QString &title, const QString &text, int secDelay = 0, QWidget *parent = 0); + SendConfirmationDialog(const QString &title, const QString &text, int secDelay = SEND_CONFIRM_DELAY, QWidget *parent = 0); int exec(); private Q_SLOTS: diff --git a/src/qt/test/paymentservertests.cpp b/src/qt/test/paymentservertests.cpp index 84ccfea730..08a76c7d49 100644 --- a/src/qt/test/paymentservertests.cpp +++ b/src/qt/test/paymentservertests.cpp @@ -142,7 +142,7 @@ void PaymentServerTests::paymentServerTests() byteArray = QByteArray((const char*)&data[0], data.size()); r.paymentRequest.parse(byteArray); // Ensure the request is initialized, because network "main" is default, even for - // uninizialized payment requests and that will fail our test here. + // uninitialized payment requests and that will fail our test here. QVERIFY(r.paymentRequest.IsInitialized()); QCOMPARE(PaymentServer::verifyNetwork(r.paymentRequest.getDetails()), false); diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index bd496f149c..dada689731 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -6,6 +6,7 @@ #include "chainparams.h" #include "consensus/validation.h" +#include "fs.h" #include "validation.h" #include "rpc/register.h" #include "rpc/server.h" @@ -17,8 +18,6 @@ #include <QDir> #include <QtGlobal> -#include <boost/filesystem.hpp> - static UniValue rpcNestedTest_rpc(const JSONRPCRequest& request) { if (request.fHelp) { @@ -148,9 +147,13 @@ void RPCNestedTests::rpcNestedTests() QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest(abc,,)"), std::runtime_error); //don't tollerate empty arguments when using , #endif + UnloadBlockIndex(); delete pcoinsTip; + pcoinsTip = nullptr; delete pcoinsdbview; + pcoinsdbview = nullptr; delete pblocktree; + pblocktree = nullptr; - boost::filesystem::remove_all(boost::filesystem::path(path)); + fs::remove_all(fs::path(path)); } diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index d44d711315..cae18f41a5 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -7,7 +7,6 @@ #endif #include "chainparams.h" -#include "key.h" #include "rpcnestedtests.h" #include "util.h" #include "uritests.h" @@ -15,20 +14,34 @@ #ifdef ENABLE_WALLET #include "paymentservertests.h" +#include "wallettests.h" #endif -#include <QCoreApplication> +#include <QApplication> #include <QObject> #include <QTest> #include <openssl/ssl.h> -#if defined(QT_STATICPLUGIN) && QT_VERSION < 0x050000 +#if defined(QT_STATICPLUGIN) #include <QtPlugin> +#if QT_VERSION < 0x050000 Q_IMPORT_PLUGIN(qcncodecs) Q_IMPORT_PLUGIN(qjpcodecs) Q_IMPORT_PLUGIN(qtwcodecs) Q_IMPORT_PLUGIN(qkrcodecs) +#else +#if defined(QT_QPA_PLATFORM_MINIMAL) +Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin); +#endif +#if defined(QT_QPA_PLATFORM_XCB) +Q_IMPORT_PLUGIN(QXcbIntegrationPlugin); +#elif defined(QT_QPA_PLATFORM_WINDOWS) +Q_IMPORT_PLUGIN(QWindowsIntegrationPlugin); +#elif defined(QT_QPA_PLATFORM_COCOA) +Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin); +#endif +#endif #endif extern void noui_connect(); @@ -36,7 +49,6 @@ extern void noui_connect(); // This is all you need to run all the tests int main(int argc, char *argv[]) { - ECC_Start(); SetupEnvironment(); SetupNetworking(); SelectParams(CBaseChainParams::MAIN); @@ -44,28 +56,42 @@ int main(int argc, char *argv[]) bool fInvalid = false; + // Prefer the "minimal" platform for the test instead of the normal default + // platform ("xcb", "windows", or "cocoa") so tests can't unintentially + // interfere with any background GUIs and don't require extra resources. + setenv("QT_QPA_PLATFORM", "minimal", 0); + // Don't remove this, it's needed to access - // QCoreApplication:: in the tests - QCoreApplication app(argc, argv); + // QApplication:: and QCoreApplication:: in the tests + QApplication app(argc, argv); app.setApplicationName("Bitcoin-Qt-test"); SSL_library_init(); URITests test1; - if (QTest::qExec(&test1) != 0) + if (QTest::qExec(&test1) != 0) { fInvalid = true; + } #ifdef ENABLE_WALLET PaymentServerTests test2; - if (QTest::qExec(&test2) != 0) + if (QTest::qExec(&test2) != 0) { fInvalid = true; + } #endif RPCNestedTests test3; - if (QTest::qExec(&test3) != 0) + if (QTest::qExec(&test3) != 0) { fInvalid = true; + } CompatTests test4; - if (QTest::qExec(&test4) != 0) + if (QTest::qExec(&test4) != 0) { fInvalid = true; + } +#ifdef ENABLE_WALLET + WalletTests test5; + if (QTest::qExec(&test5) != 0) { + fInvalid = true; + } +#endif - ECC_Stop(); return fInvalid; } diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp new file mode 100644 index 0000000000..ff1eb59f16 --- /dev/null +++ b/src/qt/test/wallettests.cpp @@ -0,0 +1,196 @@ +#include "wallettests.h" + +#include "qt/bitcoinamountfield.h" +#include "qt/callback.h" +#include "qt/optionsmodel.h" +#include "qt/platformstyle.h" +#include "qt/qvalidatedlineedit.h" +#include "qt/sendcoinsdialog.h" +#include "qt/sendcoinsentry.h" +#include "qt/transactiontablemodel.h" +#include "qt/transactionview.h" +#include "qt/walletmodel.h" +#include "test/test_bitcoin.h" +#include "validation.h" +#include "wallet/wallet.h" + +#include <QAbstractButton> +#include <QAction> +#include <QApplication> +#include <QCheckBox> +#include <QPushButton> +#include <QTimer> +#include <QVBoxLayout> + +namespace +{ +//! Press "Ok" button in message box dialog. +void ConfirmMessage(QString* text = nullptr) +{ + QTimer::singleShot(0, makeCallback([text](Callback* callback) { + for (QWidget* widget : QApplication::topLevelWidgets()) { + if (widget->inherits("QMessageBox")) { + QMessageBox* messageBox = qobject_cast<QMessageBox*>(widget); + if (text) *text = messageBox->text(); + messageBox->defaultButton()->click(); + } + } + delete callback; + }), SLOT(call())); +} + +//! Press "Yes" or "Cancel" buttons in modal send confirmation dialog. +void ConfirmSend(QString* text = nullptr, bool cancel = false) +{ + QTimer::singleShot(0, makeCallback([text, cancel](Callback* callback) { + for (QWidget* widget : QApplication::topLevelWidgets()) { + if (widget->inherits("SendConfirmationDialog")) { + SendConfirmationDialog* dialog = qobject_cast<SendConfirmationDialog*>(widget); + if (text) *text = dialog->text(); + QAbstractButton* button = dialog->button(cancel ? QMessageBox::Cancel : QMessageBox::Yes); + button->setEnabled(true); + button->click(); + } + } + delete callback; + }), SLOT(call())); +} + +//! Send coins to address and return txid. +uint256 SendCoins(CWallet& wallet, SendCoinsDialog& sendCoinsDialog, const CBitcoinAddress& address, CAmount amount, bool rbf) +{ + QVBoxLayout* entries = sendCoinsDialog.findChild<QVBoxLayout*>("entries"); + SendCoinsEntry* entry = qobject_cast<SendCoinsEntry*>(entries->itemAt(0)->widget()); + entry->findChild<QValidatedLineEdit*>("payTo")->setText(QString::fromStdString(address.ToString())); + entry->findChild<BitcoinAmountField*>("payAmount")->setValue(amount); + sendCoinsDialog.findChild<QFrame*>("frameFee") + ->findChild<QFrame*>("frameFeeSelection") + ->findChild<QCheckBox*>("optInRBF") + ->setCheckState(rbf ? Qt::Checked : Qt::Unchecked); + uint256 txid; + boost::signals2::scoped_connection c(wallet.NotifyTransactionChanged.connect([&txid](CWallet*, const uint256& hash, ChangeType status) { + if (status == CT_NEW) txid = hash; + })); + ConfirmSend(); + QMetaObject::invokeMethod(&sendCoinsDialog, "on_sendButton_clicked"); + return txid; +} + +//! Find index of txid in transaction list. +QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) +{ + QString hash = QString::fromStdString(txid.ToString()); + int rows = model.rowCount({}); + for (int row = 0; row < rows; ++row) { + QModelIndex index = model.index(row, 0, {}); + if (model.data(index, TransactionTableModel::TxHashRole) == hash) { + return index; + } + } + return {}; +} + +//! Request context menu (call method that is public in qt5, but protected in qt4). +void RequestContextMenu(QWidget* widget) +{ + class Qt4Hack : public QWidget + { + public: + using QWidget::customContextMenuRequested; + }; + static_cast<Qt4Hack*>(widget)->customContextMenuRequested({}); +} + +//! Invoke bumpfee on txid and check results. +void BumpFee(TransactionView& view, const uint256& txid, bool expectDisabled, std::string expectError, bool cancel) +{ + QTableView* table = view.findChild<QTableView*>("transactionView"); + QModelIndex index = FindTx(*table->selectionModel()->model(), txid); + QVERIFY2(index.isValid(), "Could not find BumpFee txid"); + + // Select row in table, invoke context menu, and make sure bumpfee action is + // enabled or disabled as expected. + QAction* action = view.findChild<QAction*>("bumpFeeAction"); + table->selectionModel()->select(index, QItemSelectionModel::ClearAndSelect | QItemSelectionModel::Rows); + action->setEnabled(expectDisabled); + RequestContextMenu(table); + QCOMPARE(action->isEnabled(), !expectDisabled); + + action->setEnabled(true); + QString text; + if (expectError.empty()) { + ConfirmSend(&text, cancel); + } else { + ConfirmMessage(&text); + } + action->trigger(); + QVERIFY(text.indexOf(QString::fromStdString(expectError)) != -1); +} + +//! Simple qt wallet tests. +// +// Test widgets can be debugged interactively calling show() on them and +// manually running the event loop, e.g.: +// +// sendCoinsDialog.show(); +// QEventLoop().exec(); +// +// This also requires overriding the default minimal Qt platform: +// +// src/qt/test/test_bitcoin-qt -platform xcb # Linux +// src/qt/test/test_bitcoin-qt -platform windows # Windows +// src/qt/test/test_bitcoin-qt -platform cocoa # macOS +void TestSendCoins() +{ + // Set up wallet and chain with 105 blocks (5 mature blocks for spending). + TestChain100Setup test; + for (int i = 0; i < 5; ++i) { + test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); + } + bitdb.MakeMock(); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); + CWallet wallet(std::move(dbw)); + bool firstRun; + wallet.LoadWallet(firstRun); + { + LOCK(wallet.cs_wallet); + wallet.SetAddressBook(test.coinbaseKey.GetPubKey().GetID(), "", "receive"); + wallet.AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + } + wallet.ScanForWalletTransactions(chainActive.Genesis(), true); + wallet.SetBroadcastTransactions(true); + + // Create widgets for sending coins and listing transactions. + std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); + SendCoinsDialog sendCoinsDialog(platformStyle.get()); + TransactionView transactionView(platformStyle.get()); + OptionsModel optionsModel; + WalletModel walletModel(platformStyle.get(), &wallet, &optionsModel); + sendCoinsDialog.setModel(&walletModel); + transactionView.setModel(&walletModel); + + // Send two transactions, and verify they are added to transaction list. + TransactionTableModel* transactionTableModel = walletModel.getTransactionTableModel(); + QCOMPARE(transactionTableModel->rowCount({}), 105); + uint256 txid1 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 5 * COIN, false /* rbf */); + uint256 txid2 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 10 * COIN, true /* rbf */); + QCOMPARE(transactionTableModel->rowCount({}), 107); + QVERIFY(FindTx(*transactionTableModel, txid1).isValid()); + QVERIFY(FindTx(*transactionTableModel, txid2).isValid()); + + // Call bumpfee. Test disabled, canceled, enabled, then failing cases. + BumpFee(transactionView, txid1, true /* expect disabled */, "not BIP 125 replaceable" /* expected error */, false /* cancel */); + BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, true /* cancel */); + BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, false /* cancel */); + BumpFee(transactionView, txid2, true /* expect disabled */, "already bumped" /* expected error */, false /* cancel */); + + bitdb.Flush(true); + bitdb.Reset(); +} + +} + +void WalletTests::walletTests() +{ + TestSendCoins(); +} diff --git a/src/qt/test/wallettests.h b/src/qt/test/wallettests.h new file mode 100644 index 0000000000..342f7916c3 --- /dev/null +++ b/src/qt/test/wallettests.h @@ -0,0 +1,15 @@ +#ifndef BITCOIN_QT_TEST_WALLETTESTS_H +#define BITCOIN_QT_TEST_WALLETTESTS_H + +#include <QObject> +#include <QTest> + +class WalletTests : public QObject +{ + Q_OBJECT + +private Q_SLOTS: + void walletTests(); +}; + +#endif // BITCOIN_QT_TEST_WALLETTESTS_H diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 7e16cc9dd4..233fc08772 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -273,7 +273,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco // // Debug view // - if (fDebug) + if (logCategories != BCLog::NONE) { strHTML += "<hr><br>" + tr("Debug information") + "<br><br>"; BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) @@ -293,13 +293,12 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco { COutPoint prevout = txin.prevout; - CCoins prev; - if(pcoinsTip->GetCoins(prevout.hash, prev)) + Coin prev; + if(pcoinsTip->GetCoin(prevout, prev)) { - if (prevout.n < prev.vout.size()) { strHTML += "<li>"; - const CTxOut &vout = prev.vout[prevout.n]; + const CTxOut &vout = prev.out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index a9d9b6887e..0090b0c74b 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -18,14 +18,8 @@ */ bool TransactionRecord::showTransaction(const CWalletTx &wtx) { - if (wtx.IsCoinBase()) - { - // Ensures we show generated coins / mined transactions at depth 1 - if (!wtx.IsInMainChain()) - { - return false; - } - } + // There are currently no cases where we hide transactions, but + // we may want to use this in the future for things like RBF. return true; } @@ -252,13 +246,13 @@ void TransactionRecord::updateStatus(const CWalletTx &wtx) status.status = TransactionStatus::Confirmed; } } - + status.needsUpdate = false; } bool TransactionRecord::statusUpdateNeeded() { AssertLockHeld(cs_main); - return status.cur_num_blocks != chainActive.Height(); + return status.cur_num_blocks != chainActive.Height() || status.needsUpdate; } QString TransactionRecord::getTxID() const diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index 5aabbbffa8..59f681224f 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -61,6 +61,8 @@ public: /** Current number of blocks (to know whether cached status is still valid) */ int cur_num_blocks; + + bool needsUpdate; }; /** UI model for a transaction. A core transaction can be represented by multiple UI transactions if it has diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 61466c8ed1..f27abc2104 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -168,6 +168,10 @@ public: case CT_UPDATED: // Miscellaneous updates -- nothing to do, status update will take care of this, and is only computed for // visible transactions. + for (int i = lowerIndex; i < upperIndex; i++) { + TransactionRecord *rec = &cachedWallet[i]; + rec->status.needsUpdate = true; + } break; } } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 30f4db9450..e3e070b27f 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -11,6 +11,7 @@ #include "guiutil.h" #include "optionsmodel.h" #include "platformstyle.h" +#include "sendcoinsdialog.h" #include "transactiondescdialog.h" #include "transactionfilterproxy.h" #include "transactionrecord.h" @@ -37,7 +38,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *parent) : QWidget(parent), model(0), transactionProxyModel(0), - transactionView(0), abandonAction(0), columnResizingFixer(0) + transactionView(0), abandonAction(0), bumpFeeAction(0), columnResizingFixer(0) { // Build filter row setContentsMargins(0,0,0,0); @@ -135,9 +136,12 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa view->installEventFilter(this); transactionView = view; + transactionView->setObjectName("transactionView"); // Actions abandonAction = new QAction(tr("Abandon transaction"), this); + bumpFeeAction = new QAction(tr("Increase transaction fee"), this); + bumpFeeAction->setObjectName("bumpFeeAction"); QAction *copyAddressAction = new QAction(tr("Copy address"), this); QAction *copyLabelAction = new QAction(tr("Copy label"), this); QAction *copyAmountAction = new QAction(tr("Copy amount"), this); @@ -148,6 +152,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa QAction *showDetailsAction = new QAction(tr("Show transaction details"), this); contextMenu = new QMenu(this); + contextMenu->setObjectName("contextMenu"); contextMenu->addAction(copyAddressAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(copyAmountAction); @@ -156,6 +161,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa contextMenu->addAction(copyTxPlainText); contextMenu->addAction(showDetailsAction); contextMenu->addSeparator(); + contextMenu->addAction(bumpFeeAction); contextMenu->addAction(abandonAction); contextMenu->addAction(editLabelAction); @@ -173,6 +179,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa connect(view, SIGNAL(doubleClicked(QModelIndex)), this, SIGNAL(doubleClicked(QModelIndex))); connect(view, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(contextualMenu(QPoint))); + connect(bumpFeeAction, SIGNAL(triggered()), this, SLOT(bumpFee())); connect(abandonAction, SIGNAL(triggered()), this, SLOT(abandonTx())); connect(copyAddressAction, SIGNAL(triggered()), this, SLOT(copyAddress())); connect(copyLabelAction, SIGNAL(triggered()), this, SLOT(copyLabel())); @@ -372,10 +379,11 @@ void TransactionView::contextualMenu(const QPoint &point) uint256 hash; hash.SetHex(selection.at(0).data(TransactionTableModel::TxHashRole).toString().toStdString()); abandonAction->setEnabled(model->transactionCanBeAbandoned(hash)); + bumpFeeAction->setEnabled(model->transactionCanBeBumped(hash)); if(index.isValid()) { - contextMenu->exec(QCursor::pos()); + contextMenu->popup(transactionView->viewport()->mapToGlobal(point)); } } @@ -397,6 +405,24 @@ void TransactionView::abandonTx() model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, false); } +void TransactionView::bumpFee() +{ + if(!transactionView || !transactionView->selectionModel()) + return; + QModelIndexList selection = transactionView->selectionModel()->selectedRows(0); + + // get the hash from the TxHashRole (QVariant / QString) + uint256 hash; + QString hashQStr = selection.at(0).data(TransactionTableModel::TxHashRole).toString(); + hash.SetHex(hashQStr.toStdString()); + + // Bump tx fee over the walletModel + if (model->bumpFee(hash)) { + // Update the table + model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, true); + } +} + void TransactionView::copyAddress() { GUIUtil::copyEntryData(transactionView, 0, TransactionTableModel::AddressRole); diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 595701cdd9..52e57cae4c 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -76,6 +76,7 @@ private: QDateTimeEdit *dateFrom; QDateTimeEdit *dateTo; QAction *abandonAction; + QAction *bumpFeeAction; QWidget *createDateRangeWidget(); @@ -99,6 +100,7 @@ private Q_SLOTS: void openThirdPartyTxUrl(QString url); void updateWatchOnlyColumn(bool fHaveWatchOnly); void abandonTx(); + void bumpFee(); Q_SIGNALS: void doubleClicked(const QModelIndex&); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 0a5a7c3e9f..8df0e481bd 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -8,23 +8,29 @@ #include "consensus/validation.h" #include "guiconstants.h" #include "guiutil.h" +#include "optionsmodel.h" #include "paymentserver.h" #include "recentrequeststablemodel.h" +#include "sendcoinsdialog.h" #include "transactiontablemodel.h" #include "base58.h" +#include "chain.h" #include "keystore.h" #include "validation.h" #include "net.h" // for g_connman +#include "policy/rbf.h" #include "sync.h" #include "ui_interface.h" #include "util.h" // for GetBoolArg +#include "wallet/feebumper.h" #include "wallet/wallet.h" #include "wallet/walletdb.h" // for BackupWallet #include <stdint.h> #include <QDebug> +#include <QMessageBox> #include <QSet> #include <QTimer> @@ -62,14 +68,7 @@ CAmount WalletModel::getBalance(const CCoinControl *coinControl) const { if (coinControl) { - CAmount nBalance = 0; - std::vector<COutput> vCoins; - wallet->AvailableCoins(vCoins, true, coinControl); - BOOST_FOREACH(const COutput& out, vCoins) - if(out.fSpendable) - nBalance += out.tx->tx->vout[out.i].nValue; - - return nBalance; + return wallet->GetAvailableBalance(coinControl); } return wallet->GetBalance(); @@ -580,7 +579,7 @@ void WalletModel::getOutputs(const std::vector<COutPoint>& vOutpoints, std::vect if (!wallet->mapWallet.count(outpoint.hash)) continue; int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); if (nDepth < 0) continue; - COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true, true); + COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true /* spendable */, true /* solvable */, true /* safe */); vOutputs.push_back(out); } } @@ -594,38 +593,11 @@ bool WalletModel::isSpent(const COutPoint& outpoint) const // AvailableCoins + LockedCoins grouped by wallet address (put change in one group with wallet address) void WalletModel::listCoins(std::map<QString, std::vector<COutput> >& mapCoins) const { - std::vector<COutput> vCoins; - wallet->AvailableCoins(vCoins); - - LOCK2(cs_main, wallet->cs_wallet); // ListLockedCoins, mapWallet - std::vector<COutPoint> vLockedCoins; - wallet->ListLockedCoins(vLockedCoins); - - // add locked coins - BOOST_FOREACH(const COutPoint& outpoint, vLockedCoins) - { - if (!wallet->mapWallet.count(outpoint.hash)) continue; - int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); - if (nDepth < 0) continue; - COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true, true); - if (outpoint.n < out.tx->tx->vout.size() && wallet->IsMine(out.tx->tx->vout[outpoint.n]) == ISMINE_SPENDABLE) - vCoins.push_back(out); - } - - BOOST_FOREACH(const COutput& out, vCoins) - { - COutput cout = out; - - while (wallet->IsChange(cout.tx->tx->vout[cout.i]) && cout.tx->tx->vin.size() > 0 && wallet->IsMine(cout.tx->tx->vin[0])) - { - if (!wallet->mapWallet.count(cout.tx->tx->vin[0].prevout.hash)) break; - cout = COutput(&wallet->mapWallet[cout.tx->tx->vin[0].prevout.hash], cout.tx->tx->vin[0].prevout.n, 0, true, true); + for (auto& group : wallet->ListCoins()) { + auto& resultGroup = mapCoins[QString::fromStdString(CBitcoinAddress(group.first).ToString())]; + for (auto& coin : group.second) { + resultGroup.emplace_back(std::move(coin)); } - - CTxDestination address; - if(!out.fSpendable || !ExtractDestination(cout.tx->tx->vout[cout.i].scriptPubKey, address)) - continue; - mapCoins[QString::fromStdString(CBitcoinAddress(address).ToString())].push_back(out); } } @@ -655,11 +627,7 @@ void WalletModel::listLockedCoins(std::vector<COutPoint>& vOutpts) void WalletModel::loadReceiveRequests(std::vector<std::string>& vReceiveRequests) { - LOCK(wallet->cs_wallet); - BOOST_FOREACH(const PAIRTYPE(CTxDestination, CAddressBookData)& item, wallet->mapAddressBook) - BOOST_FOREACH(const PAIRTYPE(std::string, std::string)& item2, item.second.destdata) - if (item2.first.size() > 2 && item2.first.substr(0,2) == "rr") // receive request - vReceiveRequests.push_back(item2.second); + vReceiveRequests = wallet->GetDestValues("rr"); // receive request } bool WalletModel::saveReceiveRequest(const std::string &sAddress, const int64_t nId, const std::string &sRequest) @@ -679,11 +647,7 @@ bool WalletModel::saveReceiveRequest(const std::string &sAddress, const int64_t bool WalletModel::transactionCanBeAbandoned(uint256 hash) const { - LOCK2(cs_main, wallet->cs_wallet); - const CWalletTx *wtx = wallet->GetWalletTx(hash); - if (!wtx || wtx->isAbandoned() || wtx->GetDepthInMainChain() > 0 || wtx->InMempool()) - return false; - return true; + return wallet->TransactionCanBeAbandoned(hash); } bool WalletModel::abandonTransaction(uint256 hash) const @@ -692,6 +656,84 @@ bool WalletModel::abandonTransaction(uint256 hash) const return wallet->AbandonTransaction(hash); } +bool WalletModel::transactionCanBeBumped(uint256 hash) const +{ + LOCK2(cs_main, wallet->cs_wallet); + const CWalletTx *wtx = wallet->GetWalletTx(hash); + return wtx && SignalsOptInRBF(*wtx) && !wtx->mapValue.count("replaced_by_txid"); +} + +bool WalletModel::bumpFee(uint256 hash) +{ + std::unique_ptr<CFeeBumper> feeBump; + { + LOCK2(cs_main, wallet->cs_wallet); + feeBump.reset(new CFeeBumper(wallet, hash, nTxConfirmTarget, false, 0, true)); + } + if (feeBump->getResult() != BumpFeeResult::OK) + { + QMessageBox::critical(0, tr("Fee bump error"), tr("Increasing transaction fee failed") + "<br />(" + + (feeBump->getErrors().size() ? QString::fromStdString(feeBump->getErrors()[0]) : "") +")"); + return false; + } + + // allow a user based fee verification + QString questionString = tr("Do you want to increase the fee?"); + questionString.append("<br />"); + CAmount oldFee = feeBump->getOldFee(); + CAmount newFee = feeBump->getNewFee(); + questionString.append("<table style=\"text-align: left;\">"); + questionString.append("<tr><td>"); + questionString.append(tr("Current fee:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), oldFee)); + questionString.append("</td></tr><tr><td>"); + questionString.append(tr("Increase:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee - oldFee)); + questionString.append("</td></tr><tr><td>"); + questionString.append(tr("New fee:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee)); + questionString.append("</td></tr></table>"); + SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); + confirmationDialog.exec(); + QMessageBox::StandardButton retval = (QMessageBox::StandardButton)confirmationDialog.result(); + + // cancel sign&broadcast if users doesn't want to bump the fee + if (retval != QMessageBox::Yes) { + return false; + } + + WalletModel::UnlockContext ctx(requestUnlock()); + if(!ctx.isValid()) + { + return false; + } + + // sign bumped transaction + bool res = false; + { + LOCK2(cs_main, wallet->cs_wallet); + res = feeBump->signTransaction(wallet); + } + if (!res) { + QMessageBox::critical(0, tr("Fee bump error"), tr("Can't sign transaction.")); + return false; + } + // commit the bumped transaction + { + LOCK2(cs_main, wallet->cs_wallet); + res = feeBump->commit(wallet); + } + if(!res) { + QMessageBox::critical(0, tr("Fee bump error"), tr("Could not commit transaction") + "<br />(" + + QString::fromStdString(feeBump->getErrors()[0])+")"); + return false; + } + return true; +} + bool WalletModel::isWalletEnabled() { return !GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET); @@ -706,3 +748,8 @@ int WalletModel::getDefaultConfirmTarget() const { return nTxConfirmTarget; } + +bool WalletModel::getDefaultWalletRbf() const +{ + return fWalletRbf; +} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index cd7585635f..16b0caed4e 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -207,12 +207,17 @@ public: bool transactionCanBeAbandoned(uint256 hash) const; bool abandonTransaction(uint256 hash) const; + bool transactionCanBeBumped(uint256 hash) const; + bool bumpFee(uint256 hash); + static bool isWalletEnabled(); bool hdEnabled() const; int getDefaultConfirmTarget() const; + bool getDefaultWalletRbf() const; + private: CWallet *wallet; bool fHaveWatchOnly; diff --git a/src/random.cpp b/src/random.cpp index 6634019bea..de7553c825 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -16,11 +16,26 @@ #include <stdlib.h> #include <limits> +#include <chrono> +#include <thread> #ifndef WIN32 #include <sys/time.h> #endif +#ifdef HAVE_SYS_GETRANDOM +#include <sys/syscall.h> +#include <linux/random.h> +#endif +#ifdef HAVE_GETENTROPY +#include <unistd.h> +#endif +#ifdef HAVE_SYSCTL_ARND +#include <sys/sysctl.h> +#endif + +#include <mutex> + #include <openssl/err.h> #include <openssl/rand.h> @@ -32,15 +47,22 @@ static void RandFailure() static inline int64_t GetPerformanceCounter() { - int64_t nCounter = 0; -#ifdef WIN32 - QueryPerformanceCounter((LARGE_INTEGER*)&nCounter); + // Read the hardware time stamp counter when available. + // See https://en.wikipedia.org/wiki/Time_Stamp_Counter for more information. +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) + return __rdtsc(); +#elif !defined(_MSC_VER) && defined(__i386__) + uint64_t r = 0; + __asm__ volatile ("rdtsc" : "=A"(r)); // Constrain the r variable to the eax:edx pair. + return r; +#elif !defined(_MSC_VER) && (defined(__x86_64__) || defined(__amd64__)) + uint64_t r1 = 0, r2 = 0; + __asm__ volatile ("rdtsc" : "=a"(r1), "=d"(r2)); // Constrain r1 to rax and r2 to rdx. + return (r2 << 32) | r1; #else - timeval t; - gettimeofday(&t, NULL); - nCounter = (int64_t)(t.tv_sec * 1000000 + t.tv_usec); + // Fall back to using C++11 clock (usually microsecond or nanosecond precision) + return std::chrono::high_resolution_clock::now().time_since_epoch().count(); #endif - return nCounter; } void RandAddSeed() @@ -80,7 +102,7 @@ static void RandAddSeedPerfmon() if (ret == ERROR_SUCCESS) { RAND_add(vData.data(), nSize, nSize / 100.0); memory_cleanse(vData.data(), nSize); - LogPrint("rand", "%s: %lu bytes\n", __func__, nSize); + LogPrint(BCLog::RAND, "%s: %lu bytes\n", __func__, nSize); } else { static bool warned = false; // Warn only once if (!warned) { @@ -91,34 +113,86 @@ static void RandAddSeedPerfmon() #endif } +#ifndef WIN32 +/** Fallback: get 32 bytes of system entropy from /dev/urandom. The most + * compatible way to get cryptographic randomness on UNIX-ish platforms. + */ +void GetDevURandom(unsigned char *ent32) +{ + int f = open("/dev/urandom", O_RDONLY); + if (f == -1) { + RandFailure(); + } + int have = 0; + do { + ssize_t n = read(f, ent32 + have, NUM_OS_RANDOM_BYTES - have); + if (n <= 0 || n + have > NUM_OS_RANDOM_BYTES) { + RandFailure(); + } + have += n; + } while (have < NUM_OS_RANDOM_BYTES); + close(f); +} +#endif + /** Get 32 bytes of system entropy. */ -static void GetOSRand(unsigned char *ent32) +void GetOSRand(unsigned char *ent32) { -#ifdef WIN32 +#if defined(WIN32) HCRYPTPROV hProvider; int ret = CryptAcquireContextW(&hProvider, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT); if (!ret) { RandFailure(); } - ret = CryptGenRandom(hProvider, 32, ent32); + ret = CryptGenRandom(hProvider, NUM_OS_RANDOM_BYTES, ent32); if (!ret) { RandFailure(); } CryptReleaseContext(hProvider, 0); -#else - int f = open("/dev/urandom", O_RDONLY); - if (f == -1) { +#elif defined(HAVE_SYS_GETRANDOM) + /* Linux. From the getrandom(2) man page: + * "If the urandom source has been initialized, reads of up to 256 bytes + * will always return as many bytes as requested and will not be + * interrupted by signals." + */ + int rv = syscall(SYS_getrandom, ent32, NUM_OS_RANDOM_BYTES, 0); + if (rv != NUM_OS_RANDOM_BYTES) { + if (rv < 0 && errno == ENOSYS) { + /* Fallback for kernel <3.17: the return value will be -1 and errno + * ENOSYS if the syscall is not available, in that case fall back + * to /dev/urandom. + */ + GetDevURandom(ent32); + } else { + RandFailure(); + } + } +#elif defined(HAVE_GETENTROPY) + /* On OpenBSD this can return up to 256 bytes of entropy, will return an + * error if more are requested. + * The call cannot return less than the requested number of bytes. + */ + if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); } +#elif defined(HAVE_SYSCTL_ARND) + /* FreeBSD and similar. It is possible for the call to return less + * bytes than requested, so need to read in a loop. + */ + static const int name[2] = {CTL_KERN, KERN_ARND}; int have = 0; do { - ssize_t n = read(f, ent32 + have, 32 - have); - if (n <= 0 || n + have > 32) { + size_t len = NUM_OS_RANDOM_BYTES - have; + if (sysctl(name, ARRAYLEN(name), ent32 + have, &len, NULL, 0) != 0) { RandFailure(); } - have += n; - } while (have < 32); - close(f); + have += len; + } while (have < NUM_OS_RANDOM_BYTES); +#else + /* Fall back to /dev/urandom if there is no specific method implemented to + * get system entropy for this OS. + */ + GetDevURandom(ent32); #endif } @@ -129,6 +203,43 @@ void GetRandBytes(unsigned char* buf, int num) } } +static void AddDataToRng(void* data, size_t len); + +void RandAddSeedSleep() +{ + int64_t nPerfCounter1 = GetPerformanceCounter(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + int64_t nPerfCounter2 = GetPerformanceCounter(); + + // Combine with and update state + AddDataToRng(&nPerfCounter1, sizeof(nPerfCounter1)); + AddDataToRng(&nPerfCounter2, sizeof(nPerfCounter2)); + + memory_cleanse(&nPerfCounter1, sizeof(nPerfCounter1)); + memory_cleanse(&nPerfCounter2, sizeof(nPerfCounter2)); +} + + +static std::mutex cs_rng_state; +static unsigned char rng_state[32] = {0}; +static uint64_t rng_counter = 0; + +static void AddDataToRng(void* data, size_t len) { + CSHA512 hasher; + hasher.Write((const unsigned char*)&len, sizeof(len)); + hasher.Write((const unsigned char*)data, len); + unsigned char buf[64]; + { + std::unique_lock<std::mutex> lock(cs_rng_state); + hasher.Write(rng_state, sizeof(rng_state)); + hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); + ++rng_counter; + hasher.Finalize(buf); + memcpy(rng_state, buf + 32, 32); + } + memory_cleanse(buf, 64); +} + void GetStrongRandBytes(unsigned char* out, int num) { assert(num <= 32); @@ -144,8 +255,17 @@ void GetStrongRandBytes(unsigned char* out, int num) GetOSRand(buf); hasher.Write(buf, 32); + // Combine with and update state + { + std::unique_lock<std::mutex> lock(cs_rng_state); + hasher.Write(rng_state, sizeof(rng_state)); + hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); + ++rng_counter; + hasher.Finalize(buf); + memcpy(rng_state, buf + 32, 32); + } + // Produce output - hasher.Finalize(buf); memcpy(out, buf, num); memory_cleanse(buf, 64); } @@ -177,21 +297,67 @@ uint256 GetRandHash() return hash; } -FastRandomContext::FastRandomContext(bool fDeterministic) +void FastRandomContext::RandomSeed() { - // The seed values have some unlikely fixed points which we avoid. - if (fDeterministic) { - Rz = Rw = 11; - } else { - uint32_t tmp; - do { - GetRandBytes((unsigned char*)&tmp, 4); - } while (tmp == 0 || tmp == 0x9068ffffU); - Rz = tmp; - do { - GetRandBytes((unsigned char*)&tmp, 4); - } while (tmp == 0 || tmp == 0x464fffffU); - Rw = tmp; - } + uint256 seed = GetRandHash(); + rng.SetKey(seed.begin(), 32); + requires_seed = false; +} + +FastRandomContext::FastRandomContext(const uint256& seed) : requires_seed(false), bytebuf_size(0), bitbuf_size(0) +{ + rng.SetKey(seed.begin(), 32); } +bool Random_SanityCheck() +{ + uint64_t start = GetPerformanceCounter(); + + /* This does not measure the quality of randomness, but it does test that + * OSRandom() overwrites all 32 bytes of the output given a maximum + * number of tries. + */ + static const ssize_t MAX_TRIES = 1024; + uint8_t data[NUM_OS_RANDOM_BYTES]; + bool overwritten[NUM_OS_RANDOM_BYTES] = {}; /* Tracks which bytes have been overwritten at least once */ + int num_overwritten; + int tries = 0; + /* Loop until all bytes have been overwritten at least once, or max number tries reached */ + do { + memset(data, 0, NUM_OS_RANDOM_BYTES); + GetOSRand(data); + for (int x=0; x < NUM_OS_RANDOM_BYTES; ++x) { + overwritten[x] |= (data[x] != 0); + } + + num_overwritten = 0; + for (int x=0; x < NUM_OS_RANDOM_BYTES; ++x) { + if (overwritten[x]) { + num_overwritten += 1; + } + } + + tries += 1; + } while (num_overwritten < NUM_OS_RANDOM_BYTES && tries < MAX_TRIES); + if (num_overwritten != NUM_OS_RANDOM_BYTES) return false; /* If this failed, bailed out after too many tries */ + + // Check that GetPerformanceCounter increases at least during a GetOSRand() call + 1ms sleep. + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + uint64_t stop = GetPerformanceCounter(); + if (stop == start) return false; + + // We called GetPerformanceCounter. Use it as entropy. + RAND_add((const unsigned char*)&start, sizeof(start), 1); + RAND_add((const unsigned char*)&stop, sizeof(stop), 1); + + return true; +} + +FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) +{ + if (!fDeterministic) { + return; + } + uint256 seed; + rng.SetKey(seed.begin(), 32); +} diff --git a/src/random.h b/src/random.h index 664f030eba..6a63d57429 100644 --- a/src/random.h +++ b/src/random.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_RANDOM_H #define BITCOIN_RANDOM_H +#include "crypto/chacha20.h" +#include "crypto/common.h" #include "uint256.h" #include <stdint.h> @@ -22,6 +24,13 @@ int GetRandInt(int nMax); uint256 GetRandHash(); /** + * Add a little bit of randomness to the output of GetStrongRangBytes. + * This sleeps for a millisecond, so should only be called when there is + * no other work to be done. + */ +void RandAddSeedSleep(); + +/** * Function to gather random data from multiple sources, failing whenever any * of those source fail to provide a result. */ @@ -33,17 +42,96 @@ void GetStrongRandBytes(unsigned char* buf, int num); * This class is not thread-safe. */ class FastRandomContext { +private: + bool requires_seed; + ChaCha20 rng; + + unsigned char bytebuf[64]; + int bytebuf_size; + + uint64_t bitbuf; + int bitbuf_size; + + void RandomSeed(); + + void FillByteBuffer() + { + if (requires_seed) { + RandomSeed(); + } + rng.Output(bytebuf, sizeof(bytebuf)); + bytebuf_size = sizeof(bytebuf); + } + + void FillBitBuffer() + { + bitbuf = rand64(); + bitbuf_size = 64; + } + public: - explicit FastRandomContext(bool fDeterministic=false); + explicit FastRandomContext(bool fDeterministic = false); + + /** Initialize with explicit seed (only for testing) */ + explicit FastRandomContext(const uint256& seed); - uint32_t rand32() { - Rz = 36969 * (Rz & 65535) + (Rz >> 16); - Rw = 18000 * (Rw & 65535) + (Rw >> 16); - return (Rw << 16) + Rz; + /** Generate a random 64-bit integer. */ + uint64_t rand64() + { + if (bytebuf_size < 8) FillByteBuffer(); + uint64_t ret = ReadLE64(bytebuf + 64 - bytebuf_size); + bytebuf_size -= 8; + return ret; } - uint32_t Rz; - uint32_t Rw; + /** Generate a random (bits)-bit integer. */ + uint64_t randbits(int bits) { + if (bits == 0) { + return 0; + } else if (bits > 32) { + return rand64() >> (64 - bits); + } else { + if (bitbuf_size < bits) FillBitBuffer(); + uint64_t ret = bitbuf & (~(uint64_t)0 >> (64 - bits)); + bitbuf >>= bits; + bitbuf_size -= bits; + return ret; + } + } + + /** Generate a random integer in the range [0..range). */ + uint64_t randrange(uint64_t range) + { + --range; + int bits = CountBits(range); + while (true) { + uint64_t ret = randbits(bits); + if (ret <= range) return ret; + } + } + + /** Generate a random 32-bit integer. */ + uint32_t rand32() { return randbits(32); } + + /** Generate a random boolean. */ + bool randbool() { return randbits(1); } }; +/* Number of random bytes returned by GetOSRand. + * When changing this constant make sure to change all call sites, and make + * sure that the underlying OS APIs for all platforms support the number. + * (many cap out at 256 bytes). + */ +static const ssize_t NUM_OS_RANDOM_BYTES = 32; + +/** Get 32 bytes of system entropy. Do not use this in application code: use + * GetStrongRandBytes instead. + */ +void GetOSRand(unsigned char *ent32); + +/** Check that OS randomness is available and returning the requested number + * of bytes. + */ +bool Random_SanityCheck(); + #endif // BITCOIN_RANDOM_H diff --git a/src/rest.cpp b/src/rest.cpp index 54eefcafe3..b08d7153b1 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -5,10 +5,12 @@ #include "chain.h" #include "chainparams.h" +#include "core_io.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "validation.h" #include "httpserver.h" +#include "rpc/blockchain.h" #include "rpc/server.h" #include "streams.h" #include "sync.h" @@ -40,28 +42,24 @@ static const struct { }; struct CCoin { - uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE uint32_t nHeight; CTxOut out; ADD_SERIALIZE_METHODS; + CCoin() : nHeight(0) {} + CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} + template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTxVer); + uint32_t nTxVerDummy = 0; + READWRITE(nTxVerDummy); READWRITE(nHeight); READWRITE(out); } }; -extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); -extern UniValue mempoolInfoToJSON(); -extern UniValue mempoolToJSON(bool fVerbose = false); -extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); -extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); - static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { req->WriteHeader("Content-Type", "text/plain"); @@ -385,7 +383,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) case RF_JSON: { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, hashBlock, objTx); + TxToUniv(*tx, hashBlock, objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -514,22 +512,11 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool for (size_t i = 0; i < vOutPoints.size(); i++) { - CCoins coins; - uint256 hash = vOutPoints[i].hash; bool hit = false; - if (view.GetCoins(hash, coins)) { - mempool.pruneSpent(hash, coins); - if (coins.IsAvailable(vOutPoints[i].n)) { - hit = true; - // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if - // n is valid but points to an already spent output (IsNull). - CCoin coin; - coin.nTxVer = coins.nVersion; - coin.nHeight = coins.nHeight; - coin.out = coins.vout.at(vOutPoints[i].n); - assert(!coin.out.IsNull()); - outs.push_back(coin); - } + Coin coin; + if (view.GetCoin(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) { + hit = true; + outs.emplace_back(std::move(coin)); } hits.push_back(hit); @@ -573,13 +560,12 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) UniValue utxos(UniValue::VARR); BOOST_FOREACH (const CCoin& coin, outs) { UniValue utxo(UniValue::VOBJ); - utxo.push_back(Pair("txvers", (int32_t)coin.nTxVer)); utxo.push_back(Pair("height", (int32_t)coin.nHeight)); utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); // include the script in a json output UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); utxo.push_back(Pair("scriptPubKey", o)); utxos.push_back(utxo); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 6826ce4a79..b66c1c2b64 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -3,6 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "rpc/blockchain.h" + #include "amount.h" #include "chain.h" #include "chainparams.h" @@ -10,6 +12,8 @@ #include "coins.h" #include "consensus/validation.h" #include "validation.h" +#include "core_io.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" @@ -28,7 +32,6 @@ #include <mutex> #include <condition_variable> -using namespace std; struct CUpdatedBlock { @@ -41,15 +44,7 @@ static std::condition_variable cond_blockchange; static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); - -/** - * Get the difficulty of the net wrt to the given block index, or the chain tip if - * not provided. - * - * @return A floating point number that is a multiple of the main net minimum - * difficulty (4295032833 hashes). - */ + double GetDifficulty(const CBlockIndex* blockindex) { if (blockindex == NULL) @@ -107,7 +102,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) return result; } -UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false) +UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails) { UniValue result(UniValue::VOBJ); result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); @@ -129,7 +124,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx if(txDetails) { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, uint256(), objTx); + TxToUniv(*tx, uint256(), objTx); txs.push_back(objTx); } else @@ -154,7 +149,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx UniValue getblockcount(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getblockcount\n" "\nReturns the number of blocks in the longest blockchain.\n" "\nResult:\n" @@ -171,7 +166,7 @@ UniValue getblockcount(const JSONRPCRequest& request) UniValue getbestblockhash(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getbestblockhash\n" "\nReturns the hash of the best (tip) block in the longest blockchain.\n" "\nResult:\n" @@ -192,13 +187,13 @@ void RPCNotifyBlockChange(bool ibd, const CBlockIndex * pindex) latestblock.hash = pindex->GetBlockHash(); latestblock.height = pindex->nHeight; } - cond_blockchange.notify_all(); + cond_blockchange.notify_all(); } UniValue waitfornewblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "waitfornewblock (timeout)\n" "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n" @@ -236,7 +231,7 @@ UniValue waitfornewblock(const JSONRPCRequest& request) UniValue waitforblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "waitforblock <blockhash> (timeout)\n" "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n" @@ -278,7 +273,7 @@ UniValue waitforblock(const JSONRPCRequest& request) UniValue waitforblockheight(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "waitforblockheight <height> (timeout)\n" "\nWaits for (at least) block height and returns the height and hash\n" "of the current tip.\n" @@ -320,7 +315,7 @@ UniValue waitforblockheight(const JSONRPCRequest& request) UniValue getdifficulty(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getdifficulty\n" "\nReturns the proof-of-work difficulty as a multiple of the minimum difficulty.\n" "\nResult:\n" @@ -341,8 +336,6 @@ std::string EntryDescriptionString() " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority\n" " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" " \"height\" : n, (numeric) block height when transaction entered pool\n" - " \"startingpriority\" : n, (numeric) DEPRECATED. Priority when transaction entered pool\n" - " \"currentpriority\" : n, (numeric) DEPRECATED. Transaction priority now\n" " \"descendantcount\" : n, (numeric) number of in-mempool descendant transactions (including this one)\n" " \"descendantsize\" : n, (numeric) virtual transaction size of in-mempool descendants (including this one)\n" " \"descendantfees\" : n, (numeric) modified fees (see above) of in-mempool descendants (including this one)\n" @@ -363,8 +356,6 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) info.push_back(Pair("modifiedfee", ValueFromAmount(e.GetModifiedFee()))); info.push_back(Pair("time", e.GetTime())); info.push_back(Pair("height", (int)e.GetHeight())); - info.push_back(Pair("startingpriority", e.GetPriority(e.GetHeight()))); - info.push_back(Pair("currentpriority", e.GetPriority(chainActive.Height()))); info.push_back(Pair("descendantcount", e.GetCountWithDescendants())); info.push_back(Pair("descendantsize", e.GetSizeWithDescendants())); info.push_back(Pair("descendantfees", e.GetModFeesWithDescendants())); @@ -372,7 +363,7 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) info.push_back(Pair("ancestorsize", e.GetSizeWithAncestors())); info.push_back(Pair("ancestorfees", e.GetModFeesWithAncestors())); const CTransaction& tx = e.GetTx(); - set<string> setDepends; + std::set<std::string> setDepends; BOOST_FOREACH(const CTxIn& txin, tx.vin) { if (mempool.exists(txin.prevout.hash)) @@ -380,7 +371,7 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) } UniValue depends(UniValue::VARR); - BOOST_FOREACH(const string& dep, setDepends) + BOOST_FOREACH(const std::string& dep, setDepends) { depends.push_back(dep); } @@ -388,7 +379,7 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) info.push_back(Pair("depends", depends)); } -UniValue mempoolToJSON(bool fVerbose = false) +UniValue mempoolToJSON(bool fVerbose) { if (fVerbose) { @@ -405,7 +396,7 @@ UniValue mempoolToJSON(bool fVerbose = false) } else { - vector<uint256> vtxid; + std::vector<uint256> vtxid; mempool.queryHashes(vtxid); UniValue a(UniValue::VARR); @@ -419,9 +410,10 @@ UniValue mempoolToJSON(bool fVerbose = false) UniValue getrawmempool(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "getrawmempool ( verbose )\n" "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" + "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n" "\nArguments:\n" "1. verbose (boolean, optional, default=false) True for a json object, false for array of transaction ids\n" "\nResult: (for verbose = false):\n" @@ -450,7 +442,7 @@ UniValue getrawmempool(const JSONRPCRequest& request) UniValue getmempoolancestors(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw runtime_error( + throw std::runtime_error( "getmempoolancestors txid (verbose)\n" "\nIf txid is in the mempool, returns all in-mempool ancestors.\n" "\nArguments:\n" @@ -514,7 +506,7 @@ UniValue getmempoolancestors(const JSONRPCRequest& request) UniValue getmempooldescendants(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw runtime_error( + throw std::runtime_error( "getmempooldescendants txid (verbose)\n" "\nIf txid is in the mempool, returns all in-mempool descendants.\n" "\nArguments:\n" @@ -578,7 +570,7 @@ UniValue getmempooldescendants(const JSONRPCRequest& request) UniValue getmempoolentry(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) { - throw runtime_error( + throw std::runtime_error( "getmempoolentry txid\n" "\nReturns mempool data for given transaction\n" "\nArguments:\n" @@ -611,7 +603,7 @@ UniValue getmempoolentry(const JSONRPCRequest& request) UniValue getblockhash(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "getblockhash height\n" "\nReturns hash of block in best-block-chain at height provided.\n" "\nArguments:\n" @@ -636,7 +628,7 @@ UniValue getblockhash(const JSONRPCRequest& request) UniValue getblockheader(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "getblockheader \"hash\" ( verbose )\n" "\nIf verbose is false, returns a string that is serialized, hex-encoded data for blockheader 'hash'.\n" "If verbose is true, returns an Object with information about blockheader <hash>.\n" @@ -695,14 +687,17 @@ UniValue getblockheader(const JSONRPCRequest& request) UniValue getblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( - "getblock \"blockhash\" ( verbose )\n" - "\nIf verbose is false, returns a string that is serialized, hex-encoded data for block 'hash'.\n" - "If verbose is true, returns an Object with information about block <hash>.\n" + throw std::runtime_error( + "getblock \"blockhash\" ( verbosity ) \n" + "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" + "If verbosity is 1, returns an Object with information about block <hash>.\n" + "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n" "\nArguments:\n" "1. \"blockhash\" (string, required) The block hash\n" - "2. verbose (boolean, optional, default=true) true for a json object, false for the hex encoded data\n" - "\nResult (for verbose = true):\n" + "2. verbosity (numeric, optional, default=1) 0 for hex encoded data, 1 for a json object, and 2 for json object with transaction data\n" + "\nResult (for verbosity = 0):\n" + "\"data\" (string) A string that is serialized, hex-encoded data for block 'hash'.\n" + "\nResult (for verbosity = 1):\n" "{\n" " \"hash\" : \"hash\", (string) the block hash (same as provided)\n" " \"confirmations\" : n, (numeric) The number of confirmations, or -1 if the block is not on the main chain\n" @@ -726,8 +721,14 @@ UniValue getblock(const JSONRPCRequest& request) " \"previousblockhash\" : \"hash\", (string) The hash of the previous block\n" " \"nextblockhash\" : \"hash\" (string) The hash of the next block\n" "}\n" - "\nResult (for verbose=false):\n" - "\"data\" (string) A string that is serialized, hex-encoded data for block 'hash'.\n" + "\nResult (for verbosity = 2):\n" + "{\n" + " ..., Same output as verbosity = 1.\n" + " \"tx\" : [ (array of Objects) The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 \"tx\" result.\n" + " ,...\n" + " ],\n" + " ,... Same output as verbosity = 1.\n" + "}\n" "\nExamples:\n" + HelpExampleCli("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") + HelpExampleRpc("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") @@ -738,9 +739,13 @@ UniValue getblock(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); - bool fVerbose = true; - if (request.params.size() > 1) - fVerbose = request.params[1].get_bool(); + int verbosity = 1; + if (request.params.size() > 1) { + if(request.params[1].isNum()) + verbosity = request.params[1].get_int(); + else + verbosity = request.params[1].get_bool() ? 1 : 0; + } if (mapBlockIndex.count(hash) == 0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); @@ -749,12 +754,17 @@ UniValue getblock(const JSONRPCRequest& request) CBlockIndex* pblockindex = mapBlockIndex[hash]; if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) - throw JSONRPCError(RPC_INTERNAL_ERROR, "Block not available (pruned data)"); + throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); - if(!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) - throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) + // Block not found on disk. This could be because we have the block + // header in our index but don't have the block (for example if a + // non-whitelisted node sends us an unrequested long chain of valid + // blocks, we add the headers to our index, but don't accept the + // block). + throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); - if (!fVerbose) + if (verbosity <= 0) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; @@ -762,7 +772,7 @@ UniValue getblock(const JSONRPCRequest& request) return strHex; } - return blockToJSON(block, pblockindex); + return blockToJSON(block, pblockindex, verbosity >= 2); } struct CCoinsStats @@ -771,13 +781,32 @@ struct CCoinsStats uint256 hashBlock; uint64_t nTransactions; uint64_t nTransactionOutputs; - uint64_t nSerializedSize; + uint64_t nBogoSize; uint256 hashSerialized; + uint64_t nDiskSize; CAmount nTotalAmount; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} }; +static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + assert(!outputs.empty()); + ss << hash; + ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase); + stats.nTransactions++; + for (const auto output : outputs) { + ss << VARINT(output.first + 1); + ss << *(const CScriptBase*)(&output.second.out.scriptPubKey); + ss << VARINT(output.second.out.nValue); + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + + 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; + } + ss << VARINT(0); +} + //! Calculate statistics about the unspent transaction output set static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { @@ -790,39 +819,36 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; } ss << stats.hashBlock; - CAmount nTotalAmount = 0; + uint256 prevkey; + std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - for (unsigned int i=0; i<coins.vout.size(); i++) { - const CTxOut &out = coins.vout[i]; - if (!out.IsNull()) { - stats.nTransactionOutputs++; - ss << VARINT(i+1); - ss << out; - nTotalAmount += out.nValue; - } + COutPoint key; + Coin coin; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); } - stats.nSerializedSize += 32 + pcursor->GetValueSize(); - ss << VARINT(0); + prevkey = key.hash; + outputs[key.n] = std::move(coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; + stats.nDiskSize = view->EstimateSize(); return true; } UniValue pruneblockchain(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "pruneblockchain\n" "\nArguments:\n" "1. \"height\" (numeric, required) The block height to prune up to. May be set to a discrete height, or a unix timestamp\n" @@ -834,7 +860,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) + HelpExampleRpc("pruneblockchain", "1000")); if (!fPruneMode) - throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Cannot prune blocks because node is not in prune mode."); + throw JSONRPCError(RPC_MISC_ERROR, "Cannot prune blocks because node is not in prune mode."); LOCK(cs_main); @@ -846,9 +872,9 @@ UniValue pruneblockchain(const JSONRPCRequest& request) // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - 7200); + CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW); if (!pindex) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Could not find block with at least the specified timestamp."); + throw JSONRPCError(RPC_INVALID_PARAMETER, "Could not find block with at least the specified timestamp."); } heightParam = pindex->nHeight; } @@ -856,11 +882,11 @@ UniValue pruneblockchain(const JSONRPCRequest& request) unsigned int height = (unsigned int) heightParam; unsigned int chainHeight = (unsigned int) chainActive.Height(); if (chainHeight < Params().PruneAfterHeight()) - throw JSONRPCError(RPC_INTERNAL_ERROR, "Blockchain is too short for pruning."); + throw JSONRPCError(RPC_MISC_ERROR, "Blockchain is too short for pruning."); else if (height > chainHeight) throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height."); else if (height > chainHeight - MIN_BLOCKS_TO_KEEP) { - LogPrint("rpc", "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks."); + LogPrint(BCLog::RPC, "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks."); height = chainHeight - MIN_BLOCKS_TO_KEEP; } @@ -871,7 +897,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) UniValue gettxoutsetinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "gettxoutsetinfo\n" "\nReturns statistics about the unspent transaction output set.\n" "Note this call may take some time.\n" @@ -881,8 +907,9 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"bytes_serialized\": n, (numeric) The serialized size\n" - " \"hash_serialized\": \"hash\", (string) The serialized hash\n" + " \"bogosize\": n, (numeric) A meaningless metric for UTXO set size\n" + " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" + " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" "\nExamples:\n" @@ -899,8 +926,9 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); - ret.push_back(Pair("bytes_serialized", (int64_t)stats.nSerializedSize)); - ret.push_back(Pair("hash_serialized", stats.hashSerialized.GetHex())); + ret.push_back(Pair("bogosize", (int64_t)stats.nBogoSize)); + ret.push_back(Pair("hash_serialized_2", stats.hashSerialized.GetHex())); + ret.push_back(Pair("disk_size", stats.nDiskSize)); ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); @@ -911,7 +939,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) UniValue gettxout(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 3) - throw runtime_error( + throw std::runtime_error( "gettxout \"txid\" n ( include_mempool )\n" "\nReturns details about an unspent transaction output.\n" "\nArguments:\n" @@ -953,37 +981,37 @@ UniValue gettxout(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); int n = request.params[1].get_int(); + COutPoint out(hash, n); bool fMempool = true; if (request.params.size() > 2) fMempool = request.params[2].get_bool(); - CCoins coins; + Coin coin; if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(hash, coins)) + if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { // TODO: filtering spent coins should be done by the CCoinsViewMemPool return NullUniValue; - mempool.pruneSpent(hash, coins); // TODO: this should be done by the CCoinsViewMemPool + } } else { - if (!pcoinsTip->GetCoins(hash, coins)) + if (!pcoinsTip->GetCoin(out, coin)) { return NullUniValue; + } } - if (n<0 || (unsigned int)n>=coins.vout.size() || coins.vout[n].IsNull()) - return NullUniValue; BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); CBlockIndex *pindex = it->second; ret.push_back(Pair("bestblock", pindex->GetBlockHash().GetHex())); - if ((unsigned int)coins.nHeight == MEMPOOL_HEIGHT) + if (coin.nHeight == MEMPOOL_HEIGHT) { ret.push_back(Pair("confirmations", 0)); - else - ret.push_back(Pair("confirmations", pindex->nHeight - coins.nHeight + 1)); - ret.push_back(Pair("value", ValueFromAmount(coins.vout[n].nValue))); + } else { + ret.push_back(Pair("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1))); + } + ret.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coins.vout[n].scriptPubKey, o, true); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); ret.push_back(Pair("scriptPubKey", o)); - ret.push_back(Pair("version", coins.nVersion)); - ret.push_back(Pair("coinbase", coins.fCoinBase)); + ret.push_back(Pair("coinbase", (bool)coin.fCoinBase)); return ret; } @@ -993,7 +1021,7 @@ UniValue verifychain(const JSONRPCRequest& request) int nCheckLevel = GetArg("-checklevel", DEFAULT_CHECKLEVEL); int nCheckDepth = GetArg("-checkblocks", DEFAULT_CHECKBLOCKS); if (request.fHelp || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "verifychain ( checklevel nblocks )\n" "\nVerifies blockchain database.\n" "\nArguments:\n" @@ -1064,6 +1092,17 @@ static UniValue BIP9SoftForkDesc(const Consensus::Params& consensusParams, Conse rv.push_back(Pair("startTime", consensusParams.vDeployments[id].nStartTime)); rv.push_back(Pair("timeout", consensusParams.vDeployments[id].nTimeout)); rv.push_back(Pair("since", VersionBitsTipStateSinceHeight(consensusParams, id))); + if (THRESHOLD_STARTED == thresholdState) + { + UniValue statsUV(UniValue::VOBJ); + BIP9Stats statsStruct = VersionBitsTipStatistics(consensusParams, id); + statsUV.push_back(Pair("period", statsStruct.period)); + statsUV.push_back(Pair("threshold", statsStruct.threshold)); + statsUV.push_back(Pair("elapsed", statsStruct.elapsed)); + statsUV.push_back(Pair("count", statsStruct.count)); + statsUV.push_back(Pair("possible", statsStruct.possible)); + rv.push_back(Pair("statistics", statsUV)); + } return rv; } @@ -1079,7 +1118,7 @@ void BIP9SoftForkDescPushBack(UniValue& bip9_softforks, const std::string &name, UniValue getblockchaininfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getblockchaininfo\n" "Returns an object containing various state info regarding blockchain processing.\n" "\nResult:\n" @@ -1109,7 +1148,14 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx (numeric) height of the first block to which the status applies\n" + " \"since\": xx, (numeric) height of the first block to which the status applies\n" + " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" + " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" + " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" + " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" + " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" + " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" + " }\n" " }\n" " }\n" "}\n" @@ -1172,7 +1218,7 @@ struct CompareBlocksByHeight UniValue getchaintips(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getchaintips\n" "Return information about all known tips in the block tree," " including the main chain as well as orphaned branches.\n" @@ -1205,7 +1251,7 @@ UniValue getchaintips(const JSONRPCRequest& request) LOCK(cs_main); /* - * Idea: the set of chain tips is chainActive.tip, plus orphan blocks which do not have another orphan building off of them. + * Idea: the set of chain tips is chainActive.tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: * - Make one pass through mapBlockIndex, picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. @@ -1244,7 +1290,7 @@ UniValue getchaintips(const JSONRPCRequest& request) const int branchLen = block->nHeight - chainActive.FindFork(block)->nHeight; obj.push_back(Pair("branchlen", branchLen)); - string status; + std::string status; if (chainActive.Contains(block)) { // This block is part of the currently active chain. status = "active"; @@ -1288,7 +1334,7 @@ UniValue mempoolInfoToJSON() UniValue getmempoolinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getmempoolinfo\n" "\nReturns details on the active state of the TX memory pool.\n" "\nResult:\n" @@ -1297,7 +1343,7 @@ UniValue getmempoolinfo(const JSONRPCRequest& request) " \"bytes\": xxxxx, (numeric) Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted\n" " \"usage\": xxxxx, (numeric) Total memory usage for the mempool\n" " \"maxmempool\": xxxxx, (numeric) Maximum memory usage for the mempool\n" - " \"mempoolminfee\": xxxxx (numeric) Minimum fee for tx to be accepted\n" + " \"mempoolminfee\": xxxxx (numeric) Minimum feerate (" + CURRENCY_UNIT + " per KB) for tx to be accepted\n" "}\n" "\nExamples:\n" + HelpExampleCli("getmempoolinfo", "") @@ -1310,7 +1356,7 @@ UniValue getmempoolinfo(const JSONRPCRequest& request) UniValue preciousblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "preciousblock \"blockhash\"\n" "\nTreats a block as if it were received before others with the same work.\n" "\nA later preciousblock call can override the effect of an earlier one.\n" @@ -1348,7 +1394,7 @@ UniValue preciousblock(const JSONRPCRequest& request) UniValue invalidateblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "invalidateblock \"blockhash\"\n" "\nPermanently marks a block as invalid, as if it violated a consensus rule.\n" "\nArguments:\n" @@ -1386,7 +1432,7 @@ UniValue invalidateblock(const JSONRPCRequest& request) UniValue reconsiderblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "reconsiderblock \"blockhash\"\n" "\nRemoves invalidity status of a block and its descendants, reconsider them for activation.\n" "This can be used to undo the effects of invalidateblock.\n" @@ -1420,13 +1466,79 @@ UniValue reconsiderblock(const JSONRPCRequest& request) return NullUniValue; } +UniValue getchaintxstats(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 2) + throw std::runtime_error( + "getchaintxstats ( nblocks blockhash )\n" + "\nCompute statistics about the total number and rate of transactions in the chain.\n" + "\nArguments:\n" + "1. nblocks (numeric, optional) Size of the window in number of blocks (default: one month).\n" + "2. \"blockhash\" (string, optional) The hash of the block that ends the window.\n" + "\nResult:\n" + "{\n" + " \"time\": xxxxx, (numeric) The timestamp for the statistics in UNIX format.\n" + " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" + " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window.\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getchaintxstats", "") + + HelpExampleRpc("getchaintxstats", "2016") + ); + + const CBlockIndex* pindex; + int blockcount = 30 * 24 * 60 * 60 / Params().GetConsensus().nPowTargetSpacing; // By default: 1 month + + if (request.params.size() > 0 && !request.params[0].isNull()) { + blockcount = request.params[0].get_int(); + } + + bool havehash = request.params.size() > 1 && !request.params[1].isNull(); + uint256 hash; + if (havehash) { + hash = uint256S(request.params[1].get_str()); + } + + { + LOCK(cs_main); + if (havehash) { + auto it = mapBlockIndex.find(hash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + pindex = it->second; + if (!chainActive.Contains(pindex)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Block is not in main chain"); + } + } else { + pindex = chainActive.Tip(); + } + } + + if (blockcount < 1 || blockcount >= pindex->nHeight) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block count: should be between 1 and the block's height"); + } + + const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->nHeight - blockcount); + int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); + int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + + UniValue ret(UniValue::VOBJ); + ret.push_back(Pair("time", (int64_t)pindex->nTime)); + ret.push_back(Pair("txcount", (int64_t)pindex->nChainTx)); + ret.push_back(Pair("txrate", ((double)nTxDiff) / nTimeDiff)); + + return ret; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- { "blockchain", "getblockchaininfo", &getblockchaininfo, true, {} }, + { "blockchain", "getchaintxstats", &getchaintxstats, true, {"nblocks", "blockhash"} }, { "blockchain", "getbestblockhash", &getbestblockhash, true, {} }, { "blockchain", "getblockcount", &getblockcount, true, {} }, - { "blockchain", "getblock", &getblock, true, {"blockhash","verbose"} }, + { "blockchain", "getblock", &getblock, true, {"blockhash","verbosity|verbose"} }, { "blockchain", "getblockhash", &getblockhash, true, {"height"} }, { "blockchain", "getblockheader", &getblockheader, true, {"blockhash","verbose"} }, { "blockchain", "getchaintips", &getchaintips, true, {} }, diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h new file mode 100644 index 0000000000..c021441b0a --- /dev/null +++ b/src/rpc/blockchain.h @@ -0,0 +1,40 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RPC_BLOCKCHAIN_H +#define BITCOIN_RPC_BLOCKCHAIN_H + +class CBlock; +class CBlockIndex; +class CScript; +class CTransaction; +class uint256; +class UniValue; + +/** + * Get the difficulty of the net wrt to the given block index, or the chain tip if + * not provided. + * + * @return A floating point number that is a multiple of the main net minimum + * difficulty (4295032833 hashes). + */ +double GetDifficulty(const CBlockIndex* blockindex = nullptr); + +/** Callback for when block tip changed. */ +void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); + +/** Block description to JSON */ +UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); + +/** Mempool information to JSON */ +UniValue mempoolInfoToJSON(); + +/** Mempool to JSON */ +UniValue mempoolToJSON(bool fVerbose = false); + +/** Block header to JSON */ +UniValue blockheaderToJSON(const CBlockIndex* blockindex); + +#endif + diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 29bdb37682..ed83d1da1e 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -13,8 +13,6 @@ #include <boost/algorithm/string/case_conv.hpp> // for to_lower() #include <univalue.h> -using namespace std; - class CRPCConvertParam { public: @@ -79,11 +77,13 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listunspent", 0, "minconf" }, { "listunspent", 1, "maxconf" }, { "listunspent", 2, "addresses" }, - { "getblock", 1, "verbose" }, + { "listunspent", 4, "query_options" }, + { "getblock", 1, "verbosity" }, { "getblockheader", 1, "verbose" }, + { "getchaintxstats", 0, "nblocks" }, { "gettransaction", 1, "include_watchonly" }, { "getrawtransaction", 1, "verbose" }, - { "createrawtransaction", 0, "transactions" }, + { "createrawtransaction", 0, "inputs" }, { "createrawtransaction", 1, "outputs" }, { "createrawtransaction", 2, "locktime" }, { "signrawtransaction", 1, "prevtxs" }, @@ -107,9 +107,11 @@ static const CRPCConvertParam vRPCConvertParams[] = { "keypoolrefill", 0, "newsize" }, { "getrawmempool", 0, "verbose" }, { "estimatefee", 0, "nblocks" }, - { "estimatepriority", 0, "nblocks" }, { "estimatesmartfee", 0, "nblocks" }, - { "estimatesmartpriority", 0, "nblocks" }, + { "estimatesmartfee", 1, "conservative" }, + { "estimaterawfee", 0, "nblocks" }, + { "estimaterawfee", 1, "threshold" }, + { "estimaterawfee", 2, "horizon" }, { "prioritisetransaction", 1, "priority_delta" }, { "prioritisetransaction", 2, "fee_delta" }, { "setban", 2, "bantime" }, @@ -118,6 +120,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getmempoolancestors", 1, "verbose" }, { "getmempooldescendants", 1, "verbose" }, { "bumpfee", 1, "options" }, + { "logging", 0, "include" }, + { "logging", 1, "exclude" }, + { "disconnectnode", 1, "nodeid" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, @@ -171,7 +176,7 @@ UniValue ParseNonRFCJSONValue(const std::string& strVal) UniValue jVal; if (!jVal.read(std::string("[")+strVal+std::string("]")) || !jVal.isArray() || jVal.size()!=1) - throw runtime_error(string("Error parsing JSON:")+strVal); + throw std::runtime_error(std::string("Error parsing JSON:")+strVal); return jVal[0]; } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 7708771d00..cfe42ec7d8 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -15,7 +15,9 @@ #include "validation.h" #include "miner.h" #include "net.h" +#include "policy/fees.h" #include "pow.h" +#include "rpc/blockchain.h" #include "rpc/server.h" #include "txmempool.h" #include "util.h" @@ -26,12 +28,9 @@ #include <stdint.h> #include <boost/assign/list_of.hpp> -#include <boost/shared_ptr.hpp> #include <univalue.h> -using namespace std; - /** * Return average network hashes per second based on the last 'lookup' blocks, * or from the last difficulty change if 'lookup' is nonpositive. @@ -77,7 +76,7 @@ UniValue GetNetworkHashPS(int lookup, int height) { UniValue getnetworkhashps(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "getnetworkhashps ( nblocks height )\n" "\nReturns the estimated network hashes per second based on the last n blocks.\n" "Pass in [blocks] to override # of blocks, -1 specifies since last difficulty change.\n" @@ -96,7 +95,7 @@ UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(request.params.size() > 0 ? request.params[0].get_int() : 120, request.params.size() > 1 ? request.params[1].get_int() : -1); } -UniValue generateBlocks(boost::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript) +UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript) { static const int nInnerLoopCount = 0x10000; int nHeightStart = 0; @@ -149,7 +148,7 @@ UniValue generateBlocks(boost::shared_ptr<CReserveScript> coinbaseScript, int nG UniValue generate(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "generate nblocks ( maxtries )\n" "\nMine up to nblocks blocks immediately (before the RPC call returns)\n" "\nArguments:\n" @@ -168,7 +167,7 @@ UniValue generate(const JSONRPCRequest& request) nMaxTries = request.params[1].get_int(); } - boost::shared_ptr<CReserveScript> coinbaseScript; + std::shared_ptr<CReserveScript> coinbaseScript; GetMainSignals().ScriptForMining(coinbaseScript); // If the keypool is exhausted, no script is returned at all. Catch this. @@ -185,7 +184,7 @@ UniValue generate(const JSONRPCRequest& request) UniValue generatetoaddress(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 3) - throw runtime_error( + throw std::runtime_error( "generatetoaddress nblocks address (maxtries)\n" "\nMine blocks immediately to a specified address (before the RPC call returns)\n" "\nArguments:\n" @@ -208,8 +207,8 @@ UniValue generatetoaddress(const JSONRPCRequest& request) CBitcoinAddress address(request.params[1].get_str()); if (!address.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address"); - - boost::shared_ptr<CReserveScript> coinbaseScript(new CReserveScript()); + + std::shared_ptr<CReserveScript> coinbaseScript = std::make_shared<CReserveScript>(); coinbaseScript->reserveScript = GetScriptForDestination(address.Get()); return generateBlocks(coinbaseScript, nGenerate, nMaxTries, false); @@ -218,7 +217,7 @@ UniValue generatetoaddress(const JSONRPCRequest& request) UniValue getmininginfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getmininginfo\n" "\nReturns a json object containing mining-related information." "\nResult:\n" @@ -259,14 +258,12 @@ UniValue getmininginfo(const JSONRPCRequest& request) UniValue prioritisetransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 3) - throw runtime_error( + throw std::runtime_error( "prioritisetransaction <txid> <priority delta> <fee delta>\n" "Accepts the transaction into mined blocks at a higher (or lower) priority\n" "\nArguments:\n" "1. \"txid\" (string, required) The transaction id.\n" - "2. priority_delta (numeric, required) The priority to add or subtract.\n" - " The transaction selection algorithm considers the tx as it would have a higher priority.\n" - " (priority of a transaction is calculated: coinage * value_in_satoshis / txsize) \n" + "2. priority_delta (numeric, optional) Fee-independent priority adjustment. Not supported, so must be zero or null.\n" "3. fee_delta (numeric, required) The fee value (in satoshis) to add (or subtract, if negative).\n" " The fee is not actually paid, only the algorithm for selecting transactions into a block\n" " considers the transaction as it would have paid a higher (or lower) fee.\n" @@ -282,7 +279,11 @@ UniValue prioritisetransaction(const JSONRPCRequest& request) uint256 hash = ParseHashStr(request.params[0].get_str(), "txid"); CAmount nAmount = request.params[2].get_int64(); - mempool.PrioritiseTransaction(hash, request.params[1].get_real(), nAmount); + if (!(request.params[1].isNull() || request.params[1].get_real() == 0)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Priority is not supported, and adjustment thereof must be zero."); + } + + mempool.PrioritiseTransaction(hash, nAmount); return true; } @@ -307,7 +308,7 @@ static UniValue BIP22ValidationResult(const CValidationState& state) } std::string gbt_vb_name(const Consensus::DeploymentPos pos) { - const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + const struct VBDeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; std::string s = vbinfo.name; if (!vbinfo.gbt_force) { s.insert(s.begin(), '!'); @@ -318,7 +319,7 @@ std::string gbt_vb_name(const Consensus::DeploymentPos pos) { UniValue getblocktemplate(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "getblocktemplate ( TemplateRequest )\n" "\nIf the request parameters include a 'mode' key, that is used to explicitly select between the default 'template' request or a 'proposal'.\n" "It returns data needed to construct a block to work on.\n" @@ -519,12 +520,22 @@ UniValue getblocktemplate(const JSONRPCRequest& request) // TODO: Maybe recheck connections/IBD and (if something wrong) send an expires-immediately template to stop miners? } + const struct VBDeploymentInfo& segwit_info = VersionBitsDeploymentInfo[Consensus::DEPLOYMENT_SEGWIT]; + // If the caller is indicating segwit support, then allow CreateNewBlock() + // to select witness transactions, after segwit activates (otherwise + // don't). + bool fSupportsSegwit = setClientRules.find(segwit_info.name) != setClientRules.end(); + // Update block static CBlockIndex* pindexPrev; static int64_t nStart; static std::unique_ptr<CBlockTemplate> pblocktemplate; + // Cache whether the last invocation was with segwit support, to avoid returning + // a segwit-block to a non-segwit caller. + static bool fLastTemplateSupportsSegwit = true; if (pindexPrev != chainActive.Tip() || - (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - nStart > 5)) + (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - nStart > 5) || + fLastTemplateSupportsSegwit != fSupportsSegwit) { // Clear pindexPrev so future calls make a new block, despite any failures from here on pindexPrev = nullptr; @@ -533,10 +544,11 @@ UniValue getblocktemplate(const JSONRPCRequest& request) nTransactionsUpdatedLast = mempool.GetTransactionsUpdated(); CBlockIndex* pindexPrevNew = chainActive.Tip(); nStart = GetTime(); + fLastTemplateSupportsSegwit = fSupportsSegwit; // Create new block CScript scriptDummy = CScript() << OP_TRUE; - pblocktemplate = BlockAssembler(Params()).CreateNewBlock(scriptDummy); + pblocktemplate = BlockAssembler(Params()).CreateNewBlock(scriptDummy, fSupportsSegwit); if (!pblocktemplate) throw JSONRPCError(RPC_OUT_OF_MEMORY, "Out of memory"); @@ -556,7 +568,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) UniValue aCaps(UniValue::VARR); aCaps.push_back("proposal"); UniValue transactions(UniValue::VARR); - map<uint256, int64_t> setTxIndex; + std::map<uint256, int64_t> setTxIndex; int i = 0; for (const auto& it : pblock->vtx) { const CTransaction& tx = *it; @@ -622,7 +634,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) // FALL THROUGH to get vbavailable set... case THRESHOLD_STARTED: { - const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + const struct VBDeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; vbavailable.push_back(Pair(gbt_vb_name(pos), consensusParams.vDeployments[pos].bit)); if (setClientRules.find(vbinfo.name) == setClientRules.end()) { if (!vbinfo.gbt_force) { @@ -635,7 +647,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) case THRESHOLD_ACTIVE: { // Add to rules only - const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + const struct VBDeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; aRules.push_back(gbt_vb_name(pos)); if (setClientRules.find(vbinfo.name) == setClientRules.end()) { // Not supported by the client; make sure it's safe to proceed @@ -686,8 +698,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) result.push_back(Pair("bits", strprintf("%08x", pblock->nBits))); result.push_back(Pair("height", (int64_t)(pindexPrev->nHeight+1))); - const struct BIP9DeploymentInfo& segwit_info = VersionBitsDeploymentInfo[Consensus::DEPLOYMENT_SEGWIT]; - if (!pblocktemplate->vchCoinbaseCommitment.empty() && setClientRules.find(segwit_info.name) != setClientRules.end()) { + if (!pblocktemplate->vchCoinbaseCommitment.empty() && fSupportsSegwit) { result.push_back(Pair("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment.begin(), pblocktemplate->vchCoinbaseCommitment.end()))); } @@ -704,7 +715,7 @@ public: submitblock_StateCatcher(const uint256 &hashIn) : hash(hashIn), found(false), state() {} protected: - virtual void BlockChecked(const CBlock& block, const CValidationState& stateIn) { + void BlockChecked(const CBlock& block, const CValidationState& stateIn) override { if (block.GetHash() != hash) return; found = true; @@ -714,8 +725,8 @@ protected: UniValue submitblock(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { + throw std::runtime_error( "submitblock \"hexdata\" ( \"jsonparametersobject\" )\n" "\nAttempts to submit new block to network.\n" "The 'jsonparametersobject' parameter is currently ignored.\n" @@ -732,11 +743,17 @@ UniValue submitblock(const JSONRPCRequest& request) + HelpExampleCli("submitblock", "\"mydata\"") + HelpExampleRpc("submitblock", "\"mydata\"") ); + } std::shared_ptr<CBlock> blockptr = std::make_shared<CBlock>(); CBlock& block = *blockptr; - if (!DecodeHexBlk(block, request.params[0].get_str())) + if (!DecodeHexBlk(block, request.params[0].get_str())) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); + } + + if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block does not start with a coinbase"); + } uint256 hash = block.GetHash(); bool fBlockPresent = false; @@ -745,10 +762,12 @@ UniValue submitblock(const JSONRPCRequest& request) BlockMap::iterator mi = mapBlockIndex.find(hash); if (mi != mapBlockIndex.end()) { CBlockIndex *pindex = mi->second; - if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) + if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) { return "duplicate"; - if (pindex->nStatus & BLOCK_FAILED_MASK) + } + if (pindex->nStatus & BLOCK_FAILED_MASK) { return "duplicate-invalid"; + } // Otherwise, we might only have the header - process the block before returning fBlockPresent = true; } @@ -766,22 +785,24 @@ UniValue submitblock(const JSONRPCRequest& request) RegisterValidationInterface(&sc); bool fAccepted = ProcessNewBlock(Params(), blockptr, true, NULL); UnregisterValidationInterface(&sc); - if (fBlockPresent) - { - if (fAccepted && !sc.found) + if (fBlockPresent) { + if (fAccepted && !sc.found) { return "duplicate-inconclusive"; + } return "duplicate"; } - if (!sc.found) + if (!sc.found) { return "inconclusive"; + } return BIP22ValidationResult(sc.state); } UniValue estimatefee(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "estimatefee nblocks\n" + "\nDEPRECATED. Please use estimatesmartfee for more intelligent estimates." "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks. Uses virtual transaction size of transaction\n" "as defined in BIP 141 (witness data is discounted).\n" @@ -804,52 +825,28 @@ UniValue estimatefee(const JSONRPCRequest& request) if (nBlocks < 1) nBlocks = 1; - CFeeRate feeRate = mempool.estimateFee(nBlocks); + CFeeRate feeRate = ::feeEstimator.estimateFee(nBlocks); if (feeRate == CFeeRate(0)) return -1.0; return ValueFromAmount(feeRate.GetFeePerK()); } -UniValue estimatepriority(const JSONRPCRequest& request) -{ - if (request.fHelp || request.params.size() != 1) - throw runtime_error( - "estimatepriority nblocks\n" - "\nDEPRECATED. Estimates the approximate priority a zero-fee transaction needs to begin\n" - "confirmation within nblocks blocks.\n" - "\nArguments:\n" - "1. nblocks (numeric, required)\n" - "\nResult:\n" - "n (numeric) estimated priority\n" - "\n" - "A negative value is returned if not enough transactions and blocks\n" - "have been observed to make an estimate.\n" - "\nExample:\n" - + HelpExampleCli("estimatepriority", "6") - ); - - RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)); - - int nBlocks = request.params[0].get_int(); - if (nBlocks < 1) - nBlocks = 1; - - return mempool.estimatePriority(nBlocks); -} - UniValue estimatesmartfee(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) - throw runtime_error( - "estimatesmartfee nblocks\n" - "\nWARNING: This interface is unstable and may disappear or change!\n" + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) + throw std::runtime_error( + "estimatesmartfee nblocks (conservative)\n" "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks if possible and return the number of blocks\n" "for which the estimate is valid. Uses virtual transaction size as defined\n" "in BIP 141 (witness data is discounted).\n" "\nArguments:\n" - "1. nblocks (numeric)\n" + "1. nblocks (numeric)\n" + "2. conservative (bool, optional, default=true) Whether to return a more conservative estimate which\n" + " also satisfies a longer history. A conservative estimate potentially returns a higher\n" + " feerate and is more likely to be sufficient for the desired target, but is not as\n" + " responsive to short term drops in the prevailing fee market\n" "\nResult:\n" "{\n" " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" @@ -866,48 +863,99 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)); int nBlocks = request.params[0].get_int(); + bool conservative = true; + if (request.params.size() > 1 && !request.params[1].isNull()) { + RPCTypeCheckArgument(request.params[1], UniValue::VBOOL); + conservative = request.params[1].get_bool(); + } UniValue result(UniValue::VOBJ); int answerFound; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocks, &answerFound); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool, conservative); result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); result.push_back(Pair("blocks", answerFound)); return result; } -UniValue estimatesmartpriority(const JSONRPCRequest& request) +UniValue estimaterawfee(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) - throw runtime_error( - "estimatesmartpriority nblocks\n" - "\nDEPRECATED. WARNING: This interface is unstable and may disappear or change!\n" - "\nEstimates the approximate priority a zero-fee transaction needs to begin\n" - "confirmation within nblocks blocks if possible and return the number of blocks\n" - "for which the estimate is valid.\n" + if (request.fHelp || request.params.size() < 1|| request.params.size() > 3) + throw std::runtime_error( + "estimaterawfee nblocks (threshold horizon)\n" + "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" + " implementation of fee estimation. The parameters it can be called with\n" + " and the results it returns will change if the internal implementation changes.\n" + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within nblocks blocks if possible. Uses virtual transaction size as defined\n" + "in BIP 141 (witness data is discounted).\n" "\nArguments:\n" - "1. nblocks (numeric, required)\n" + "1. nblocks (numeric)\n" + "2. threshold (numeric, optional) The proportion of transactions in a given feerate range that must have been\n" + " confirmed within nblocks in order to consider those feerates as high enough and proceed to check\n" + " lower buckets. Default: 0.95\n" + "3. horizon (numeric, optional) How long a history of estimates to consider. 0=short, 1=medium, 2=long.\n" + " Default: 1\n" "\nResult:\n" "{\n" - " \"priority\" : x.x, (numeric) estimated priority\n" - " \"blocks\" : n (numeric) block number where estimate was found\n" + " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" + " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" + " \"scale\" : x, (numeric) The resolution of confirmation targets at this time horizon\n" + " \"pass\" : { (json object) information about the lowest range of feerates to succeed in meeting the threshold\n" + " \"startrange\" : x.x, (numeric) start of feerate range\n" + " \"endrange\" : x.x, (numeric) end of feerate range\n" + " \"withintarget\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed within target\n" + " \"totalconfirmed\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed at any point\n" + " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" + " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" + " }\n" + " \"fail\" : { ... } (json object) information about the highest range of feerates to fail to meet the threshold\n" "}\n" "\n" - "A negative value is returned if not enough transactions and blocks\n" - "have been observed to make an estimate for any number of blocks.\n" - "However if the mempool reject fee is set it will return 1e9 * MAX_MONEY.\n" + "A negative feerate is returned if no answer can be given.\n" "\nExample:\n" - + HelpExampleCli("estimatesmartpriority", "6") + + HelpExampleCli("estimaterawfee", "6 0.9 1") ); - RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)); - + RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VNUM)(UniValue::VNUM), true); + RPCTypeCheckArgument(request.params[0], UniValue::VNUM); int nBlocks = request.params[0].get_int(); - + double threshold = 0.95; + if (!request.params[1].isNull()) + threshold = request.params[1].get_real(); + FeeEstimateHorizon horizon = FeeEstimateHorizon::MED_HALFLIFE; + if (!request.params[2].isNull()) { + int horizonInt = request.params[2].get_int(); + if (horizonInt < 0 || horizonInt > 2) { + throw JSONRPCError(RPC_TYPE_ERROR, "Invalid horizon for fee estimates"); + } else { + horizon = (FeeEstimateHorizon)horizonInt; + } + } UniValue result(UniValue::VOBJ); - int answerFound; - double priority = mempool.estimateSmartPriority(nBlocks, &answerFound); - result.push_back(Pair("priority", priority)); - result.push_back(Pair("blocks", answerFound)); + CFeeRate feeRate; + EstimationResult buckets; + feeRate = ::feeEstimator.estimateRawFee(nBlocks, threshold, horizon, &buckets); + + result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); + result.push_back(Pair("decay", buckets.decay)); + result.push_back(Pair("scale", (int)buckets.scale)); + UniValue passbucket(UniValue::VOBJ); + passbucket.push_back(Pair("startrange", round(buckets.pass.start))); + passbucket.push_back(Pair("endrange", round(buckets.pass.end))); + passbucket.push_back(Pair("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0)); + passbucket.push_back(Pair("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0)); + passbucket.push_back(Pair("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0)); + passbucket.push_back(Pair("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("pass", passbucket)); + UniValue failbucket(UniValue::VOBJ); + failbucket.push_back(Pair("startrange", round(buckets.fail.start))); + failbucket.push_back(Pair("endrange", round(buckets.fail.end))); + failbucket.push_back(Pair("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0)); + failbucket.push_back(Pair("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0)); + failbucket.push_back(Pair("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0)); + failbucket.push_back(Pair("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("fail", failbucket)); return result; } @@ -924,9 +972,9 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, true, {"nblocks","address","maxtries"} }, { "util", "estimatefee", &estimatefee, true, {"nblocks"} }, - { "util", "estimatepriority", &estimatepriority, true, {"nblocks"} }, - { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks"} }, - { "util", "estimatesmartpriority", &estimatesmartpriority, true, {"nblocks"} }, + { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks", "conservative"} }, + + { "hidden", "estimaterawfee", &estimaterawfee, true, {"nblocks", "threshold", "horizon"} }, }; void RegisterMiningRPCCommands(CRPCTable &t) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index c84eab7d23..1f973a0c18 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -4,28 +4,33 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "base58.h" +#include "chain.h" #include "clientversion.h" #include "init.h" #include "validation.h" +#include "httpserver.h" #include "net.h" #include "netbase.h" +#include "rpc/blockchain.h" #include "rpc/server.h" #include "timedata.h" #include "util.h" #include "utilstrencodings.h" #ifdef ENABLE_WALLET +#include "wallet/rpcwallet.h" #include "wallet/wallet.h" #include "wallet/walletdb.h" #endif #include <stdint.h> +#ifdef HAVE_MALLOC_INFO +#include <malloc.h> +#endif #include <boost/assign/list_of.hpp> #include <univalue.h> -using namespace std; - /** * @note Do not add or change anything in the information returned by this * method. `getinfo` exists for backwards-compatibility only. It combines @@ -42,7 +47,7 @@ using namespace std; UniValue getinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getinfo\n" "\nDEPRECATED. Returns an object containing various state info.\n" "\nResult:\n" @@ -61,7 +66,7 @@ UniValue getinfo(const JSONRPCRequest& request) " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee set in " + CURRENCY_UNIT + "/kB\n" - " \"relayfee\": x.xxxx, (numeric) minimum relay fee for non-free transactions in " + CURRENCY_UNIT + "/kB\n" + " \"relayfee\": x.xxxx, (numeric) minimum relay fee for transactions in " + CURRENCY_UNIT + "/kB\n" " \"errors\": \"...\" (string) any error messages\n" "}\n" "\nExamples:\n" @@ -70,7 +75,9 @@ UniValue getinfo(const JSONRPCRequest& request) ); #ifdef ENABLE_WALLET - LOCK2(cs_main, pwalletMain ? &pwalletMain->cs_wallet : NULL); + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + + LOCK2(cs_main, pwallet ? &pwallet->cs_wallet : NULL); #else LOCK(cs_main); #endif @@ -82,25 +89,26 @@ UniValue getinfo(const JSONRPCRequest& request) obj.push_back(Pair("version", CLIENT_VERSION)); obj.push_back(Pair("protocolversion", PROTOCOL_VERSION)); #ifdef ENABLE_WALLET - if (pwalletMain) { - obj.push_back(Pair("walletversion", pwalletMain->GetVersion())); - obj.push_back(Pair("balance", ValueFromAmount(pwalletMain->GetBalance()))); + if (pwallet) { + obj.push_back(Pair("walletversion", pwallet->GetVersion())); + obj.push_back(Pair("balance", ValueFromAmount(pwallet->GetBalance()))); } #endif obj.push_back(Pair("blocks", (int)chainActive.Height())); obj.push_back(Pair("timeoffset", GetTimeOffset())); if(g_connman) obj.push_back(Pair("connections", (int)g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL))); - obj.push_back(Pair("proxy", (proxy.IsValid() ? proxy.proxy.ToStringIPPort() : string()))); + obj.push_back(Pair("proxy", (proxy.IsValid() ? proxy.proxy.ToStringIPPort() : std::string()))); obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("testnet", Params().NetworkIDString() == CBaseChainParams::TESTNET)); #ifdef ENABLE_WALLET - if (pwalletMain) { - obj.push_back(Pair("keypoololdest", pwalletMain->GetOldestKeyPoolTime())); - obj.push_back(Pair("keypoolsize", (int)pwalletMain->GetKeyPoolSize())); + if (pwallet) { + obj.push_back(Pair("keypoololdest", pwallet->GetOldestKeyPoolTime())); + obj.push_back(Pair("keypoolsize", (int)pwallet->GetKeyPoolSize())); + } + if (pwallet && pwallet->IsCrypted()) { + obj.push_back(Pair("unlocked_until", pwallet->nRelockTime)); } - if (pwalletMain && pwalletMain->IsCrypted()) - obj.push_back(Pair("unlocked_until", nWalletUnlockTime)); obj.push_back(Pair("paytxfee", ValueFromAmount(payTxFee.GetFeePerK()))); #endif obj.push_back(Pair("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK()))); @@ -112,13 +120,17 @@ UniValue getinfo(const JSONRPCRequest& request) class DescribeAddressVisitor : public boost::static_visitor<UniValue> { public: + CWallet * const pwallet; + + DescribeAddressVisitor(CWallet *_pwallet) : pwallet(_pwallet) {} + UniValue operator()(const CNoDestination &dest) const { return UniValue(UniValue::VOBJ); } UniValue operator()(const CKeyID &keyID) const { UniValue obj(UniValue::VOBJ); CPubKey vchPubKey; obj.push_back(Pair("isscript", false)); - if (pwalletMain && pwalletMain->GetPubKey(keyID, vchPubKey)) { + if (pwallet && pwallet->GetPubKey(keyID, vchPubKey)) { obj.push_back(Pair("pubkey", HexStr(vchPubKey))); obj.push_back(Pair("iscompressed", vchPubKey.IsCompressed())); } @@ -129,7 +141,7 @@ public: UniValue obj(UniValue::VOBJ); CScript subscript; obj.push_back(Pair("isscript", true)); - if (pwalletMain && pwalletMain->GetCScript(scriptID, subscript)) { + if (pwallet && pwallet->GetCScript(scriptID, subscript)) { std::vector<CTxDestination> addresses; txnouttype whichType; int nRequired; @@ -151,7 +163,7 @@ public: UniValue validateaddress(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "validateaddress \"address\"\n" "\nReturn information about the given bitcoin address.\n" "\nArguments:\n" @@ -177,7 +189,9 @@ UniValue validateaddress(const JSONRPCRequest& request) ); #ifdef ENABLE_WALLET - LOCK2(cs_main, pwalletMain ? &pwalletMain->cs_wallet : NULL); + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + + LOCK2(cs_main, pwallet ? &pwallet->cs_wallet : NULL); #else LOCK(cs_main); #endif @@ -190,23 +204,24 @@ UniValue validateaddress(const JSONRPCRequest& request) if (isValid) { CTxDestination dest = address.Get(); - string currentAddress = address.ToString(); + std::string currentAddress = address.ToString(); ret.push_back(Pair("address", currentAddress)); CScript scriptPubKey = GetScriptForDestination(dest); ret.push_back(Pair("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end()))); #ifdef ENABLE_WALLET - isminetype mine = pwalletMain ? IsMine(*pwalletMain, dest) : ISMINE_NO; + isminetype mine = pwallet ? IsMine(*pwallet, dest) : ISMINE_NO; ret.push_back(Pair("ismine", (mine & ISMINE_SPENDABLE) ? true : false)); ret.push_back(Pair("iswatchonly", (mine & ISMINE_WATCH_ONLY) ? true: false)); - UniValue detail = boost::apply_visitor(DescribeAddressVisitor(), dest); + UniValue detail = boost::apply_visitor(DescribeAddressVisitor(pwallet), dest); ret.pushKVs(detail); - if (pwalletMain && pwalletMain->mapAddressBook.count(dest)) - ret.push_back(Pair("account", pwalletMain->mapAddressBook[dest].name)); + if (pwallet && pwallet->mapAddressBook.count(dest)) { + ret.push_back(Pair("account", pwallet->mapAddressBook[dest].name)); + } CKeyID keyID; - if (pwalletMain) { - const auto& meta = pwalletMain->mapKeyMetadata; + if (pwallet) { + const auto& meta = pwallet->mapKeyMetadata; auto it = address.GetKeyID(keyID) ? meta.find(keyID) : meta.end(); if (it == meta.end()) { it = meta.find(CScriptID(scriptPubKey)); @@ -224,23 +239,26 @@ UniValue validateaddress(const JSONRPCRequest& request) return ret; } +// Needed even with !ENABLE_WALLET, to pass (ignored) pointers around +class CWallet; + /** * Used by addmultisigaddress / createmultisig: */ -CScript _createmultisig_redeemScript(const UniValue& params) +CScript _createmultisig_redeemScript(CWallet * const pwallet, const UniValue& params) { int nRequired = params[0].get_int(); const UniValue& keys = params[1].get_array(); // Gather public keys if (nRequired < 1) - throw runtime_error("a multisignature address must require at least one key to redeem"); + throw std::runtime_error("a multisignature address must require at least one key to redeem"); if ((int)keys.size() < nRequired) - throw runtime_error( + throw std::runtime_error( strprintf("not enough keys supplied " "(got %u keys, but need at least %d to redeem)", keys.size(), nRequired)); if (keys.size() > 16) - throw runtime_error("Number of addresses involved in the multisignature address creation > 16\nReduce the number"); + throw std::runtime_error("Number of addresses involved in the multisignature address creation > 16\nReduce the number"); std::vector<CPubKey> pubkeys; pubkeys.resize(keys.size()); for (unsigned int i = 0; i < keys.size(); i++) @@ -249,18 +267,18 @@ CScript _createmultisig_redeemScript(const UniValue& params) #ifdef ENABLE_WALLET // Case 1: Bitcoin address and we have full public key: CBitcoinAddress address(ks); - if (pwalletMain && address.IsValid()) - { + if (pwallet && address.IsValid()) { CKeyID keyID; if (!address.GetKeyID(keyID)) - throw runtime_error( + throw std::runtime_error( strprintf("%s does not refer to a key",ks)); CPubKey vchPubKey; - if (!pwalletMain->GetPubKey(keyID, vchPubKey)) - throw runtime_error( + if (!pwallet->GetPubKey(keyID, vchPubKey)) { + throw std::runtime_error( strprintf("no full public key for address %s",ks)); + } if (!vchPubKey.IsFullyValid()) - throw runtime_error(" Invalid public key: "+ks); + throw std::runtime_error(" Invalid public key: "+ks); pubkeys[i] = vchPubKey; } @@ -271,18 +289,18 @@ CScript _createmultisig_redeemScript(const UniValue& params) { CPubKey vchPubKey(ParseHex(ks)); if (!vchPubKey.IsFullyValid()) - throw runtime_error(" Invalid public key: "+ks); + throw std::runtime_error(" Invalid public key: "+ks); pubkeys[i] = vchPubKey; } else { - throw runtime_error(" Invalid public key: "+ks); + throw std::runtime_error(" Invalid public key: "+ks); } } CScript result = GetScriptForMultisig(nRequired, pubkeys); if (result.size() > MAX_SCRIPT_ELEMENT_SIZE) - throw runtime_error( + throw std::runtime_error( strprintf("redeemScript exceeds size limit: %d > %d", result.size(), MAX_SCRIPT_ELEMENT_SIZE)); return result; @@ -290,9 +308,15 @@ CScript _createmultisig_redeemScript(const UniValue& params) UniValue createmultisig(const JSONRPCRequest& request) { +#ifdef ENABLE_WALLET + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); +#else + CWallet * const pwallet = NULL; +#endif + if (request.fHelp || request.params.size() < 2 || request.params.size() > 2) { - string msg = "createmultisig nrequired [\"key\",...]\n" + std::string msg = "createmultisig nrequired [\"key\",...]\n" "\nCreates a multi-signature address with n signature of m keys required.\n" "It returns a json object with the address and redeemScript.\n" @@ -316,11 +340,11 @@ UniValue createmultisig(const JSONRPCRequest& request) "\nAs a json rpc call\n" + HelpExampleRpc("createmultisig", "2, \"[\\\"16sSauSf5pF2UkUwvKGq4qjNRzBZYqgEL5\\\",\\\"171sgjn4YtPu27adkKGrdDwzRTxnRkBfKV\\\"]\"") ; - throw runtime_error(msg); + throw std::runtime_error(msg); } // Construct using pay-to-script-hash: - CScript inner = _createmultisig_redeemScript(request.params); + CScript inner = _createmultisig_redeemScript(pwallet, request.params); CScriptID innerID(inner); CBitcoinAddress address(innerID); @@ -334,7 +358,7 @@ UniValue createmultisig(const JSONRPCRequest& request) UniValue verifymessage(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 3) - throw runtime_error( + throw std::runtime_error( "verifymessage \"address\" \"signature\" \"message\"\n" "\nVerify a signed message\n" "\nArguments:\n" @@ -356,9 +380,9 @@ UniValue verifymessage(const JSONRPCRequest& request) LOCK(cs_main); - string strAddress = request.params[0].get_str(); - string strSign = request.params[1].get_str(); - string strMessage = request.params[2].get_str(); + std::string strAddress = request.params[0].get_str(); + std::string strSign = request.params[1].get_str(); + std::string strMessage = request.params[2].get_str(); CBitcoinAddress addr(strAddress); if (!addr.IsValid()) @@ -369,7 +393,7 @@ UniValue verifymessage(const JSONRPCRequest& request) throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); bool fInvalid = false; - vector<unsigned char> vchSig = DecodeBase64(strSign.c_str(), &fInvalid); + std::vector<unsigned char> vchSig = DecodeBase64(strSign.c_str(), &fInvalid); if (fInvalid) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Malformed base64 encoding"); @@ -388,7 +412,7 @@ UniValue verifymessage(const JSONRPCRequest& request) UniValue signmessagewithprivkey(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 2) - throw runtime_error( + throw std::runtime_error( "signmessagewithprivkey \"privkey\" \"message\"\n" "\nSign a message with the private key of an address\n" "\nArguments:\n" @@ -405,8 +429,8 @@ UniValue signmessagewithprivkey(const JSONRPCRequest& request) + HelpExampleRpc("signmessagewithprivkey", "\"privkey\", \"my message\"") ); - string strPrivkey = request.params[0].get_str(); - string strMessage = request.params[1].get_str(); + std::string strPrivkey = request.params[0].get_str(); + std::string strMessage = request.params[1].get_str(); CBitcoinSecret vchSecret; bool fGood = vchSecret.SetString(strPrivkey); @@ -420,7 +444,7 @@ UniValue signmessagewithprivkey(const JSONRPCRequest& request) ss << strMessageMagic; ss << strMessage; - vector<unsigned char> vchSig; + std::vector<unsigned char> vchSig; if (!key.SignCompact(ss.GetHash(), vchSig)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Sign failed"); @@ -430,7 +454,7 @@ UniValue signmessagewithprivkey(const JSONRPCRequest& request) UniValue setmocktime(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "setmocktime timestamp\n" "\nSet the local time to given timestamp (-regtest only)\n" "\nArguments:\n" @@ -439,7 +463,7 @@ UniValue setmocktime(const JSONRPCRequest& request) ); if (!Params().MineBlocksOnDemand()) - throw runtime_error("setmocktime for regression testing (-regtest mode) only"); + throw std::runtime_error("setmocktime for regression testing (-regtest mode) only"); // For now, don't change mocktime if we're in the middle of validation, as // this could have an effect on mempool time-based eviction, as well as @@ -467,16 +491,39 @@ static UniValue RPCLockedMemoryInfo() return obj; } +#ifdef HAVE_MALLOC_INFO +static std::string RPCMallocInfo() +{ + char *ptr = nullptr; + size_t size = 0; + FILE *f = open_memstream(&ptr, &size); + if (f) { + malloc_info(0, f); + fclose(f); + if (ptr) { + std::string rv(ptr, size); + free(ptr); + return rv; + } + } + return ""; +} +#endif + UniValue getmemoryinfo(const JSONRPCRequest& request) { /* Please, avoid using the word "pool" here in the RPC interface or help, * as users will undoubtedly confuse it with the other "memory pool" */ - if (request.fHelp || request.params.size() != 0) - throw runtime_error( - "getmemoryinfo\n" + if (request.fHelp || request.params.size() > 1) + throw std::runtime_error( + "getmemoryinfo (\"mode\")\n" "Returns an object containing information about memory usage.\n" - "\nResult:\n" + "Arguments:\n" + "1. \"mode\" determines what kind of information is returned. This argument is optional, the default mode is \"stats\".\n" + " - \"stats\" returns general statistics about memory usage in the daemon.\n" + " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc 2.10+).\n" + "\nResult (mode \"stats\"):\n" "{\n" " \"locked\": { (json object) Information about locked memory manager\n" " \"used\": xxxxx, (numeric) Number of bytes used\n" @@ -487,19 +534,100 @@ UniValue getmemoryinfo(const JSONRPCRequest& request) " \"chunks_free\": xxxxx, (numeric) Number unused chunks\n" " }\n" "}\n" + "\nResult (mode \"mallocinfo\"):\n" + "\"<malloc version=\"1\">...\"\n" "\nExamples:\n" + HelpExampleCli("getmemoryinfo", "") + HelpExampleRpc("getmemoryinfo", "") ); - UniValue obj(UniValue::VOBJ); - obj.push_back(Pair("locked", RPCLockedMemoryInfo())); - return obj; + + std::string mode = (request.params.size() < 1 || request.params[0].isNull()) ? "stats" : request.params[0].get_str(); + if (mode == "stats") { + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("locked", RPCLockedMemoryInfo())); + return obj; + } else if (mode == "mallocinfo") { +#ifdef HAVE_MALLOC_INFO + return RPCMallocInfo(); +#else + throw JSONRPCError(RPC_INVALID_PARAMETER, "mallocinfo is only available when compiled with glibc 2.10+"); +#endif + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown mode " + mode); + } +} + +uint32_t getCategoryMask(UniValue cats) { + cats = cats.get_array(); + uint32_t mask = 0; + for (unsigned int i = 0; i < cats.size(); ++i) { + uint32_t flag = 0; + std::string cat = cats[i].get_str(); + if (!GetLogCategory(&flag, &cat)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown logging category " + cat); + } + mask |= flag; + } + return mask; +} + +UniValue logging(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 2) { + throw std::runtime_error( + "logging [include,...] <exclude>\n" + "Gets and sets the logging configuration.\n" + "When called without an argument, returns the list of categories that are currently being debug logged.\n" + "When called with arguments, adds or removes categories from debug logging.\n" + "The valid logging categories are: " + ListLogCategories() + "\n" + "libevent logging is configured on startup and cannot be modified by this RPC during runtime." + "Arguments:\n" + "1. \"include\" (array of strings) add debug logging for these categories.\n" + "2. \"exclude\" (array of strings) remove debug logging for these categories.\n" + "\nResult: <categories> (string): a list of the logging categories that are active.\n" + "\nExamples:\n" + + HelpExampleCli("logging", "\"[\\\"all\\\"]\" \"[\\\"http\\\"]\"") + + HelpExampleRpc("logging", "[\"all\"], \"[libevent]\"") + ); + } + + uint32_t originalLogCategories = logCategories; + if (request.params.size() > 0 && request.params[0].isArray()) { + logCategories |= getCategoryMask(request.params[0]); + } + + if (request.params.size() > 1 && request.params[1].isArray()) { + logCategories &= ~getCategoryMask(request.params[1]); + } + + // Update libevent logging if BCLog::LIBEVENT has changed. + // If the library version doesn't allow it, UpdateHTTPServerLogging() returns false, + // in which case we should clear the BCLog::LIBEVENT flag. + // Throw an error if the user has explicitly asked to change only the libevent + // flag and it failed. + uint32_t changedLogCategories = originalLogCategories ^ logCategories; + if (changedLogCategories & BCLog::LIBEVENT) { + if (!UpdateHTTPServerLogging(logCategories & BCLog::LIBEVENT)) { + logCategories &= ~BCLog::LIBEVENT; + if (changedLogCategories == BCLog::LIBEVENT) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "libevent logging cannot be updated when using libevent before v2.1.1."); + } + } + } + + UniValue result(UniValue::VOBJ); + std::vector<CLogCategoryActive> vLogCatActive = ListActiveLogCategories(); + for (const auto& logCatActive : vLogCatActive) { + result.pushKV(logCatActive.category, logCatActive.active); + } + + return result; } UniValue echo(const JSONRPCRequest& request) { if (request.fHelp) - throw runtime_error( + throw std::runtime_error( "echo|echojson \"message\" ...\n" "\nSimply echo back the input arguments. This command is for testing.\n" "\nThe difference between echo and echojson is that echojson has argument conversion enabled in the client-side table in" @@ -513,7 +641,7 @@ static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- { "control", "getinfo", &getinfo, true, {} }, /* uses wallet if enabled */ - { "control", "getmemoryinfo", &getmemoryinfo, true, {} }, + { "control", "getmemoryinfo", &getmemoryinfo, true, {"mode"} }, { "util", "validateaddress", &validateaddress, true, {"address"} }, /* uses wallet if enabled */ { "util", "createmultisig", &createmultisig, true, {"nrequired","keys"} }, { "util", "verifymessage", &verifymessage, true, {"address","signature","message"} }, @@ -522,7 +650,8 @@ static const CRPCCommand commands[] = /* Not shown in help */ { "hidden", "setmocktime", &setmocktime, true, {"timestamp"}}, { "hidden", "echo", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "echojson", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", "echojson", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", "logging", &logging, true, {"include", "exclude"}}, }; void RegisterMiscRPCCommands(CRPCTable &t) diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index f590db5efa..10bf99eb38 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -23,12 +23,10 @@ #include <univalue.h> -using namespace std; - UniValue getconnectioncount(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getconnectioncount\n" "\nReturns the number of connections to other nodes.\n" "\nResult:\n" @@ -47,7 +45,7 @@ UniValue getconnectioncount(const JSONRPCRequest& request) UniValue ping(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "ping\n" "\nRequests that a ping be sent to all other nodes, to measure ping time.\n" "Results provided in getpeerinfo, pingtime and pingwait fields are decimal seconds.\n" @@ -70,7 +68,7 @@ UniValue ping(const JSONRPCRequest& request) UniValue getpeerinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getpeerinfo\n" "\nReturns data about each connected network node as a json array of objects.\n" "\nResult:\n" @@ -78,7 +76,8 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " {\n" " \"id\": n, (numeric) Peer index\n" " \"addr\":\"host:port\", (string) The ip address and port of the peer\n" - " \"addrlocal\":\"ip:port\", (string) local address\n" + " \"addrbind\":\"ip:port\", (string) Bind address of the connection to the peer\n" + " \"addrlocal\":\"ip:port\", (string) Local address as reported by the peer\n" " \"services\":\"xxxxxxxxxxxxxxxx\", (string) The services offered\n" " \"relaytxes\":true|false, (boolean) Whether peer has asked us to relay transactions to it\n" " \"lastsend\": ttt, (numeric) The time in seconds since epoch (Jan 1 1970 GMT) of the last send\n" @@ -102,7 +101,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " n, (numeric) The heights of blocks we're currently asking from this peer\n" " ...\n" " ],\n" - " \"whitelisted\": true|false, (boolean) Whether the peer is whitelisted\n" + " \"whitelisted\": true|false, (boolean) Whether the peer is whitelisted\n" " \"bytessent_per_msg\": {\n" " \"addr\": n, (numeric) The total bytes sent aggregated by message type\n" " ...\n" @@ -122,7 +121,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - vector<CNodeStats> vstats; + std::vector<CNodeStats> vstats; g_connman->GetNodeStats(vstats); UniValue ret(UniValue::VARR); @@ -135,6 +134,8 @@ UniValue getpeerinfo(const JSONRPCRequest& request) obj.push_back(Pair("addr", stats.addrName)); if (!(stats.addrLocal.empty())) obj.push_back(Pair("addrlocal", stats.addrLocal)); + if (stats.addrBind.IsValid()) + obj.push_back(Pair("addrbind", stats.addrBind.ToString())); obj.push_back(Pair("services", strprintf("%016x", stats.nServices))); obj.push_back(Pair("relaytxes", stats.fRelayTxes)); obj.push_back(Pair("lastsend", stats.nLastSend)); @@ -191,12 +192,12 @@ UniValue getpeerinfo(const JSONRPCRequest& request) UniValue addnode(const JSONRPCRequest& request) { - string strCommand; + std::string strCommand; if (request.params.size() == 2) strCommand = request.params[1].get_str(); if (request.fHelp || request.params.size() != 2 || (strCommand != "onetry" && strCommand != "add" && strCommand != "remove")) - throw runtime_error( + throw std::runtime_error( "addnode \"node\" \"add|remove|onetry\"\n" "\nAttempts add or remove a node from the addnode list.\n" "Or try a connection to a node once.\n" @@ -211,7 +212,7 @@ UniValue addnode(const JSONRPCRequest& request) if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - string strNode = request.params[0].get_str(); + std::string strNode = request.params[0].get_str(); if (strCommand == "onetry") { @@ -236,23 +237,43 @@ UniValue addnode(const JSONRPCRequest& request) UniValue disconnectnode(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) - throw runtime_error( - "disconnectnode \"node\" \n" - "\nImmediately disconnects from the specified node.\n" + if (request.fHelp || request.params.size() == 0 || request.params.size() >= 3) + throw std::runtime_error( + "disconnectnode \"[address]\" [nodeid]\n" + "\nImmediately disconnects from the specified peer node.\n" + "\nStrictly one out of 'address' and 'nodeid' can be provided to identify the node.\n" + "\nTo disconnect by nodeid, either set 'address' to the empty string, or call using the named 'nodeid' argument only.\n" "\nArguments:\n" - "1. \"node\" (string, required) The node (see getpeerinfo for nodes)\n" + "1. \"address\" (string, optional) The IP address/port of the node\n" + "2. \"nodeid\" (number, optional) The node ID (see getpeerinfo for node IDs)\n" "\nExamples:\n" + HelpExampleCli("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleCli("disconnectnode", "\"\" 1") + HelpExampleRpc("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleRpc("disconnectnode", "\"\", 1") ); if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - bool ret = g_connman->DisconnectNode(request.params[0].get_str()); - if (!ret) + bool success; + const UniValue &address_arg = request.params[0]; + const UniValue &id_arg = request.params.size() < 2 ? NullUniValue : request.params[1]; + + if (!address_arg.isNull() && id_arg.isNull()) { + /* handle disconnect-by-address */ + success = g_connman->DisconnectNode(address_arg.get_str()); + } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { + /* handle disconnect-by-id */ + NodeId nodeid = (NodeId) id_arg.get_int64(); + success = g_connman->DisconnectNode(nodeid); + } else { + throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); + } + + if (!success) { throw JSONRPCError(RPC_CLIENT_NODE_NOT_CONNECTED, "Node not found in connected nodes"); + } return NullUniValue; } @@ -260,7 +281,7 @@ UniValue disconnectnode(const JSONRPCRequest& request) UniValue getaddednodeinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "getaddednodeinfo ( \"node\" )\n" "\nReturns information about the given added node, or all added nodes\n" "(note that onetry addnodes are not listed here)\n" @@ -328,7 +349,7 @@ UniValue getaddednodeinfo(const JSONRPCRequest& request) UniValue getnettotals(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 0) - throw runtime_error( + throw std::runtime_error( "getnettotals\n" "\nReturns information about network traffic, including bytes in, bytes out,\n" "and current time.\n" @@ -384,7 +405,7 @@ static UniValue GetNetworksInfo() obj.push_back(Pair("name", GetNetworkName(network))); obj.push_back(Pair("limited", IsLimited(network))); obj.push_back(Pair("reachable", IsReachable(network))); - obj.push_back(Pair("proxy", proxy.IsValid() ? proxy.proxy.ToStringIPPort() : string())); + obj.push_back(Pair("proxy", proxy.IsValid() ? proxy.proxy.ToStringIPPort() : std::string())); obj.push_back(Pair("proxy_randomize_credentials", proxy.randomize_credentials)); networks.push_back(obj); } @@ -394,7 +415,7 @@ static UniValue GetNetworksInfo() UniValue getnetworkinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getnetworkinfo\n" "Returns an object containing various state info regarding P2P networking.\n" "\nResult:\n" @@ -417,7 +438,7 @@ UniValue getnetworkinfo(const JSONRPCRequest& request) " }\n" " ,...\n" " ],\n" - " \"relayfee\": x.xxxxxxxx, (numeric) minimum relay fee for non-free transactions in " + CURRENCY_UNIT + "/kB\n" + " \"relayfee\": x.xxxxxxxx, (numeric) minimum relay fee for transactions in " + CURRENCY_UNIT + "/kB\n" " \"incrementalfee\": x.xxxxxxxx, (numeric) minimum fee increment for mempool limiting or BIP 125 replacement in " + CURRENCY_UNIT + "/kB\n" " \"localaddresses\": [ (array) list of local addresses\n" " {\n" @@ -469,12 +490,12 @@ UniValue getnetworkinfo(const JSONRPCRequest& request) UniValue setban(const JSONRPCRequest& request) { - string strCommand; + std::string strCommand; if (request.params.size() >= 2) strCommand = request.params[1].get_str(); if (request.fHelp || request.params.size() < 2 || (strCommand != "add" && strCommand != "remove")) - throw runtime_error( + throw std::runtime_error( "setban \"subnet\" \"add|remove\" (bantime) (absolute)\n" "\nAttempts add or remove a IP/Subnet from the banned list.\n" "\nArguments:\n" @@ -494,7 +515,7 @@ UniValue setban(const JSONRPCRequest& request) CNetAddr netAddr; bool isSubnet = false; - if (request.params[0].get_str().find("/") != string::npos) + if (request.params[0].get_str().find("/") != std::string::npos) isSubnet = true; if (!isSubnet) { @@ -506,7 +527,7 @@ UniValue setban(const JSONRPCRequest& request) LookupSubNet(request.params[0].get_str().c_str(), subNet); if (! (isSubnet ? subNet.IsValid() : netAddr.IsValid()) ) - throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: Invalid IP/Subnet"); + throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Invalid IP/Subnet"); if (strCommand == "add") { @@ -526,7 +547,7 @@ UniValue setban(const JSONRPCRequest& request) else if(strCommand == "remove") { if (!( isSubnet ? g_connman->Unban(subNet) : g_connman->Unban(netAddr) )) - throw JSONRPCError(RPC_MISC_ERROR, "Error: Unban failed"); + throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously banned."); } return NullUniValue; } @@ -534,7 +555,7 @@ UniValue setban(const JSONRPCRequest& request) UniValue listbanned(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "listbanned\n" "\nList all banned IPs/Subnets.\n" "\nExamples:\n" @@ -567,7 +588,7 @@ UniValue listbanned(const JSONRPCRequest& request) UniValue clearbanned(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "clearbanned\n" "\nClear all banned IPs.\n" "\nExamples:\n" @@ -585,7 +606,7 @@ UniValue clearbanned(const JSONRPCRequest& request) UniValue setnetworkactive(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) { - throw runtime_error( + throw std::runtime_error( "setnetworkactive true|false\n" "\nDisable/enable all p2p network activity.\n" "\nArguments:\n" @@ -609,7 +630,7 @@ static const CRPCCommand commands[] = { "network", "ping", &ping, true, {} }, { "network", "getpeerinfo", &getpeerinfo, true, {} }, { "network", "addnode", &addnode, true, {"node","command"} }, - { "network", "disconnectnode", &disconnectnode, true, {"node"} }, + { "network", "disconnectnode", &disconnectnode, true, {"address", "nodeid"} }, { "network", "getaddednodeinfo", &getaddednodeinfo, true, {"node"} }, { "network", "getnettotals", &getnettotals, true, {} }, { "network", "getnetworkinfo", &getnetworkinfo, true, {} }, diff --git a/src/rpc/protocol.cpp b/src/rpc/protocol.cpp index dc710d939f..823a5775f6 100644 --- a/src/rpc/protocol.cpp +++ b/src/rpc/protocol.cpp @@ -15,8 +15,6 @@ #include <stdint.h> #include <fstream> -using namespace std; - /** * JSON-RPC protocol. Bitcoin speaks version 1.0 for maximum compatibility, * but uses JSON-RPC 1.1/2.0 standards for parts of the 1.0 standard that were @@ -26,7 +24,7 @@ using namespace std; * 1.2 spec: http://jsonrpc.org/historical/json-rpc-over-http.html */ -UniValue JSONRPCRequestObj(const string& strMethod, const UniValue& params, const UniValue& id) +UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id) { UniValue request(UniValue::VOBJ); request.push_back(Pair("method", strMethod)); @@ -47,13 +45,13 @@ UniValue JSONRPCReplyObj(const UniValue& result, const UniValue& error, const Un return reply; } -string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id) +std::string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id) { UniValue reply = JSONRPCReplyObj(result, error, id); return reply.write() + "\n"; } -UniValue JSONRPCError(int code, const string& message) +UniValue JSONRPCError(int code, const std::string& message) { UniValue error(UniValue::VOBJ); error.push_back(Pair("code", code)); @@ -68,9 +66,9 @@ static const std::string COOKIEAUTH_USER = "__cookie__"; /** Default name for auth cookie file */ static const std::string COOKIEAUTH_FILE = ".cookie"; -boost::filesystem::path GetAuthCookieFile() +fs::path GetAuthCookieFile() { - boost::filesystem::path path(GetArg("-rpccookiefile", COOKIEAUTH_FILE)); + fs::path path(GetArg("-rpccookiefile", COOKIEAUTH_FILE)); if (!path.is_complete()) path = GetDataDir() / path; return path; } @@ -86,7 +84,7 @@ bool GenerateAuthCookie(std::string *cookie_out) * these are set to 077 in init.cpp unless overridden with -sysperms. */ std::ofstream file; - boost::filesystem::path filepath = GetAuthCookieFile(); + fs::path filepath = GetAuthCookieFile(); file.open(filepath.string().c_str()); if (!file.is_open()) { LogPrintf("Unable to open cookie authentication file %s for writing\n", filepath.string()); @@ -105,7 +103,7 @@ bool GetAuthCookie(std::string *cookie_out) { std::ifstream file; std::string cookie; - boost::filesystem::path filepath = GetAuthCookieFile(); + fs::path filepath = GetAuthCookieFile(); file.open(filepath.string().c_str()); if (!file.is_open()) return false; @@ -120,8 +118,8 @@ bool GetAuthCookie(std::string *cookie_out) void DeleteAuthCookie() { try { - boost::filesystem::remove(GetAuthCookieFile()); - } catch (const boost::filesystem::filesystem_error& e) { + fs::remove(GetAuthCookieFile()); + } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove random auth cookie file: %s\n", __func__, e.what()); } } diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index 47e56e712b..70f7092cfe 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -6,11 +6,12 @@ #ifndef BITCOIN_RPCPROTOCOL_H #define BITCOIN_RPCPROTOCOL_H +#include "fs.h" + #include <list> #include <map> #include <stdint.h> #include <string> -#include <boost/filesystem.hpp> #include <univalue.h> @@ -31,9 +32,15 @@ enum HTTPStatusCode enum RPCErrorCode { //! Standard JSON-RPC 2.0 errors + // RPC_INVALID_REQUEST is internally mapped to HTTP_BAD_REQUEST (400). + // It should not be used for application-layer errors. RPC_INVALID_REQUEST = -32600, + // RPC_METHOD_NOT_FOUND is internally mapped to HTTP_NOT_FOUND (404). + // It should not be used for application-layer errors. RPC_METHOD_NOT_FOUND = -32601, RPC_INVALID_PARAMS = -32602, + // RPC_INTERNAL_ERROR should only be used for genuine errors in bitcoind + // (for example datadir corruption). RPC_INTERNAL_ERROR = -32603, RPC_PARSE_ERROR = -32700, @@ -83,7 +90,7 @@ std::string JSONRPCReply(const UniValue& result, const UniValue& error, const Un UniValue JSONRPCError(int code, const std::string& message); /** Get name of RPC authentication cookie file */ -boost::filesystem::path GetAuthCookieFile(); +fs::path GetAuthCookieFile(); /** Generate a new RPC authentication cookie and write it to disk */ bool GenerateAuthCookie(std::string *cookie_out); /** Read the RPC authentication cookie from disk */ diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index bf16f27498..e27c2a77c7 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -24,6 +24,7 @@ #include "uint256.h" #include "utilstrencodings.h" #ifdef ENABLE_WALLET +#include "wallet/rpcwallet.h" #include "wallet/wallet.h" #endif @@ -33,79 +34,15 @@ #include <univalue.h> -using namespace std; - -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex) -{ - txnouttype type; - vector<CTxDestination> addresses; - int nRequired; - - out.push_back(Pair("asm", ScriptToAsmStr(scriptPubKey))); - if (fIncludeHex) - out.push_back(Pair("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end()))); - - if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired)) { - out.push_back(Pair("type", GetTxnOutputType(type))); - return; - } - - out.push_back(Pair("reqSigs", nRequired)); - out.push_back(Pair("type", GetTxnOutputType(type))); - - UniValue a(UniValue::VARR); - BOOST_FOREACH(const CTxDestination& addr, addresses) - a.push_back(CBitcoinAddress(addr).ToString()); - out.push_back(Pair("addresses", a)); -} void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { - entry.push_back(Pair("txid", tx.GetHash().GetHex())); - entry.push_back(Pair("hash", tx.GetWitnessHash().GetHex())); - entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); - entry.push_back(Pair("vsize", (int)::GetVirtualTransactionSize(tx))); - entry.push_back(Pair("version", tx.nVersion)); - entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); - - UniValue vin(UniValue::VARR); - for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxIn& txin = tx.vin[i]; - UniValue in(UniValue::VOBJ); - if (tx.IsCoinBase()) - in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); - else { - in.push_back(Pair("txid", txin.prevout.hash.GetHex())); - in.push_back(Pair("vout", (int64_t)txin.prevout.n)); - UniValue o(UniValue::VOBJ); - o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); - o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); - in.push_back(Pair("scriptSig", o)); - } - if (tx.HasWitness()) { - UniValue txinwitness(UniValue::VARR); - for (unsigned int j = 0; j < tx.vin[i].scriptWitness.stack.size(); j++) { - std::vector<unsigned char> item = tx.vin[i].scriptWitness.stack[j]; - txinwitness.push_back(HexStr(item.begin(), item.end())); - } - in.push_back(Pair("txinwitness", txinwitness)); - } - in.push_back(Pair("sequence", (int64_t)txin.nSequence)); - vin.push_back(in); - } - entry.push_back(Pair("vin", vin)); - UniValue vout(UniValue::VARR); - for (unsigned int i = 0; i < tx.vout.size(); i++) { - const CTxOut& txout = tx.vout[i]; - UniValue out(UniValue::VOBJ); - out.push_back(Pair("value", ValueFromAmount(txout.nValue))); - out.push_back(Pair("n", (int64_t)i)); - UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(txout.scriptPubKey, o, true); - out.push_back(Pair("scriptPubKey", o)); - vout.push_back(out); - } - entry.push_back(Pair("vout", vout)); + // Call into TxToUniv() in bitcoin-common to decode the transaction hex. + // + // Blockchain contextual information (confirmations and blocktime) is not + // available to code in bitcoin-common, so we query them here and push the + // data into the returned UniValue. + TxToUniv(tx, uint256(), entry); if (!hashBlock.IsNull()) { entry.push_back(Pair("blockhash", hashBlock.GetHex())); @@ -126,7 +63,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) UniValue getrawtransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "getrawtransaction \"txid\" ( verbose )\n" "\nNOTE: By default this function only works for mempool transactions. If the -txindex option is\n" @@ -139,7 +76,7 @@ UniValue getrawtransaction(const JSONRPCRequest& request) "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" - "2. verbose (bool, optional, default=false) If true, return a string, other return a json object\n" + "2. verbose (bool, optional, default=false) If false, return a string, otherwise return a json object\n" "\nResult (if verbose is not set or set to false):\n" "\"data\" (string) The serialized, hex-encoded data for 'txid'\n" @@ -214,7 +151,7 @@ UniValue getrawtransaction(const JSONRPCRequest& request) } else { throw JSONRPCError(RPC_TYPE_ERROR, "Invalid type provided. Verbose parameter must be a boolean."); - } + } } CTransactionRef tx; @@ -224,7 +161,7 @@ UniValue getrawtransaction(const JSONRPCRequest& request) : "No such mempool transaction. Use -txindex to enable blockchain transaction queries") + ". Use gettransaction for wallet transactions."); - string strHex = EncodeHexTx(*tx, RPCSerializationFlags()); + std::string strHex = EncodeHexTx(*tx, RPCSerializationFlags()); if (!fVerbose) return strHex; @@ -238,7 +175,7 @@ UniValue getrawtransaction(const JSONRPCRequest& request) UniValue gettxoutproof(const JSONRPCRequest& request) { if (request.fHelp || (request.params.size() != 1 && request.params.size() != 2)) - throw runtime_error( + throw std::runtime_error( "gettxoutproof [\"txid\",...] ( blockhash )\n" "\nReturns a hex-encoded proof that \"txid\" was included in a block.\n" "\nNOTE: By default this function only works sometimes. This is when there is an\n" @@ -256,16 +193,16 @@ UniValue gettxoutproof(const JSONRPCRequest& request) "\"data\" (string) A string that is a serialized, hex-encoded data for the proof.\n" ); - set<uint256> setTxids; + std::set<uint256> setTxids; uint256 oneTxid; UniValue txids = request.params[0].get_array(); for (unsigned int idx = 0; idx < txids.size(); idx++) { const UniValue& txid = txids[idx]; if (txid.get_str().length() != 64 || !IsHex(txid.get_str())) - throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid txid ")+txid.get_str()); + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid txid ")+txid.get_str()); uint256 hash(uint256S(txid.get_str())); if (setTxids.count(hash)) - throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid parameter, duplicated txid: ")+txid.get_str()); + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ")+txid.get_str()); setTxids.insert(hash); oneTxid = hash; } @@ -282,9 +219,10 @@ UniValue gettxoutproof(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); pblockindex = mapBlockIndex[hashBlock]; } else { - CCoins coins; - if (pcoinsTip->GetCoins(oneTxid, coins) && coins.nHeight > 0 && coins.nHeight <= chainActive.Height()) - pblockindex = chainActive[coins.nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, oneTxid); + if (!coin.IsSpent() && coin.nHeight > 0 && coin.nHeight <= chainActive.Height()) { + pblockindex = chainActive[coin.nHeight]; + } } if (pblockindex == NULL) @@ -318,7 +256,7 @@ UniValue gettxoutproof(const JSONRPCRequest& request) UniValue verifytxoutproof(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "verifytxoutproof \"proof\"\n" "\nVerifies that a proof points to a transaction in a block, returning the transaction it commits to\n" "and throwing an RPC error if the block is not in our best chain\n" @@ -334,8 +272,8 @@ UniValue verifytxoutproof(const JSONRPCRequest& request) UniValue res(UniValue::VARR); - vector<uint256> vMatch; - vector<unsigned int> vIndex; + std::vector<uint256> vMatch; + std::vector<unsigned int> vIndex; if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) != merkleBlock.header.hashMerkleRoot) return res; @@ -352,7 +290,7 @@ UniValue verifytxoutproof(const JSONRPCRequest& request) UniValue createrawtransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 3) - throw runtime_error( + throw std::runtime_error( "createrawtransaction [{\"txid\":\"id\",\"vout\":n},...] {\"address\":amount,\"data\":\"hex\",...} ( locktime )\n" "\nCreate a transaction spending the given inputs and creating new outputs.\n" "Outputs can be addresses or data.\n" @@ -361,7 +299,7 @@ UniValue createrawtransaction(const JSONRPCRequest& request) "it is not stored in the wallet or transmitted to the network.\n" "\nArguments:\n" - "1. \"inputs\" (string, required) A json array of json objects\n" + "1. \"inputs\" (array, required) A json array of json objects\n" " [\n" " {\n" " \"txid\":\"id\", (string, required) The transaction id\n" @@ -370,7 +308,7 @@ UniValue createrawtransaction(const JSONRPCRequest& request) " } \n" " ,...\n" " ]\n" - "2. \"outputs\" (string, required) a json object with outputs\n" + "2. \"outputs\" (object, required) a json object with outputs\n" " {\n" " \"address\": x.xxx, (numeric or string, required) The key is the bitcoin address, the numeric value (can be string) is the " + CURRENCY_UNIT + " amount\n" " \"data\": \"hex\" (string, required) The key is \"data\", the value is hex encoded data\n" @@ -433,9 +371,9 @@ UniValue createrawtransaction(const JSONRPCRequest& request) rawTx.vin.push_back(in); } - set<CBitcoinAddress> setAddress; - vector<string> addrList = sendTo.getKeys(); - BOOST_FOREACH(const string& name_, addrList) { + std::set<CBitcoinAddress> setAddress; + std::vector<std::string> addrList = sendTo.getKeys(); + BOOST_FOREACH(const std::string& name_, addrList) { if (name_ == "data") { std::vector<unsigned char> data = ParseHexV(sendTo[name_].getValStr(),"Data"); @@ -445,10 +383,10 @@ UniValue createrawtransaction(const JSONRPCRequest& request) } else { CBitcoinAddress address(name_); if (!address.IsValid()) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, string("Invalid Bitcoin address: ")+name_); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ")+name_); if (setAddress.count(address)) - throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid parameter, duplicated address: ")+name_); + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ")+name_); setAddress.insert(address); CScript scriptPubKey = GetScriptForDestination(address.Get()); @@ -465,7 +403,7 @@ UniValue createrawtransaction(const JSONRPCRequest& request) UniValue decoderawtransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "decoderawtransaction \"hexstring\"\n" "\nReturn a JSON object representing the serialized, hex-encoded transaction.\n" @@ -526,7 +464,7 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); UniValue result(UniValue::VOBJ); - TxToJSON(CTransaction(std::move(mtx)), uint256(), result); + TxToUniv(CTransaction(std::move(mtx)), uint256(), result); return result; } @@ -534,7 +472,7 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) UniValue decodescript(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "decodescript \"hexstring\"\n" "\nDecode a hex-encoded script.\n" "\nArguments:\n" @@ -561,12 +499,12 @@ UniValue decodescript(const JSONRPCRequest& request) UniValue r(UniValue::VOBJ); CScript script; if (request.params[0].get_str().size() > 0){ - vector<unsigned char> scriptData(ParseHexV(request.params[0], "argument")); + std::vector<unsigned char> scriptData(ParseHexV(request.params[0], "argument")); script = CScript(scriptData.begin(), scriptData.end()); } else { // Empty scripts are valid } - ScriptPubKeyToJSON(script, r, false); + ScriptPubKeyToUniv(script, r, false); UniValue type; type = find_value(r, "type"); @@ -586,6 +524,11 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: UniValue entry(UniValue::VOBJ); entry.push_back(Pair("txid", txin.prevout.hash.ToString())); entry.push_back(Pair("vout", (uint64_t)txin.prevout.n)); + UniValue witness(UniValue::VARR); + for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { + witness.push_back(HexStr(txin.scriptWitness.stack[i].begin(), txin.scriptWitness.stack[i].end())); + } + entry.push_back(Pair("witness", witness)); entry.push_back(Pair("scriptSig", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); entry.push_back(Pair("sequence", (uint64_t)txin.nSequence)); entry.push_back(Pair("error", strMessage)); @@ -594,8 +537,12 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: UniValue signrawtransaction(const JSONRPCRequest& request) { +#ifdef ENABLE_WALLET + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); +#endif + if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) - throw runtime_error( + throw std::runtime_error( "signrawtransaction \"hexstring\" ( [{\"txid\":\"id\",\"vout\":n,\"scriptPubKey\":\"hex\",\"redeemScript\":\"hex\"},...] [\"privatekey1\",...] sighashtype )\n" "\nSign inputs for raw transaction (serialized, hex-encoded).\n" "The second optional argument (may be null) is an array of previous transaction outputs that\n" @@ -603,7 +550,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) "The third optional argument (may be null) is an array of base58-encoded private\n" "keys that, if given, will be the only keys used to sign the transaction.\n" #ifdef ENABLE_WALLET - + HelpRequiringPassphrase() + "\n" + + HelpRequiringPassphrase(pwallet) + "\n" #endif "\nArguments:\n" @@ -654,15 +601,15 @@ UniValue signrawtransaction(const JSONRPCRequest& request) ); #ifdef ENABLE_WALLET - LOCK2(cs_main, pwalletMain ? &pwalletMain->cs_wallet : NULL); + LOCK2(cs_main, pwallet ? &pwallet->cs_wallet : NULL); #else LOCK(cs_main); #endif RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VSTR)(UniValue::VARR)(UniValue::VARR)(UniValue::VSTR), true); - vector<unsigned char> txData(ParseHexV(request.params[0], "argument 1")); + std::vector<unsigned char> txData(ParseHexV(request.params[0], "argument 1")); CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION); - vector<CMutableTransaction> txVariants; + std::vector<CMutableTransaction> txVariants; while (!ssData.empty()) { try { CMutableTransaction tx; @@ -691,9 +638,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view BOOST_FOREACH(const CTxIn& txin, mergedTx.vin) { - const uint256& prevHash = txin.prevout.hash; - CCoins coins; - view.AccessCoins(prevHash); // this is certainly allowed to fail + view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. } view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long @@ -717,8 +662,9 @@ UniValue signrawtransaction(const JSONRPCRequest& request) } } #ifdef ENABLE_WALLET - else if (pwalletMain) - EnsureWalletIsUnlocked(); + else if (pwallet) { + EnsureWalletIsUnlocked(pwallet); + } #endif // Add previous txouts given in the RPC call: @@ -744,24 +690,26 @@ UniValue signrawtransaction(const JSONRPCRequest& request) if (nOut < 0) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout must be positive"); - vector<unsigned char> pkData(ParseHexO(prevOut, "scriptPubKey")); + COutPoint out(txid, nOut); + std::vector<unsigned char> pkData(ParseHexO(prevOut, "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { - string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { + std::string err("Previous output scriptPubKey mismatch:\n"); + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(find_value(prevOut, "amount")); + newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and not using the local wallet (private keys @@ -776,7 +724,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) }); UniValue v = find_value(prevOut, "redeemScript"); if (!v.isNull()) { - vector<unsigned char> rsData(ParseHexV(v, "redeemScript")); + std::vector<unsigned char> rsData(ParseHexV(v, "redeemScript")); CScript redeemScript(rsData.begin(), rsData.end()); tempKeystore.AddCScript(redeemScript); } @@ -785,23 +733,23 @@ UniValue signrawtransaction(const JSONRPCRequest& request) } #ifdef ENABLE_WALLET - const CKeyStore& keystore = ((fGivenKeys || !pwalletMain) ? tempKeystore : *pwalletMain); + const CKeyStore& keystore = ((fGivenKeys || !pwallet) ? tempKeystore : *pwallet); #else const CKeyStore& keystore = tempKeystore; #endif int nHashType = SIGHASH_ALL; if (request.params.size() > 3 && !request.params[3].isNull()) { - static map<string, int> mapSigHashValues = + static std::map<std::string, int> mapSigHashValues = boost::assign::map_list_of - (string("ALL"), int(SIGHASH_ALL)) - (string("ALL|ANYONECANPAY"), int(SIGHASH_ALL|SIGHASH_ANYONECANPAY)) - (string("NONE"), int(SIGHASH_NONE)) - (string("NONE|ANYONECANPAY"), int(SIGHASH_NONE|SIGHASH_ANYONECANPAY)) - (string("SINGLE"), int(SIGHASH_SINGLE)) - (string("SINGLE|ANYONECANPAY"), int(SIGHASH_SINGLE|SIGHASH_ANYONECANPAY)) + (std::string("ALL"), int(SIGHASH_ALL)) + (std::string("ALL|ANYONECANPAY"), int(SIGHASH_ALL|SIGHASH_ANYONECANPAY)) + (std::string("NONE"), int(SIGHASH_NONE)) + (std::string("NONE|ANYONECANPAY"), int(SIGHASH_NONE|SIGHASH_ANYONECANPAY)) + (std::string("SINGLE"), int(SIGHASH_SINGLE)) + (std::string("SINGLE|ANYONECANPAY"), int(SIGHASH_SINGLE|SIGHASH_ANYONECANPAY)) ; - string strHashType = request.params[3].get_str(); + std::string strHashType = request.params[3].get_str(); if (mapSigHashValues.count(strHashType)) nHashType = mapSigHashValues[strHashType]; else @@ -819,13 +767,13 @@ UniValue signrawtransaction(const JSONRPCRequest& request) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (coins == NULL || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: @@ -861,7 +809,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) UniValue sendrawtransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "sendrawtransaction \"hexstring\" ( allowhighfees )\n" "\nSubmits raw transaction (serialized, hex-encoded) to local node and network.\n" "\nAlso see createrawtransaction and signrawtransaction calls.\n" @@ -891,15 +839,18 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) CTransactionRef tx(MakeTransactionRef(std::move(mtx))); const uint256& hashTx = tx->GetHash(); - bool fLimitFree = false; + bool fLimitFree = true; CAmount nMaxRawTxFee = maxTxFee; if (request.params.size() > 1 && request.params[1].get_bool()) nMaxRawTxFee = 0; CCoinsViewCache &view = *pcoinsTip; - const CCoins* existingCoins = view.AccessCoins(hashTx); + bool fHaveChain = false; + for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { + const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); + fHaveChain = !existingCoin.IsSpent(); + } bool fHaveMempool = mempool.exists(hashTx); - bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; if (!fHaveMempool && !fHaveChain) { // push to local node and sync with wallets CValidationState state; @@ -932,7 +883,7 @@ static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- { "rawtransactions", "getrawtransaction", &getrawtransaction, true, {"txid","verbose"} }, - { "rawtransactions", "createrawtransaction", &createrawtransaction, true, {"transactions","outputs","locktime"} }, + { "rawtransactions", "createrawtransaction", &createrawtransaction, true, {"inputs","outputs","locktime"} }, { "rawtransactions", "decoderawtransaction", &decoderawtransaction, true, {"hexstring"} }, { "rawtransactions", "decodescript", &decodescript, true, {"hexstring"} }, { "rawtransactions", "sendrawtransaction", &sendrawtransaction, false, {"hexstring","allowhighfees"} }, diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 0b763acd45..c5fbff0077 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -6,6 +6,7 @@ #include "rpc/server.h" #include "base58.h" +#include "fs.h" #include "init.h" #include "random.h" #include "sync.h" @@ -16,19 +17,17 @@ #include <univalue.h> #include <boost/bind.hpp> -#include <boost/filesystem.hpp> #include <boost/foreach.hpp> #include <boost/shared_ptr.hpp> #include <boost/signals2/signal.hpp> #include <boost/thread.hpp> #include <boost/algorithm/string/case_conv.hpp> // for to_upper() +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> #include <memory> // for unique_ptr #include <unordered_map> -using namespace RPCServer; -using namespace std; - static bool fRPCRunning = false; static bool fRPCInWarmup = true; static std::string rpcWarmupStatus("RPC server started"); @@ -43,31 +42,25 @@ static struct CRPCSignals boost::signals2::signal<void ()> Started; boost::signals2::signal<void ()> Stopped; boost::signals2::signal<void (const CRPCCommand&)> PreCommand; - boost::signals2::signal<void (const CRPCCommand&)> PostCommand; } g_rpcSignals; -void RPCServer::OnStarted(boost::function<void ()> slot) +void RPCServer::OnStarted(std::function<void ()> slot) { g_rpcSignals.Started.connect(slot); } -void RPCServer::OnStopped(boost::function<void ()> slot) +void RPCServer::OnStopped(std::function<void ()> slot) { g_rpcSignals.Stopped.connect(slot); } -void RPCServer::OnPreCommand(boost::function<void (const CRPCCommand&)> slot) +void RPCServer::OnPreCommand(std::function<void (const CRPCCommand&)> slot) { g_rpcSignals.PreCommand.connect(boost::bind(slot, _1)); } -void RPCServer::OnPostCommand(boost::function<void (const CRPCCommand&)> slot) -{ - g_rpcSignals.PostCommand.connect(boost::bind(slot, _1)); -} - void RPCTypeCheck(const UniValue& params, - const list<UniValue::VType>& typesExpected, + const std::list<UniValue::VType>& typesExpected, bool fAllowNull) { unsigned int i = 0; @@ -92,7 +85,7 @@ void RPCTypeCheckArgument(const UniValue& value, UniValue::VType typeExpected) } void RPCTypeCheckObj(const UniValue& o, - const map<string, UniValueType>& typesExpected, + const std::map<std::string, UniValueType>& typesExpected, bool fAllowNull, bool fStrict) { @@ -102,7 +95,7 @@ void RPCTypeCheckObj(const UniValue& o, throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing %s", t.first)); if (!(t.second.typeAny || v.type() == t.second.type || (fAllowNull && v.isNull()))) { - string err = strprintf("Expected type %s for %s, got %s", + std::string err = strprintf("Expected type %s for %s, got %s", uvTypeName(t.second.type), t.first, uvTypeName(v.type())); throw JSONRPCError(RPC_TYPE_ERROR, err); } @@ -110,11 +103,11 @@ void RPCTypeCheckObj(const UniValue& o, if (fStrict) { - BOOST_FOREACH(const string& k, o.getKeys()) + BOOST_FOREACH(const std::string& k, o.getKeys()) { if (typesExpected.count(k) == 0) { - string err = strprintf("Unexpected key %s", k); + std::string err = strprintf("Unexpected key %s", k); throw JSONRPCError(RPC_TYPE_ERROR, err); } } @@ -143,9 +136,9 @@ UniValue ValueFromAmount(const CAmount& amount) strprintf("%s%d.%08d", sign ? "-" : "", quotient, remainder)); } -uint256 ParseHashV(const UniValue& v, string strName) +uint256 ParseHashV(const UniValue& v, std::string strName) { - string strHex; + std::string strHex; if (v.isStr()) strHex = v.get_str(); if (!IsHex(strHex)) // Note: IsHex("") is false @@ -156,20 +149,20 @@ uint256 ParseHashV(const UniValue& v, string strName) result.SetHex(strHex); return result; } -uint256 ParseHashO(const UniValue& o, string strKey) +uint256 ParseHashO(const UniValue& o, std::string strKey) { return ParseHashV(find_value(o, strKey), strKey); } -vector<unsigned char> ParseHexV(const UniValue& v, string strName) +std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) { - string strHex; + std::string strHex; if (v.isStr()) strHex = v.get_str(); if (!IsHex(strHex)) throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); return ParseHex(strHex); } -vector<unsigned char> ParseHexO(const UniValue& o, string strKey) +std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) { return ParseHexV(find_value(o, strKey), strKey); } @@ -178,30 +171,30 @@ vector<unsigned char> ParseHexO(const UniValue& o, string strKey) * Note: This interface may still be subject to change. */ -std::string CRPCTable::help(const std::string& strCommand) const +std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& helpreq) const { - string strRet; - string category; - set<rpcfn_type> setDone; - vector<pair<string, const CRPCCommand*> > vCommands; + std::string strRet; + std::string category; + std::set<rpcfn_type> setDone; + std::vector<std::pair<std::string, const CRPCCommand*> > vCommands; - for (map<string, const CRPCCommand*>::const_iterator mi = mapCommands.begin(); mi != mapCommands.end(); ++mi) + for (std::map<std::string, const CRPCCommand*>::const_iterator mi = mapCommands.begin(); mi != mapCommands.end(); ++mi) vCommands.push_back(make_pair(mi->second->category + mi->first, mi->second)); sort(vCommands.begin(), vCommands.end()); - BOOST_FOREACH(const PAIRTYPE(string, const CRPCCommand*)& command, vCommands) + JSONRPCRequest jreq(helpreq); + jreq.fHelp = true; + jreq.params = UniValue(); + + BOOST_FOREACH(const PAIRTYPE(std::string, const CRPCCommand*)& command, vCommands) { const CRPCCommand *pcmd = command.second; - string strMethod = pcmd->name; - // We already filter duplicates, but these deprecated screw up the sort order - if (strMethod.find("label") != string::npos) - continue; + std::string strMethod = pcmd->name; if ((strCommand != "" || pcmd->category == "hidden") && strMethod != strCommand) continue; + jreq.strMethod = strMethod; try { - JSONRPCRequest jreq; - jreq.fHelp = true; rpcfn_type pfn = pcmd->actor; if (setDone.insert(pfn).second) (*pfn)(jreq); @@ -209,10 +202,10 @@ std::string CRPCTable::help(const std::string& strCommand) const catch (const std::exception& e) { // Help text is returned in an exception - string strHelp = string(e.what()); + std::string strHelp = std::string(e.what()); if (strCommand == "") { - if (strHelp.find('\n') != string::npos) + if (strHelp.find('\n') != std::string::npos) strHelp = strHelp.substr(0, strHelp.find('\n')); if (category != pcmd->category) @@ -220,7 +213,7 @@ std::string CRPCTable::help(const std::string& strCommand) const if (!category.empty()) strRet += "\n"; category = pcmd->category; - string firstLetter = category.substr(0,1); + std::string firstLetter = category.substr(0,1); boost::to_upper(firstLetter); strRet += "== " + firstLetter + category.substr(1) + " ==\n"; } @@ -237,7 +230,7 @@ std::string CRPCTable::help(const std::string& strCommand) const UniValue help(const JSONRPCRequest& jsonRequest) { if (jsonRequest.fHelp || jsonRequest.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "help ( \"command\" )\n" "\nList all commands, or get help for a specified command.\n" "\nArguments:\n" @@ -246,11 +239,11 @@ UniValue help(const JSONRPCRequest& jsonRequest) "\"text\" (string) The help text\n" ); - string strCommand; + std::string strCommand; if (jsonRequest.params.size() > 0) strCommand = jsonRequest.params[0].get_str(); - return tableRPC.help(strCommand); + return tableRPC.help(strCommand, jsonRequest); } @@ -258,7 +251,7 @@ UniValue stop(const JSONRPCRequest& jsonRequest) { // Accept the deprecated and ignored 'detach' boolean argument if (jsonRequest.fHelp || jsonRequest.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "stop\n" "\nStop Bitcoin server."); // Event loop will exit after current HTTP requests have been handled, so @@ -292,7 +285,7 @@ CRPCTable::CRPCTable() const CRPCCommand *CRPCTable::operator[](const std::string &name) const { - map<string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); + std::map<std::string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); if (it == mapCommands.end()) return NULL; return (*it).second; @@ -304,7 +297,7 @@ bool CRPCTable::appendCommand(const std::string& name, const CRPCCommand* pcmd) return false; // don't allow overwriting for now - map<string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); + std::map<std::string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); if (it != mapCommands.end()) return false; @@ -314,7 +307,7 @@ bool CRPCTable::appendCommand(const std::string& name, const CRPCCommand* pcmd) bool StartRPC() { - LogPrint("rpc", "Starting RPC\n"); + LogPrint(BCLog::RPC, "Starting RPC\n"); fRPCRunning = true; g_rpcSignals.Started(); return true; @@ -322,15 +315,16 @@ bool StartRPC() void InterruptRPC() { - LogPrint("rpc", "Interrupting RPC\n"); + LogPrint(BCLog::RPC, "Interrupting RPC\n"); // Interrupt e.g. running longpolls fRPCRunning = false; } void StopRPC() { - LogPrint("rpc", "Stopping RPC\n"); + LogPrint(BCLog::RPC, "Stopping RPC\n"); deadlineTimers.clear(); + DeleteAuthCookie(); g_rpcSignals.Stopped(); } @@ -377,8 +371,7 @@ void JSONRPCRequest::parse(const UniValue& valRequest) if (!valMethod.isStr()) throw JSONRPCError(RPC_INVALID_REQUEST, "Method must be a string"); strMethod = valMethod.get_str(); - if (strMethod != "getblocktemplate") - LogPrint("rpc", "ThreadRPCServer method=%s\n", SanitizeString(strMethod)); + LogPrint(BCLog::RPC, "ThreadRPCServer method=%s\n", SanitizeString(strMethod)); // Parse params UniValue valParams = find_value(request, "params"); @@ -441,8 +434,16 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c } // Process expected parameters. int hole = 0; - for (const std::string &argName: argNames) { - auto fr = argsIn.find(argName); + for (const std::string &argNamePattern: argNames) { + std::vector<std::string> vargNames; + boost::algorithm::split(vargNames, argNamePattern, boost::algorithm::is_any_of("|")); + auto fr = argsIn.end(); + for (const std::string & argName : vargNames) { + fr = argsIn.find(argName); + if (fr != argsIn.end()) { + break; + } + } if (fr != argsIn.end()) { for (int i = 0; i < hole; ++i) { // Fill hole between specified parameters with JSON nulls, @@ -494,8 +495,6 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const { throw JSONRPCError(RPC_MISC_ERROR, e.what()); } - - g_rpcSignals.PostCommand(*pcmd); } std::vector<std::string> CRPCTable::listCommands() const @@ -537,12 +536,12 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface) timerInterface = NULL; } -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds) +void RPCRunLater(const std::string& name, std::function<void(void)> func, int64_t nSeconds) { if (!timerInterface) throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); deadlineTimers.erase(name); - LogPrint("rpc", "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); + LogPrint(BCLog::RPC, "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); } diff --git a/src/rpc/server.h b/src/rpc/server.h index 52f82866dc..1e984cbc0d 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -15,8 +15,6 @@ #include <stdint.h> #include <string> -#include <boost/function.hpp> - #include <univalue.h> static const unsigned int DEFAULT_RPC_SERIALIZE_VERSION = 1; @@ -25,10 +23,9 @@ class CRPCCommand; namespace RPCServer { - void OnStarted(boost::function<void ()> slot); - void OnStopped(boost::function<void ()> slot); - void OnPreCommand(boost::function<void (const CRPCCommand&)> slot); - void OnPostCommand(boost::function<void (const CRPCCommand&)> slot); + void OnStarted(std::function<void ()> slot); + void OnStopped(std::function<void ()> slot); + void OnPreCommand(std::function<void (const CRPCCommand&)> slot); } class CBlockIndex; @@ -69,7 +66,7 @@ void SetRPCWarmupStatus(const std::string& newStatus); void SetRPCWarmupFinished(); /* returns the current warmup state. */ -bool RPCIsInWarmup(std::string *statusOut); +bool RPCIsInWarmup(std::string *outStatus); /** * Type-check arguments; throws JSONRPCError if wrong type given. Does not check that @@ -116,7 +113,7 @@ public: * This is needed to cope with the case in which there is no HTTP server, but * only GUI RPC console, and to break the dependency of pcserver on httprpc. */ - virtual RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) = 0; + virtual RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) = 0; }; /** Set the factory function for timers */ @@ -130,7 +127,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface); * Run func nSeconds from now. * Overrides previous timer <name> (if any). */ -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds); +void RPCRunLater(const std::string& name, std::function<void(void)> func, int64_t nSeconds); typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); @@ -154,7 +151,7 @@ private: public: CRPCTable(); const CRPCCommand* operator[](const std::string& name) const; - std::string help(const std::string& name) const; + std::string help(const std::string& name, const JSONRPCRequest& helpreq) const; /** * Execute a method. @@ -190,21 +187,15 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); -extern int64_t nWalletUnlockTime; extern CAmount AmountFromValue(const UniValue& value); extern UniValue ValueFromAmount(const CAmount& amount); -extern double GetDifficulty(const CBlockIndex* blockindex = NULL); -extern std::string HelpRequiringPassphrase(); extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); -extern void EnsureWalletIsUnlocked(); - bool StartRPC(); void InterruptRPC(); void StopRPC(); std::string JSONRPCExecBatch(const UniValue& vReq); -void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); // Retrieves any serialization flags requested in command line argument int RPCSerializationFlags(); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index b01170074d..923ba2c231 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -4,6 +4,7 @@ #include "scheduler.h" +#include "random.h" #include "reverselock.h" #include <assert.h> @@ -23,7 +24,9 @@ CScheduler::~CScheduler() #if BOOST_VERSION < 105000 static boost::system_time toPosixTime(const boost::chrono::system_clock::time_point& t) { - return boost::posix_time::from_time_t(boost::chrono::system_clock::to_time_t(t)); + // Creating the posix_time using from_time_t loses sub-second precision. So rather than exporting the time_point to time_t, + // start with a posix_time at the epoch (0) and add the milliseconds that have passed since then. + return boost::posix_time::from_time_t(0) + boost::posix_time::milliseconds(boost::chrono::duration_cast<boost::chrono::milliseconds>(t.time_since_epoch()).count()); } #endif @@ -37,6 +40,11 @@ void CScheduler::serviceQueue() // is called. while (!shouldStop()) { try { + if (!shouldStop() && taskQueue.empty()) { + reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); + // Use this chance to get a tiny bit more entropy + RandAddSeedSleep(); + } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. newTaskScheduled.wait(lock); @@ -104,20 +112,20 @@ void CScheduler::schedule(CScheduler::Function f, boost::chrono::system_clock::t newTaskScheduled.notify_one(); } -void CScheduler::scheduleFromNow(CScheduler::Function f, int64_t deltaSeconds) +void CScheduler::scheduleFromNow(CScheduler::Function f, int64_t deltaMilliSeconds) { - schedule(f, boost::chrono::system_clock::now() + boost::chrono::seconds(deltaSeconds)); + schedule(f, boost::chrono::system_clock::now() + boost::chrono::milliseconds(deltaMilliSeconds)); } -static void Repeat(CScheduler* s, CScheduler::Function f, int64_t deltaSeconds) +static void Repeat(CScheduler* s, CScheduler::Function f, int64_t deltaMilliSeconds) { f(); - s->scheduleFromNow(boost::bind(&Repeat, s, f, deltaSeconds), deltaSeconds); + s->scheduleFromNow(boost::bind(&Repeat, s, f, deltaMilliSeconds), deltaMilliSeconds); } -void CScheduler::scheduleEvery(CScheduler::Function f, int64_t deltaSeconds) +void CScheduler::scheduleEvery(CScheduler::Function f, int64_t deltaMilliSeconds) { - scheduleFromNow(boost::bind(&Repeat, this, f, deltaSeconds), deltaSeconds); + scheduleFromNow(boost::bind(&Repeat, this, f, deltaMilliSeconds), deltaMilliSeconds); } size_t CScheduler::getQueueInfo(boost::chrono::system_clock::time_point &first, diff --git a/src/scheduler.h b/src/scheduler.h index 436659e58b..27412a15b4 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -7,10 +7,9 @@ // // NOTE: -// boost::thread / boost::function / boost::chrono should be ported to -// std::thread / std::function / std::chrono when we support C++11. +// boost::thread / boost::chrono should be ported to std::thread / std::chrono +// when we support C++11. // -#include <boost/function.hpp> #include <boost/chrono/chrono.hpp> #include <boost/thread.hpp> #include <map> @@ -23,7 +22,7 @@ // // CScheduler* s = new CScheduler(); // s->scheduleFromNow(doSomething, 11); // Assuming a: void doSomething() { } -// s->scheduleFromNow(boost::bind(Class::func, this, argument), 3); +// s->scheduleFromNow(std::bind(Class::func, this, argument), 3); // boost::thread* t = new boost::thread(boost::bind(CScheduler::serviceQueue, s)); // // ... then at program shutdown, clean up the thread running serviceQueue: @@ -39,20 +38,20 @@ public: CScheduler(); ~CScheduler(); - typedef boost::function<void(void)> Function; + typedef std::function<void(void)> Function; // Call func at/after time t void schedule(Function f, boost::chrono::system_clock::time_point t); // Convenience method: call f once deltaSeconds from now - void scheduleFromNow(Function f, int64_t deltaSeconds); + void scheduleFromNow(Function f, int64_t deltaMilliSeconds); // Another convenience method: call f approximately // every deltaSeconds forever, starting deltaSeconds from now. // To be more precise: every time f is finished, it // is rescheduled to run deltaSeconds later. If you // need more accurate scheduling, don't use this method. - void scheduleEvery(Function f, int64_t deltaSeconds); + void scheduleEvery(Function f, int64_t deltaMilliSeconds); // To keep things as simple as possible, there is no unschedule. diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index f9b7835882..222cff59ea 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -13,9 +13,7 @@ #include "script/script.h" #include "uint256.h" -using namespace std; - -typedef vector<unsigned char> valtype; +typedef std::vector<unsigned char> valtype; namespace { @@ -56,10 +54,10 @@ bool CastToBool(const valtype& vch) */ #define stacktop(i) (stack.at(stack.size()+(i))) #define altstacktop(i) (altstack.at(altstack.size()+(i))) -static inline void popstack(vector<valtype>& stack) +static inline void popstack(std::vector<valtype>& stack) { if (stack.empty()) - throw runtime_error("popstack(): stack empty"); + throw std::runtime_error("popstack(): stack empty"); stack.pop_back(); } @@ -194,7 +192,7 @@ bool static IsDefinedHashtypeSignature(const valtype &vchSig) { return true; } -bool CheckSignatureEncoding(const vector<unsigned char> &vchSig, unsigned int flags, ScriptError* serror) { +bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, unsigned int flags, ScriptError* serror) { // Empty signature. Not strictly DER encoded, but allowed to provide a // compact way to provide an invalid signature for use with CHECK(MULTI)SIG if (vchSig.size() == 0) { @@ -245,14 +243,14 @@ bool static CheckMinimalPush(const valtype& data, opcodetype opcode) { return true; } -bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror) +bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& script, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror) { static const CScriptNum bnZero(0); static const CScriptNum bnOne(1); - static const CScriptNum bnFalse(0); - static const CScriptNum bnTrue(1); + // static const CScriptNum bnFalse(0); + // static const CScriptNum bnTrue(1); static const valtype vchFalse(0); - static const valtype vchZero(0); + // static const valtype vchZero(0); static const valtype vchTrue(1, 1); CScript::const_iterator pc = script.begin(); @@ -260,8 +258,8 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, un CScript::const_iterator pbegincodehash = script.begin(); opcodetype opcode; valtype vchPushValue; - vector<bool> vfExec; - vector<valtype> altstack; + std::vector<bool> vfExec; + std::vector<valtype> altstack; set_error(serror, SCRIPT_ERR_UNKNOWN_ERROR); if (script.size() > MAX_SCRIPT_SIZE) return set_error(serror, SCRIPT_ERR_SCRIPT_SIZE); @@ -1030,7 +1028,7 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, un } // Size limits - if (stack.size() + altstack.size() > 1000) + if (stack.size() + altstack.size() > MAX_STACK_SIZE) return set_error(serror, SCRIPT_ERR_STACK_SIZE); } } @@ -1144,24 +1142,24 @@ public: uint256 GetPrevoutHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vin.size(); n++) { - ss << txTo.vin[n].prevout; + for (const auto& txin : txTo.vin) { + ss << txin.prevout; } return ss.GetHash(); } uint256 GetSequenceHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vin.size(); n++) { - ss << txTo.vin[n].nSequence; + for (const auto& txin : txTo.vin) { + ss << txin.nSequence; } return ss.GetHash(); } uint256 GetOutputsHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vout.size(); n++) { - ss << txTo.vout[n]; + for (const auto& txout : txTo.vout) { + ss << txout; } return ss.GetHash(); } @@ -1250,14 +1248,14 @@ bool TransactionSignatureChecker::VerifySignature(const std::vector<unsigned cha return pubkey.Verify(sighash, vchSig); } -bool TransactionSignatureChecker::CheckSig(const vector<unsigned char>& vchSigIn, const vector<unsigned char>& vchPubKey, const CScript& scriptCode, SigVersion sigversion) const +bool TransactionSignatureChecker::CheckSig(const std::vector<unsigned char>& vchSigIn, const std::vector<unsigned char>& vchPubKey, const CScript& scriptCode, SigVersion sigversion) const { CPubKey pubkey(vchPubKey); if (!pubkey.IsValid()) return false; // Hash type is one byte tacked on to the end of the signature - vector<unsigned char> vchSig(vchSigIn); + std::vector<unsigned char> vchSig(vchSigIn); if (vchSig.empty()) return false; int nHashType = vchSig.back(); @@ -1355,7 +1353,7 @@ bool TransactionSignatureChecker::CheckSequence(const CScriptNum& nSequence) con static bool VerifyWitnessProgram(const CScriptWitness& witness, int witversion, const std::vector<unsigned char>& program, unsigned int flags, const BaseSignatureChecker& checker, ScriptError* serror) { - vector<vector<unsigned char> > stack; + std::vector<std::vector<unsigned char> > stack; CScript scriptPubKey; if (witversion == 0) { @@ -1420,7 +1418,7 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, const C return set_error(serror, SCRIPT_ERR_SIG_PUSHONLY); } - vector<vector<unsigned char> > stack, stackCopy; + std::vector<std::vector<unsigned char> > stack, stackCopy; if (!EvalScript(stack, scriptSig, flags, checker, SIGVERSION_BASE, serror)) // serror is set return false; @@ -1558,7 +1556,7 @@ size_t CountWitnessSigOps(const CScript& scriptSig, const CScript& scriptPubKey, if (scriptPubKey.IsPayToScriptHash() && scriptSig.IsPushOnly()) { CScript::const_iterator pc = scriptSig.begin(); - vector<unsigned char> data; + std::vector<unsigned char> data; while (pc < scriptSig.end()) { opcodetype opcode; scriptSig.GetOp(pc, opcode, data); diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 79894c5300..60f6f711e6 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -171,7 +171,7 @@ private: const CTransaction txTo; public: - MutableTransactionSignatureChecker(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amount) : TransactionSignatureChecker(&txTo, nInIn, amount), txTo(*txToIn) {} + MutableTransactionSignatureChecker(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn) : TransactionSignatureChecker(&txTo, nInIn, amountIn), txTo(*txToIn) {} }; bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& script, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* error = NULL); diff --git a/src/script/ismine.cpp b/src/script/ismine.cpp index 608a8de8f5..a4743281b1 100644 --- a/src/script/ismine.cpp +++ b/src/script/ismine.cpp @@ -13,11 +13,9 @@ #include <boost/foreach.hpp> -using namespace std; +typedef std::vector<unsigned char> valtype; -typedef vector<unsigned char> valtype; - -unsigned int HaveKeys(const vector<valtype>& pubkeys, const CKeyStore& keystore) +unsigned int HaveKeys(const std::vector<valtype>& pubkeys, const CKeyStore& keystore) { unsigned int nResult = 0; BOOST_FOREACH(const valtype& pubkey, pubkeys) @@ -49,7 +47,7 @@ isminetype IsMine(const CKeyStore &keystore, const CTxDestination& dest, bool& i isminetype IsMine(const CKeyStore &keystore, const CScript& scriptPubKey, bool& isInvalid, SigVersion sigversion) { - vector<valtype> vSolutions; + std::vector<valtype> vSolutions; txnouttype whichType; if (!Solver(scriptPubKey, whichType, vSolutions)) { if (keystore.HaveWatchOnly(scriptPubKey)) @@ -132,7 +130,7 @@ isminetype IsMine(const CKeyStore &keystore, const CScript& scriptPubKey, bool& // partially owned (somebody else has a key that can spend // them) enable spend-out-from-under-you attacks, especially // in shared-wallet situations. - vector<valtype> keys(vSolutions.begin()+1, vSolutions.begin()+vSolutions.size()-1); + std::vector<valtype> keys(vSolutions.begin()+1, vSolutions.begin()+vSolutions.size()-1); if (sigversion != SIGVERSION_BASE) { for (size_t i = 0; i < keys.size(); i++) { if (keys[i].size() != 33) { diff --git a/src/script/script.cpp b/src/script/script.cpp index 9f4741b1cd..70eb8a139b 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -8,8 +8,6 @@ #include "tinyformat.h" #include "utilstrencodings.h" -using namespace std; - const char* GetOpName(opcodetype opcode) { switch (opcode) @@ -186,18 +184,18 @@ unsigned int CScript::GetSigOpCount(const CScript& scriptSig) const // get the last item that the scriptSig // pushes onto the stack: const_iterator pc = scriptSig.begin(); - vector<unsigned char> data; + std::vector<unsigned char> vData; while (pc < scriptSig.end()) { opcodetype opcode; - if (!scriptSig.GetOp(pc, opcode, data)) + if (!scriptSig.GetOp(pc, opcode, vData)) return 0; if (opcode > OP_16) return 0; } /// ... and return its opcount: - CScript subscript(data.begin(), data.end()); + CScript subscript(vData.begin(), vData.end()); return subscript.GetSigOpCount(true); } diff --git a/src/script/script.h b/src/script/script.h index 654dff4625..95a5999a13 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -30,6 +30,9 @@ static const int MAX_PUBKEYS_PER_MULTISIG = 20; // Maximum script length in bytes static const int MAX_SCRIPT_SIZE = 10000; +// Maximum number of values on script interpreter stack +static const int MAX_STACK_SIZE = 1000; + // Threshold for nLockTime: below this value it is interpreted as block number, // otherwise as UNIX timestamp. static const unsigned int LOCKTIME_THRESHOLD = 500000000; // Tue Nov 5 00:53:20 1985 UTC @@ -448,16 +451,16 @@ public: else if (b.size() <= 0xffff) { insert(end(), OP_PUSHDATA2); - uint8_t data[2]; - WriteLE16(data, b.size()); - insert(end(), data, data + sizeof(data)); + uint8_t _data[2]; + WriteLE16(_data, b.size()); + insert(end(), _data, _data + sizeof(_data)); } else { insert(end(), OP_PUSHDATA4); - uint8_t data[4]; - WriteLE32(data, b.size()); - insert(end(), data, data + sizeof(data)); + uint8_t _data[4]; + WriteLE32(_data, b.size()); + insert(end(), _data, _data + sizeof(_data)); } insert(end(), b.begin(), b.end()); return *this; diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 6f47b725fb..7bb8d9941b 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -15,28 +15,6 @@ #include <boost/thread.hpp> namespace { - -/** - * We're hashing a nonce into the entries themselves, so we don't need extra - * blinding in the set hash computation. - * - * This may exhibit platform endian dependent behavior but because these are - * nonced hashes (random) and this state is only ever used locally it is safe. - * All that matters is local consistency. - */ -class SignatureCacheHasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin()+4*hash_select, 4); - return u; - } -}; - /** * Valid signature cache, to avoid doing expensive ECDSA signature checking * twice for every transaction (once when accepted into memory pool, and diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 238952bb95..55cec4cc8d 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -19,13 +19,34 @@ static const int64_t MAX_MAX_SIG_CACHE_SIZE = 16384; class CPubKey; +/** + * We're hashing a nonce into the entries themselves, so we don't need extra + * blinding in the set hash computation. + * + * This may exhibit platform endian dependent behavior but because these are + * nonced hashes (random) and this state is only ever used locally it is safe. + * All that matters is local consistency. + */ +class SignatureCacheHasher +{ +public: + template <uint8_t hash_select> + uint32_t operator()(const uint256& key) const + { + static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); + uint32_t u; + std::memcpy(&u, key.begin()+4*hash_select, 4); + return u; + } +}; + class CachingTransactionSignatureChecker : public TransactionSignatureChecker { private: bool store; public: - CachingTransactionSignatureChecker(const CTransaction* txToIn, unsigned int nInIn, const CAmount& amount, bool storeIn, PrecomputedTransactionData& txdataIn) : TransactionSignatureChecker(txToIn, nInIn, amount, txdataIn), store(storeIn) {} + CachingTransactionSignatureChecker(const CTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, bool storeIn, PrecomputedTransactionData& txdataIn) : TransactionSignatureChecker(txToIn, nInIn, amountIn, txdataIn), store(storeIn) {} bool VerifySignature(const std::vector<unsigned char>& vchSig, const CPubKey& vchPubKey, const uint256& sighash) const; }; diff --git a/src/script/sign.cpp b/src/script/sign.cpp index b008df2591..5682418546 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -14,8 +14,6 @@ #include <boost/foreach.hpp> -using namespace std; - typedef std::vector<unsigned char> valtype; TransactionSignatureCreator::TransactionSignatureCreator(const CKeyStore* keystoreIn, const CTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, int nHashTypeIn) : BaseSignatureCreator(keystoreIn), txTo(txToIn), nIn(nInIn), nHashType(nHashTypeIn), amount(amountIn), checker(txTo, nIn, amountIn) {} @@ -39,14 +37,14 @@ bool TransactionSignatureCreator::CreateSig(std::vector<unsigned char>& vchSig, static bool Sign1(const CKeyID& address, const BaseSignatureCreator& creator, const CScript& scriptCode, std::vector<valtype>& ret, SigVersion sigversion) { - vector<unsigned char> vchSig; + std::vector<unsigned char> vchSig; if (!creator.CreateSig(vchSig, address, scriptCode, sigversion)) return false; ret.push_back(vchSig); return true; } -static bool SignN(const vector<valtype>& multisigdata, const BaseSignatureCreator& creator, const CScript& scriptCode, std::vector<valtype>& ret, SigVersion sigversion) +static bool SignN(const std::vector<valtype>& multisigdata, const BaseSignatureCreator& creator, const CScript& scriptCode, std::vector<valtype>& ret, SigVersion sigversion) { int nSigned = 0; int nRequired = multisigdata.front()[0]; @@ -73,7 +71,7 @@ static bool SignStep(const BaseSignatureCreator& creator, const CScript& scriptP uint160 h160; ret.clear(); - vector<valtype> vSolutions; + std::vector<valtype> vSolutions; if (!Solver(scriptPubKey, whichTypeRet, vSolutions)) return false; @@ -125,7 +123,7 @@ static bool SignStep(const BaseSignatureCreator& creator, const CScript& scriptP } } -static CScript PushAll(const vector<valtype>& values) +static CScript PushAll(const std::vector<valtype>& values) { CScript result; BOOST_FOREACH(const valtype& v, values) { @@ -228,12 +226,12 @@ bool SignSignature(const CKeyStore &keystore, const CTransaction& txFrom, CMutab return SignSignature(keystore, txout.scriptPubKey, txTo, nIn, txout.nValue, nHashType); } -static vector<valtype> CombineMultisig(const CScript& scriptPubKey, const BaseSignatureChecker& checker, - const vector<valtype>& vSolutions, - const vector<valtype>& sigs1, const vector<valtype>& sigs2, SigVersion sigversion) +static std::vector<valtype> CombineMultisig(const CScript& scriptPubKey, const BaseSignatureChecker& checker, + const std::vector<valtype>& vSolutions, + const std::vector<valtype>& sigs1, const std::vector<valtype>& sigs2, SigVersion sigversion) { // Combine all the signatures we've got: - set<valtype> allsigs; + std::set<valtype> allsigs; BOOST_FOREACH(const valtype& v, sigs1) { if (!v.empty()) @@ -249,7 +247,7 @@ static vector<valtype> CombineMultisig(const CScript& scriptPubKey, const BaseSi assert(vSolutions.size() > 1); unsigned int nSigsRequired = vSolutions.front()[0]; unsigned int nPubKeys = vSolutions.size()-2; - map<valtype, valtype> sigs; + std::map<valtype, valtype> sigs; BOOST_FOREACH(const valtype& sig, allsigs) { for (unsigned int i = 0; i < nPubKeys; i++) @@ -306,7 +304,7 @@ struct Stacks } static Stacks CombineSignatures(const CScript& scriptPubKey, const BaseSignatureChecker& checker, - const txnouttype txType, const vector<valtype>& vSolutions, + const txnouttype txType, const std::vector<valtype>& vSolutions, Stacks sigs1, Stacks sigs2, SigVersion sigversion) { switch (txType) @@ -340,7 +338,7 @@ static Stacks CombineSignatures(const CScript& scriptPubKey, const BaseSignature CScript pubKey2(spk.begin(), spk.end()); txnouttype txType2; - vector<vector<unsigned char> > vSolutions2; + std::vector<std::vector<unsigned char> > vSolutions2; Solver(pubKey2, txType2, vSolutions2); sigs1.script.pop_back(); sigs2.script.pop_back(); @@ -360,7 +358,7 @@ static Stacks CombineSignatures(const CScript& scriptPubKey, const BaseSignature // Recur to combine: CScript pubKey2(sigs1.witness.back().begin(), sigs1.witness.back().end()); txnouttype txType2; - vector<valtype> vSolutions2; + std::vector<valtype> vSolutions2; Solver(pubKey2, txType2, vSolutions2); sigs1.witness.pop_back(); sigs1.script = sigs1.witness; @@ -383,7 +381,7 @@ SignatureData CombineSignatures(const CScript& scriptPubKey, const BaseSignature const SignatureData& scriptSig1, const SignatureData& scriptSig2) { txnouttype txType; - vector<vector<unsigned char> > vSolutions; + std::vector<std::vector<unsigned char> > vSolutions; Solver(scriptPubKey, txType, vSolutions); return CombineSignatures(scriptPubKey, checker, txType, vSolutions, Stacks(scriptSig1), Stacks(scriptSig2), SIGVERSION_BASE).Output(); diff --git a/src/script/sign.h b/src/script/sign.h index 1cfc53c6c1..f3c0be4139 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -48,7 +48,7 @@ class MutableTransactionSignatureCreator : public TransactionSignatureCreator { CTransaction tx; public: - MutableTransactionSignatureCreator(const CKeyStore* keystoreIn, const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amount, int nHashTypeIn) : TransactionSignatureCreator(keystoreIn, &tx, nInIn, amount, nHashTypeIn), tx(*txToIn) {} + MutableTransactionSignatureCreator(const CKeyStore* keystoreIn, const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, int nHashTypeIn) : TransactionSignatureCreator(keystoreIn, &tx, nInIn, amountIn, nHashTypeIn), tx(*txToIn) {} }; /** A signature creator that just produces 72-byte empty signatures. */ diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 4b9bec9aa1..63f20b0993 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -12,9 +12,7 @@ #include <boost/foreach.hpp> -using namespace std; - -typedef vector<unsigned char> valtype; +typedef std::vector<unsigned char> valtype; bool fAcceptDatacarrier = DEFAULT_ACCEPT_DATACARRIER; unsigned nMaxDatacarrierBytes = MAX_OP_RETURN_RELAY; @@ -40,20 +38,20 @@ const char* GetTxnOutputType(txnouttype t) /** * Return public keys or hashes from scriptPubKey, for 'standard' transaction types. */ -bool Solver(const CScript& scriptPubKey, txnouttype& typeRet, vector<vector<unsigned char> >& vSolutionsRet) +bool Solver(const CScript& scriptPubKey, txnouttype& typeRet, std::vector<std::vector<unsigned char> >& vSolutionsRet) { // Templates - static multimap<txnouttype, CScript> mTemplates; + static std::multimap<txnouttype, CScript> mTemplates; if (mTemplates.empty()) { // Standard tx, sender provides pubkey, receiver adds signature - mTemplates.insert(make_pair(TX_PUBKEY, CScript() << OP_PUBKEY << OP_CHECKSIG)); + mTemplates.insert(std::make_pair(TX_PUBKEY, CScript() << OP_PUBKEY << OP_CHECKSIG)); // Bitcoin address tx, sender provides hash of pubkey, receiver provides signature and pubkey - mTemplates.insert(make_pair(TX_PUBKEYHASH, CScript() << OP_DUP << OP_HASH160 << OP_PUBKEYHASH << OP_EQUALVERIFY << OP_CHECKSIG)); + mTemplates.insert(std::make_pair(TX_PUBKEYHASH, CScript() << OP_DUP << OP_HASH160 << OP_PUBKEYHASH << OP_EQUALVERIFY << OP_CHECKSIG)); // Sender provides N pubkeys, receivers provides M signatures - mTemplates.insert(make_pair(TX_MULTISIG, CScript() << OP_SMALLINTEGER << OP_PUBKEYS << OP_SMALLINTEGER << OP_CHECKMULTISIG)); + mTemplates.insert(std::make_pair(TX_MULTISIG, CScript() << OP_SMALLINTEGER << OP_PUBKEYS << OP_SMALLINTEGER << OP_CHECKMULTISIG)); } vSolutionsRet.clear(); @@ -63,7 +61,7 @@ bool Solver(const CScript& scriptPubKey, txnouttype& typeRet, vector<vector<unsi if (scriptPubKey.IsPayToScriptHash()) { typeRet = TX_SCRIPTHASH; - vector<unsigned char> hashBytes(scriptPubKey.begin()+2, scriptPubKey.begin()+22); + std::vector<unsigned char> hashBytes(scriptPubKey.begin()+2, scriptPubKey.begin()+22); vSolutionsRet.push_back(hashBytes); return true; } @@ -102,7 +100,7 @@ bool Solver(const CScript& scriptPubKey, txnouttype& typeRet, vector<vector<unsi vSolutionsRet.clear(); opcodetype opcode1, opcode2; - vector<unsigned char> vch1, vch2; + std::vector<unsigned char> vch1, vch2; // Compare CScript::const_iterator pc1 = script1.begin(); @@ -181,7 +179,7 @@ bool Solver(const CScript& scriptPubKey, txnouttype& typeRet, vector<vector<unsi bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) { - vector<valtype> vSolutions; + std::vector<valtype> vSolutions; txnouttype whichType; if (!Solver(scriptPubKey, whichType, vSolutions)) return false; @@ -209,11 +207,11 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) return false; } -bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, vector<CTxDestination>& addressRet, int& nRequiredRet) +bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) { addressRet.clear(); typeRet = TX_NONSTANDARD; - vector<valtype> vSolutions; + std::vector<valtype> vSolutions; if (!Solver(scriptPubKey, typeRet, vSolutions)) return false; if (typeRet == TX_NULL_DATA){ diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index e5657f7f31..c071fbe275 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -93,7 +93,10 @@ TESTS = if USE_TESTS noinst_PROGRAMS += tests tests_SOURCES = src/tests.c -tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +if !ENABLE_COVERAGE +tests_CPPFLAGS += -DVERIFY +endif tests_LDADD = $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) tests_LDFLAGS = -static TESTS += tests @@ -102,7 +105,10 @@ endif if USE_EXHAUSTIVE_TESTS noinst_PROGRAMS += exhaustive_tests exhaustive_tests_SOURCES = src/tests_exhaustive.c -exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src $(SECP_INCLUDES) +exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src $(SECP_INCLUDES) +if !ENABLE_COVERAGE +exhaustive_tests_CPPFLAGS += -DVERIFY +endif exhaustive_tests_LDADD = $(SECP_LIBS) exhaustive_tests_LDFLAGS = -static TESTS += exhaustive_tests diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index ec50ffe3a2..e5fcbcb4ed 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -20,7 +20,7 @@ AC_PATH_TOOL(STRIP, strip) AX_PROG_CC_FOR_BUILD if test "x$CFLAGS" = "x"; then - CFLAGS="-O3 -g" + CFLAGS="-g" fi AM_PROG_CC_C_O @@ -89,6 +89,11 @@ AC_ARG_ENABLE(benchmark, [use_benchmark=$enableval], [use_benchmark=no]) +AC_ARG_ENABLE(coverage, + AS_HELP_STRING([--enable-coverage],[enable compiler flags to support kcov coverage analysis]), + [enable_coverage=$enableval], + [enable_coverage=no]) + AC_ARG_ENABLE(tests, AS_HELP_STRING([--enable-tests],[compile tests (default is yes)]), [use_tests=$enableval], @@ -154,6 +159,14 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[void myfunc() {__builtin_expect(0,0);}]])], [ AC_MSG_RESULT([no]) ]) +if test x"$enable_coverage" = x"yes"; then + AC_DEFINE(COVERAGE, 1, [Define this symbol to compile out all VERIFY code]) + CFLAGS="$CFLAGS -O0 --coverage" + LDFLAGS="--coverage" +else + CFLAGS="$CFLAGS -O3" +fi + if test x"$use_ecmult_static_precomputation" != x"no"; then save_cross_compiling=$cross_compiling cross_compiling=no @@ -434,6 +447,7 @@ AC_MSG_NOTICE([Using field implementation: $set_field]) AC_MSG_NOTICE([Using bignum implementation: $set_bignum]) AC_MSG_NOTICE([Using scalar implementation: $set_scalar]) AC_MSG_NOTICE([Using endomorphism optimizations: $use_endomorphism]) +AC_MSG_NOTICE([Building for coverage analysis: $enable_coverage]) AC_MSG_NOTICE([Building ECDH module: $enable_module_ecdh]) AC_MSG_NOTICE([Building ECDSA pubkey recovery module: $enable_module_recovery]) AC_MSG_NOTICE([Using jni: $use_jni]) @@ -460,6 +474,7 @@ AC_SUBST(SECP_INCLUDES) AC_SUBST(SECP_LIBS) AC_SUBST(SECP_TEST_LIBS) AC_SUBST(SECP_TEST_INCLUDES) +AM_CONDITIONAL([ENABLE_COVERAGE], [test x"$enable_coverage" = x"yes"]) AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) AM_CONDITIONAL([USE_EXHAUSTIVE_TESTS], [test x"$use_exhaustive_tests" != x"no"]) AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index f268e309d0..fc4c5cefbb 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -163,6 +163,8 @@ typedef int (*secp256k1_nonce_function)( * * Returns: a newly created context object. * In: flags: which parts of the context to initialize. + * + * See also secp256k1_context_randomize. */ SECP256K1_API secp256k1_context* secp256k1_context_create( unsigned int flags @@ -485,6 +487,28 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_create( const unsigned char *seckey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +/** Negates a private key in place. + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * In/Out: pubkey: pointer to the public key to be negated (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_negate( + const secp256k1_context* ctx, + unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + +/** Negates a public key in place. + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * In/Out: pubkey: pointer to the public key to be negated (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_negate( + const secp256k1_context* ctx, + secp256k1_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + /** Tweak a private key by adding tweak to it. * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for * uniformly random 32-byte arrays, or if the resulting private key @@ -543,11 +567,24 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( const unsigned char *tweak ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Updates the context randomization. +/** Updates the context randomization to protect against side-channel leakage. * Returns: 1: randomization successfully updated * 0: error * Args: ctx: pointer to a context object (cannot be NULL) * In: seed32: pointer to a 32-byte random seed (NULL resets to initial state) + * + * While secp256k1 code is written to be constant-time no matter what secret + * values are, it's possible that a future compiler may output code which isn't, + * and also that the CPU may not emit the same radio frequencies or draw the same + * amount power for all values. + * + * This function provides a seed which is combined into the blinding value: that + * blinding value is added before each multiplication (and removed afterwards) so + * that it does not affect function results, but shields against attacks which + * rely on any input-dependent behaviour. + * + * You should call this after secp256k1_context_create or + * secp256k1_context_clone, and may call this repeatedly afterwards. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( secp256k1_context* ctx, diff --git a/src/secp256k1/src/bench.h b/src/secp256k1/src/bench.h index 3a71b4aafa..d67f08a426 100644 --- a/src/secp256k1/src/bench.h +++ b/src/secp256k1/src/bench.h @@ -23,7 +23,7 @@ void print_number(double x) { if (y < 0.0) { y = -y; } - while (y < 100.0) { + while (y > 0 && y < 100.0) { y *= 10.0; c++; } diff --git a/src/secp256k1/src/bench_schnorr_verify.c b/src/secp256k1/src/bench_schnorr_verify.c deleted file mode 100644 index 5f137dda23..0000000000 --- a/src/secp256k1/src/bench_schnorr_verify.c +++ /dev/null @@ -1,73 +0,0 @@ -/********************************************************************** - * Copyright (c) 2014 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php.* - **********************************************************************/ - -#include <stdio.h> -#include <string.h> - -#include "include/secp256k1.h" -#include "include/secp256k1_schnorr.h" -#include "util.h" -#include "bench.h" - -typedef struct { - unsigned char key[32]; - unsigned char sig[64]; - unsigned char pubkey[33]; - size_t pubkeylen; -} benchmark_schnorr_sig_t; - -typedef struct { - secp256k1_context *ctx; - unsigned char msg[32]; - benchmark_schnorr_sig_t sigs[64]; - int numsigs; -} benchmark_schnorr_verify_t; - -static void benchmark_schnorr_init(void* arg) { - int i, k; - benchmark_schnorr_verify_t* data = (benchmark_schnorr_verify_t*)arg; - - for (i = 0; i < 32; i++) { - data->msg[i] = 1 + i; - } - for (k = 0; k < data->numsigs; k++) { - secp256k1_pubkey pubkey; - for (i = 0; i < 32; i++) { - data->sigs[k].key[i] = 33 + i + k; - } - secp256k1_schnorr_sign(data->ctx, data->sigs[k].sig, data->msg, data->sigs[k].key, NULL, NULL); - data->sigs[k].pubkeylen = 33; - CHECK(secp256k1_ec_pubkey_create(data->ctx, &pubkey, data->sigs[k].key)); - CHECK(secp256k1_ec_pubkey_serialize(data->ctx, data->sigs[k].pubkey, &data->sigs[k].pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED)); - } -} - -static void benchmark_schnorr_verify(void* arg) { - int i; - benchmark_schnorr_verify_t* data = (benchmark_schnorr_verify_t*)arg; - - for (i = 0; i < 20000 / data->numsigs; i++) { - secp256k1_pubkey pubkey; - data->sigs[0].sig[(i >> 8) % 64] ^= (i & 0xFF); - CHECK(secp256k1_ec_pubkey_parse(data->ctx, &pubkey, data->sigs[0].pubkey, data->sigs[0].pubkeylen)); - CHECK(secp256k1_schnorr_verify(data->ctx, data->sigs[0].sig, data->msg, &pubkey) == ((i & 0xFF) == 0)); - data->sigs[0].sig[(i >> 8) % 64] ^= (i & 0xFF); - } -} - - - -int main(void) { - benchmark_schnorr_verify_t data; - - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - - data.numsigs = 1; - run_benchmark("schnorr_verify", benchmark_schnorr_verify, benchmark_schnorr_init, NULL, &data, 10, 20000); - - secp256k1_context_destroy(data.ctx); - return 0; -} diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index 9a42e519bd..453bb11880 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -225,14 +225,12 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const #if defined(EXHAUSTIVE_TEST_ORDER) { secp256k1_scalar computed_r; - int overflow = 0; secp256k1_ge pr_ge; secp256k1_ge_set_gej(&pr_ge, &pr); secp256k1_fe_normalize(&pr_ge.x); secp256k1_fe_get_b32(c, &pr_ge.x); - secp256k1_scalar_set_b32(&computed_r, c, &overflow); - /* we fully expect overflow */ + secp256k1_scalar_set_b32(&computed_r, c, NULL); return secp256k1_scalar_eq(sigr, &computed_r); } #else @@ -285,14 +283,10 @@ static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context *ctx, sec secp256k1_fe_normalize(&r.y); secp256k1_fe_get_b32(b, &r.x); secp256k1_scalar_set_b32(sigr, b, &overflow); - if (secp256k1_scalar_is_zero(sigr)) { - /* P.x = order is on the curve, so technically sig->r could end up zero, which would be an invalid signature. - * This branch is cryptographically unreachable as hitting it requires finding the discrete log of P.x = N. - */ - secp256k1_gej_clear(&rp); - secp256k1_ge_clear(&r); - return 0; - } + /* These two conditions should be checked before calling */ + VERIFY_CHECK(!secp256k1_scalar_is_zero(sigr)); + VERIFY_CHECK(overflow == 0); + if (recid) { /* The overflow condition is cryptographically unreachable as hitting it requires finding the discrete log * of some P where P.x >= order, and only 1 in about 2^127 points meet this criteria. diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 7b8c079608..234c13a644 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -38,10 +38,6 @@ static void secp256k1_fe_verify(const secp256k1_fe *a) { } VERIFY_CHECK(r == 1); } -#else -static void secp256k1_fe_verify(const secp256k1_fe *a) { - (void)a; -} #endif static void secp256k1_fe_normalize(secp256k1_fe *r) { @@ -325,17 +321,17 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { } static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { - int i; - r->n[0] = r->n[1] = r->n[2] = r->n[3] = r->n[4] = 0; - r->n[5] = r->n[6] = r->n[7] = r->n[8] = r->n[9] = 0; - for (i=0; i<32; i++) { - int j; - for (j=0; j<4; j++) { - int limb = (8*i+2*j)/26; - int shift = (8*i+2*j)%26; - r->n[limb] |= (uint32_t)((a[31-i] >> (2*j)) & 0x3) << shift; - } - } + r->n[0] = (uint32_t)a[31] | ((uint32_t)a[30] << 8) | ((uint32_t)a[29] << 16) | ((uint32_t)(a[28] & 0x3) << 24); + r->n[1] = (uint32_t)((a[28] >> 2) & 0x3f) | ((uint32_t)a[27] << 6) | ((uint32_t)a[26] << 14) | ((uint32_t)(a[25] & 0xf) << 22); + r->n[2] = (uint32_t)((a[25] >> 4) & 0xf) | ((uint32_t)a[24] << 4) | ((uint32_t)a[23] << 12) | ((uint32_t)(a[22] & 0x3f) << 20); + r->n[3] = (uint32_t)((a[22] >> 6) & 0x3) | ((uint32_t)a[21] << 2) | ((uint32_t)a[20] << 10) | ((uint32_t)a[19] << 18); + r->n[4] = (uint32_t)a[18] | ((uint32_t)a[17] << 8) | ((uint32_t)a[16] << 16) | ((uint32_t)(a[15] & 0x3) << 24); + r->n[5] = (uint32_t)((a[15] >> 2) & 0x3f) | ((uint32_t)a[14] << 6) | ((uint32_t)a[13] << 14) | ((uint32_t)(a[12] & 0xf) << 22); + r->n[6] = (uint32_t)((a[12] >> 4) & 0xf) | ((uint32_t)a[11] << 4) | ((uint32_t)a[10] << 12) | ((uint32_t)(a[9] & 0x3f) << 20); + r->n[7] = (uint32_t)((a[9] >> 6) & 0x3) | ((uint32_t)a[8] << 2) | ((uint32_t)a[7] << 10) | ((uint32_t)a[6] << 18); + r->n[8] = (uint32_t)a[5] | ((uint32_t)a[4] << 8) | ((uint32_t)a[3] << 16) | ((uint32_t)(a[2] & 0x3) << 24); + r->n[9] = (uint32_t)((a[2] >> 2) & 0x3f) | ((uint32_t)a[1] << 6) | ((uint32_t)a[0] << 14); + if (r->n[9] == 0x3FFFFFUL && (r->n[8] & r->n[7] & r->n[6] & r->n[5] & r->n[4] & r->n[3] & r->n[2]) == 0x3FFFFFFUL && (r->n[1] + 0x40UL + ((r->n[0] + 0x3D1UL) >> 26)) > 0x3FFFFFFUL) { return 0; } @@ -349,21 +345,42 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { - int i; #ifdef VERIFY VERIFY_CHECK(a->normalized); secp256k1_fe_verify(a); #endif - for (i=0; i<32; i++) { - int j; - int c = 0; - for (j=0; j<4; j++) { - int limb = (8*i+2*j)/26; - int shift = (8*i+2*j)%26; - c |= ((a->n[limb] >> shift) & 0x3) << (2 * j); - } - r[31-i] = c; - } + r[0] = (a->n[9] >> 14) & 0xff; + r[1] = (a->n[9] >> 6) & 0xff; + r[2] = ((a->n[9] & 0x3F) << 2) | ((a->n[8] >> 24) & 0x3); + r[3] = (a->n[8] >> 16) & 0xff; + r[4] = (a->n[8] >> 8) & 0xff; + r[5] = a->n[8] & 0xff; + r[6] = (a->n[7] >> 18) & 0xff; + r[7] = (a->n[7] >> 10) & 0xff; + r[8] = (a->n[7] >> 2) & 0xff; + r[9] = ((a->n[7] & 0x3) << 6) | ((a->n[6] >> 20) & 0x3f); + r[10] = (a->n[6] >> 12) & 0xff; + r[11] = (a->n[6] >> 4) & 0xff; + r[12] = ((a->n[6] & 0xf) << 4) | ((a->n[5] >> 22) & 0xf); + r[13] = (a->n[5] >> 14) & 0xff; + r[14] = (a->n[5] >> 6) & 0xff; + r[15] = ((a->n[5] & 0x3f) << 2) | ((a->n[4] >> 24) & 0x3); + r[16] = (a->n[4] >> 16) & 0xff; + r[17] = (a->n[4] >> 8) & 0xff; + r[18] = a->n[4] & 0xff; + r[19] = (a->n[3] >> 18) & 0xff; + r[20] = (a->n[3] >> 10) & 0xff; + r[21] = (a->n[3] >> 2) & 0xff; + r[22] = ((a->n[3] & 0x3) << 6) | ((a->n[2] >> 20) & 0x3f); + r[23] = (a->n[2] >> 12) & 0xff; + r[24] = (a->n[2] >> 4) & 0xff; + r[25] = ((a->n[2] & 0xf) << 4) | ((a->n[1] >> 22) & 0xf); + r[26] = (a->n[1] >> 14) & 0xff; + r[27] = (a->n[1] >> 6) & 0xff; + r[28] = ((a->n[1] & 0x3f) << 2) | ((a->n[0] >> 24) & 0x3); + r[29] = (a->n[0] >> 16) & 0xff; + r[30] = (a->n[0] >> 8) & 0xff; + r[31] = a->n[0] & 0xff; } SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index 7a99eb21ec..8e8b286baf 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -49,10 +49,6 @@ static void secp256k1_fe_verify(const secp256k1_fe *a) { } VERIFY_CHECK(r == 1); } -#else -static void secp256k1_fe_verify(const secp256k1_fe *a) { - (void)a; -} #endif static void secp256k1_fe_normalize(secp256k1_fe *r) { @@ -288,16 +284,40 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { } static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { - int i; - r->n[0] = r->n[1] = r->n[2] = r->n[3] = r->n[4] = 0; - for (i=0; i<32; i++) { - int j; - for (j=0; j<2; j++) { - int limb = (8*i+4*j)/52; - int shift = (8*i+4*j)%52; - r->n[limb] |= (uint64_t)((a[31-i] >> (4*j)) & 0xF) << shift; - } - } + r->n[0] = (uint64_t)a[31] + | ((uint64_t)a[30] << 8) + | ((uint64_t)a[29] << 16) + | ((uint64_t)a[28] << 24) + | ((uint64_t)a[27] << 32) + | ((uint64_t)a[26] << 40) + | ((uint64_t)(a[25] & 0xF) << 48); + r->n[1] = (uint64_t)((a[25] >> 4) & 0xF) + | ((uint64_t)a[24] << 4) + | ((uint64_t)a[23] << 12) + | ((uint64_t)a[22] << 20) + | ((uint64_t)a[21] << 28) + | ((uint64_t)a[20] << 36) + | ((uint64_t)a[19] << 44); + r->n[2] = (uint64_t)a[18] + | ((uint64_t)a[17] << 8) + | ((uint64_t)a[16] << 16) + | ((uint64_t)a[15] << 24) + | ((uint64_t)a[14] << 32) + | ((uint64_t)a[13] << 40) + | ((uint64_t)(a[12] & 0xF) << 48); + r->n[3] = (uint64_t)((a[12] >> 4) & 0xF) + | ((uint64_t)a[11] << 4) + | ((uint64_t)a[10] << 12) + | ((uint64_t)a[9] << 20) + | ((uint64_t)a[8] << 28) + | ((uint64_t)a[7] << 36) + | ((uint64_t)a[6] << 44); + r->n[4] = (uint64_t)a[5] + | ((uint64_t)a[4] << 8) + | ((uint64_t)a[3] << 16) + | ((uint64_t)a[2] << 24) + | ((uint64_t)a[1] << 32) + | ((uint64_t)a[0] << 40); if (r->n[4] == 0x0FFFFFFFFFFFFULL && (r->n[3] & r->n[2] & r->n[1]) == 0xFFFFFFFFFFFFFULL && r->n[0] >= 0xFFFFEFFFFFC2FULL) { return 0; } @@ -311,21 +331,42 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { - int i; #ifdef VERIFY VERIFY_CHECK(a->normalized); secp256k1_fe_verify(a); #endif - for (i=0; i<32; i++) { - int j; - int c = 0; - for (j=0; j<2; j++) { - int limb = (8*i+4*j)/52; - int shift = (8*i+4*j)%52; - c |= ((a->n[limb] >> shift) & 0xF) << (4 * j); - } - r[31-i] = c; - } + r[0] = (a->n[4] >> 40) & 0xFF; + r[1] = (a->n[4] >> 32) & 0xFF; + r[2] = (a->n[4] >> 24) & 0xFF; + r[3] = (a->n[4] >> 16) & 0xFF; + r[4] = (a->n[4] >> 8) & 0xFF; + r[5] = a->n[4] & 0xFF; + r[6] = (a->n[3] >> 44) & 0xFF; + r[7] = (a->n[3] >> 36) & 0xFF; + r[8] = (a->n[3] >> 28) & 0xFF; + r[9] = (a->n[3] >> 20) & 0xFF; + r[10] = (a->n[3] >> 12) & 0xFF; + r[11] = (a->n[3] >> 4) & 0xFF; + r[12] = ((a->n[2] >> 48) & 0xF) | ((a->n[3] & 0xF) << 4); + r[13] = (a->n[2] >> 40) & 0xFF; + r[14] = (a->n[2] >> 32) & 0xFF; + r[15] = (a->n[2] >> 24) & 0xFF; + r[16] = (a->n[2] >> 16) & 0xFF; + r[17] = (a->n[2] >> 8) & 0xFF; + r[18] = a->n[2] & 0xFF; + r[19] = (a->n[1] >> 44) & 0xFF; + r[20] = (a->n[1] >> 36) & 0xFF; + r[21] = (a->n[1] >> 28) & 0xFF; + r[22] = (a->n[1] >> 20) & 0xFF; + r[23] = (a->n[1] >> 12) & 0xFF; + r[24] = (a->n[1] >> 4) & 0xFF; + r[25] = ((a->n[0] >> 48) & 0xF) | ((a->n[1] & 0xF) << 4); + r[26] = (a->n[0] >> 40) & 0xFF; + r[27] = (a->n[0] >> 32) & 0xFF; + r[28] = (a->n[0] >> 24) & 0xFF; + r[29] = (a->n[0] >> 16) & 0xFF; + r[30] = (a->n[0] >> 8) & 0xFF; + r[31] = a->n[0] & 0xFF; } SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 2e192b62fd..7d723532ff 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -200,12 +200,6 @@ static void secp256k1_gej_set_infinity(secp256k1_gej *r) { secp256k1_fe_clear(&r->z); } -static void secp256k1_ge_set_infinity(secp256k1_ge *r) { - r->infinity = 1; - secp256k1_fe_clear(&r->x); - secp256k1_fe_clear(&r->y); -} - static void secp256k1_gej_clear(secp256k1_gej *r) { r->infinity = 0; secp256k1_fe_clear(&r->x); diff --git a/src/secp256k1/src/modules/ecdh/main_impl.h b/src/secp256k1/src/modules/ecdh/main_impl.h index c23e4f82f7..9e30fb73dd 100644 --- a/src/secp256k1/src/modules/ecdh/main_impl.h +++ b/src/secp256k1/src/modules/ecdh/main_impl.h @@ -16,10 +16,10 @@ int secp256k1_ecdh(const secp256k1_context* ctx, unsigned char *result, const se secp256k1_gej res; secp256k1_ge pt; secp256k1_scalar s; + VERIFY_CHECK(ctx != NULL); ARG_CHECK(result != NULL); ARG_CHECK(point != NULL); ARG_CHECK(scalar != NULL); - (void)ctx; secp256k1_pubkey_load(ctx, &pt, point); secp256k1_scalar_set_b32(&s, scalar, &overflow); diff --git a/src/secp256k1/src/modules/ecdh/tests_impl.h b/src/secp256k1/src/modules/ecdh/tests_impl.h index 7badc9033f..85a5d0a9a6 100644 --- a/src/secp256k1/src/modules/ecdh/tests_impl.h +++ b/src/secp256k1/src/modules/ecdh/tests_impl.h @@ -7,6 +7,35 @@ #ifndef _SECP256K1_MODULE_ECDH_TESTS_ #define _SECP256K1_MODULE_ECDH_TESTS_ +void test_ecdh_api(void) { + /* Setup context that just counts errors */ + secp256k1_context *tctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_pubkey point; + unsigned char res[32]; + unsigned char s_one[32] = { 0 }; + int32_t ecount = 0; + s_one[31] = 1; + + secp256k1_context_set_error_callback(tctx, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(tctx, counting_illegal_callback_fn, &ecount); + CHECK(secp256k1_ec_pubkey_create(tctx, &point, s_one) == 1); + + /* Check all NULLs are detected */ + CHECK(secp256k1_ecdh(tctx, res, &point, s_one) == 1); + CHECK(ecount == 0); + CHECK(secp256k1_ecdh(tctx, NULL, &point, s_one) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdh(tctx, res, NULL, s_one) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdh(tctx, res, &point, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdh(tctx, res, &point, s_one) == 1); + CHECK(ecount == 3); + + /* Cleanup */ + secp256k1_context_destroy(tctx); +} + void test_ecdh_generator_basepoint(void) { unsigned char s_one[32] = { 0 }; secp256k1_pubkey point[2]; @@ -68,6 +97,7 @@ void test_bad_scalar(void) { } void run_ecdh_tests(void) { + test_ecdh_api(); test_ecdh_generator_basepoint(); test_bad_scalar(); } diff --git a/src/secp256k1/src/modules/recovery/main_impl.h b/src/secp256k1/src/modules/recovery/main_impl.h index 86f2f0cb2b..c6fbe23981 100755 --- a/src/secp256k1/src/modules/recovery/main_impl.h +++ b/src/secp256k1/src/modules/recovery/main_impl.h @@ -179,7 +179,7 @@ int secp256k1_ecdsa_recover(const secp256k1_context* ctx, secp256k1_pubkey *pubk ARG_CHECK(pubkey != NULL); secp256k1_ecdsa_recoverable_signature_load(ctx, &r, &s, &recid, signature); - ARG_CHECK(recid >= 0 && recid < 4); + VERIFY_CHECK(recid >= 0 && recid < 4); /* should have been caught in parse_compact */ secp256k1_scalar_set_b32(&m, msg32, NULL); if (secp256k1_ecdsa_sig_recover(&ctx->ecmult_ctx, &r, &s, &q, &m, recid)) { secp256k1_pubkey_save(pubkey, &q); diff --git a/src/secp256k1/src/modules/recovery/tests_impl.h b/src/secp256k1/src/modules/recovery/tests_impl.h index 8932d5f0af..765c7dd81e 100644 --- a/src/secp256k1/src/modules/recovery/tests_impl.h +++ b/src/secp256k1/src/modules/recovery/tests_impl.h @@ -7,6 +7,146 @@ #ifndef _SECP256K1_MODULE_RECOVERY_TESTS_ #define _SECP256K1_MODULE_RECOVERY_TESTS_ +static int recovery_test_nonce_function(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *algo16, void *data, unsigned int counter) { + (void) msg32; + (void) key32; + (void) algo16; + (void) data; + + /* On the first run, return 0 to force a second run */ + if (counter == 0) { + memset(nonce32, 0, 32); + return 1; + } + /* On the second run, return an overflow to force a third run */ + if (counter == 1) { + memset(nonce32, 0xff, 32); + return 1; + } + /* On the next run, return a valid nonce, but flip a coin as to whether or not to fail signing. */ + memset(nonce32, 1, 32); + return secp256k1_rand_bits(1); +} + +void test_ecdsa_recovery_api(void) { + /* Setup contexts that just count errors */ + secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + secp256k1_context *sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_context *vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + secp256k1_context *both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + secp256k1_pubkey pubkey; + secp256k1_pubkey recpubkey; + secp256k1_ecdsa_signature normal_sig; + secp256k1_ecdsa_recoverable_signature recsig; + unsigned char privkey[32] = { 1 }; + unsigned char message[32] = { 2 }; + int32_t ecount = 0; + int recid = 0; + unsigned char sig[74]; + unsigned char zero_privkey[32] = { 0 }; + unsigned char over_privkey[32] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + + secp256k1_context_set_error_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(sign, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(vrfy, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(both, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(sign, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(both, counting_illegal_callback_fn, &ecount); + + /* Construct and verify corresponding public key. */ + CHECK(secp256k1_ec_seckey_verify(ctx, privkey) == 1); + CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey, privkey) == 1); + + /* Check bad contexts and NULLs for signing */ + ecount = 0; + CHECK(secp256k1_ecdsa_sign_recoverable(none, &recsig, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(sign, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(vrfy, &recsig, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_sign_recoverable(both, NULL, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, NULL, privkey, NULL, NULL) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, NULL, NULL, NULL) == 0); + CHECK(ecount == 5); + /* This will fail or succeed randomly, and in either case will not ARG_CHECK failure */ + secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, recovery_test_nonce_function, NULL); + CHECK(ecount == 5); + /* These will all fail, but not in ARG_CHECK way */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, zero_privkey, NULL, NULL) == 0); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, over_privkey, NULL, NULL) == 0); + /* This one will succeed. */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 5); + + /* Check signing with a goofy nonce function */ + + /* Check bad contexts and NULLs for recovery */ + ecount = 0; + CHECK(secp256k1_ecdsa_recover(none, &recpubkey, &recsig, message) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recover(sign, &recpubkey, &recsig, message) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(vrfy, &recpubkey, &recsig, message) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, &recsig, message) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(both, NULL, &recsig, message) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, NULL, message) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, &recsig, NULL) == 0); + CHECK(ecount == 5); + + /* Check NULLs for conversion */ + CHECK(secp256k1_ecdsa_sign(both, &normal_sig, message, privkey, NULL, NULL) == 1); + ecount = 0; + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, NULL, &recsig) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, &normal_sig, NULL) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, &normal_sig, &recsig) == 1); + + /* Check NULLs for de/serialization */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + ecount = 0; + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, NULL, &recid, &recsig) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, NULL, &recsig) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, &recid, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, &recid, &recsig) == 1); + + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, NULL, sig, recid) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, NULL, recid) == 0); + CHECK(ecount == 5); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, -1) == 0); + CHECK(ecount == 6); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, 5) == 0); + CHECK(ecount == 7); + /* overflow in signature will fail but not affect ecount */ + memcpy(sig, over_privkey, 32); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, recid) == 0); + CHECK(ecount == 7); + + /* cleanup */ + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(vrfy); + secp256k1_context_destroy(both); +} + void test_ecdsa_recovery_end_to_end(void) { unsigned char extra[32] = {0x00}; unsigned char privkey[32]; @@ -241,6 +381,9 @@ void test_ecdsa_recovery_edge_cases(void) { void run_recovery_tests(void) { int i; + for (i = 0; i < count; i++) { + test_ecdsa_recovery_api(); + } for (i = 0; i < 64*count; i++) { test_ecdsa_recovery_end_to_end(); } diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index f5b2376407..2690d86558 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -66,88 +66,79 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar #else secp256k1_scalar *t; int i; - /* First compute x ^ (2^N - 1) for some values of N. */ - secp256k1_scalar x2, x3, x4, x6, x7, x8, x15, x30, x60, x120, x127; + /* First compute xN as x ^ (2^N - 1) for some values of N, + * and uM as x ^ M for some values of M. */ + secp256k1_scalar x2, x3, x6, x8, x14, x28, x56, x112, x126; + secp256k1_scalar u2, u5, u9, u11, u13; - secp256k1_scalar_sqr(&x2, x); - secp256k1_scalar_mul(&x2, &x2, x); + secp256k1_scalar_sqr(&u2, x); + secp256k1_scalar_mul(&x2, &u2, x); + secp256k1_scalar_mul(&u5, &u2, &x2); + secp256k1_scalar_mul(&x3, &u5, &u2); + secp256k1_scalar_mul(&u9, &x3, &u2); + secp256k1_scalar_mul(&u11, &u9, &u2); + secp256k1_scalar_mul(&u13, &u11, &u2); - secp256k1_scalar_sqr(&x3, &x2); - secp256k1_scalar_mul(&x3, &x3, x); - - secp256k1_scalar_sqr(&x4, &x3); - secp256k1_scalar_mul(&x4, &x4, x); - - secp256k1_scalar_sqr(&x6, &x4); + secp256k1_scalar_sqr(&x6, &u13); secp256k1_scalar_sqr(&x6, &x6); - secp256k1_scalar_mul(&x6, &x6, &x2); - - secp256k1_scalar_sqr(&x7, &x6); - secp256k1_scalar_mul(&x7, &x7, x); + secp256k1_scalar_mul(&x6, &x6, &u11); - secp256k1_scalar_sqr(&x8, &x7); - secp256k1_scalar_mul(&x8, &x8, x); + secp256k1_scalar_sqr(&x8, &x6); + secp256k1_scalar_sqr(&x8, &x8); + secp256k1_scalar_mul(&x8, &x8, &x2); - secp256k1_scalar_sqr(&x15, &x8); - for (i = 0; i < 6; i++) { - secp256k1_scalar_sqr(&x15, &x15); + secp256k1_scalar_sqr(&x14, &x8); + for (i = 0; i < 5; i++) { + secp256k1_scalar_sqr(&x14, &x14); } - secp256k1_scalar_mul(&x15, &x15, &x7); + secp256k1_scalar_mul(&x14, &x14, &x6); - secp256k1_scalar_sqr(&x30, &x15); - for (i = 0; i < 14; i++) { - secp256k1_scalar_sqr(&x30, &x30); + secp256k1_scalar_sqr(&x28, &x14); + for (i = 0; i < 13; i++) { + secp256k1_scalar_sqr(&x28, &x28); } - secp256k1_scalar_mul(&x30, &x30, &x15); + secp256k1_scalar_mul(&x28, &x28, &x14); - secp256k1_scalar_sqr(&x60, &x30); - for (i = 0; i < 29; i++) { - secp256k1_scalar_sqr(&x60, &x60); + secp256k1_scalar_sqr(&x56, &x28); + for (i = 0; i < 27; i++) { + secp256k1_scalar_sqr(&x56, &x56); } - secp256k1_scalar_mul(&x60, &x60, &x30); + secp256k1_scalar_mul(&x56, &x56, &x28); - secp256k1_scalar_sqr(&x120, &x60); - for (i = 0; i < 59; i++) { - secp256k1_scalar_sqr(&x120, &x120); + secp256k1_scalar_sqr(&x112, &x56); + for (i = 0; i < 55; i++) { + secp256k1_scalar_sqr(&x112, &x112); } - secp256k1_scalar_mul(&x120, &x120, &x60); + secp256k1_scalar_mul(&x112, &x112, &x56); - secp256k1_scalar_sqr(&x127, &x120); - for (i = 0; i < 6; i++) { - secp256k1_scalar_sqr(&x127, &x127); + secp256k1_scalar_sqr(&x126, &x112); + for (i = 0; i < 13; i++) { + secp256k1_scalar_sqr(&x126, &x126); } - secp256k1_scalar_mul(&x127, &x127, &x7); + secp256k1_scalar_mul(&x126, &x126, &x14); - /* Then accumulate the final result (t starts at x127). */ - t = &x127; - for (i = 0; i < 2; i++) { /* 0 */ + /* Then accumulate the final result (t starts at x126). */ + t = &x126; + for (i = 0; i < 3; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ + for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 3; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } @@ -156,38 +147,26 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 4; i++) { /* 00 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ + for (i = 0; i < 3; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &x3); /* 111 */ for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x4); /* 1111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) { /* 000 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ + for (i = 0; i < 6; i++) { /* 000 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ for (i = 0; i < 10; i++) { /* 0000000 */ secp256k1_scalar_sqr(t, t); } @@ -200,50 +179,34 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x8); /* 11111111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x4); /* 1111 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 5; i++) { /* 000 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 4; i++) { /* 00 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 5; i++) { secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 2; i++) { /* 0 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 8; i++) { /* 000000 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 3; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 10; i++) { /* 000000 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 3; i++) { /* 00 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ for (i = 0; i < 6; i++) { /* 00000 */ secp256k1_scalar_sqr(t, t); } diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index fb8b882faa..4f8c01655b 100755..100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -424,6 +424,33 @@ int secp256k1_ec_pubkey_create(const secp256k1_context* ctx, secp256k1_pubkey *p return ret; } +int secp256k1_ec_privkey_negate(const secp256k1_context* ctx, unsigned char *seckey) { + secp256k1_scalar sec; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(seckey != NULL); + + secp256k1_scalar_set_b32(&sec, seckey, NULL); + secp256k1_scalar_negate(&sec, &sec); + secp256k1_scalar_get_b32(seckey, &sec); + + return 1; +} + +int secp256k1_ec_pubkey_negate(const secp256k1_context* ctx, secp256k1_pubkey *pubkey) { + int ret = 0; + secp256k1_ge p; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(pubkey != NULL); + + ret = secp256k1_pubkey_load(ctx, &p, pubkey); + memset(pubkey, 0, sizeof(*pubkey)); + if (ret) { + secp256k1_ge_neg(&p, &p); + secp256k1_pubkey_save(pubkey, &p); + } + return ret; +} + int secp256k1_ec_privkey_tweak_add(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { secp256k1_scalar term; secp256k1_scalar sec; @@ -552,10 +579,6 @@ int secp256k1_ec_pubkey_combine(const secp256k1_context* ctx, secp256k1_pubkey * # include "modules/ecdh/main_impl.h" #endif -#ifdef ENABLE_MODULE_SCHNORR -# include "modules/schnorr/main_impl.h" -#endif - #ifdef ENABLE_MODULE_RECOVERY # include "modules/recovery/main_impl.h" #endif diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index 9ae7d30281..3d9bd5ebb4 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -10,6 +10,7 @@ #include <stdio.h> #include <stdlib.h> +#include <string.h> #include <time.h> @@ -135,6 +136,7 @@ void random_scalar_order(secp256k1_scalar *num) { void run_context_tests(void) { secp256k1_pubkey pubkey; + secp256k1_pubkey zero_pubkey; secp256k1_ecdsa_signature sig; unsigned char ctmp[32]; int32_t ecount; @@ -149,6 +151,8 @@ void run_context_tests(void) { secp256k1_scalar msg, key, nonce; secp256k1_scalar sigr, sigs; + memset(&zero_pubkey, 0, sizeof(zero_pubkey)); + ecount = 0; ecount2 = 10; secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); @@ -201,12 +205,20 @@ void run_context_tests(void) { CHECK(ecount == 2); CHECK(secp256k1_ec_pubkey_tweak_mul(sign, &pubkey, ctmp) == 0); CHECK(ecount2 == 13); - CHECK(secp256k1_ec_pubkey_tweak_mul(vrfy, &pubkey, ctmp) == 1); + CHECK(secp256k1_ec_pubkey_negate(vrfy, &pubkey) == 1); CHECK(ecount == 2); - CHECK(secp256k1_context_randomize(vrfy, ctmp) == 0); + CHECK(secp256k1_ec_pubkey_negate(sign, &pubkey) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ec_pubkey_negate(sign, NULL) == 0); + CHECK(ecount2 == 14); + CHECK(secp256k1_ec_pubkey_negate(vrfy, &zero_pubkey) == 0); CHECK(ecount == 3); + CHECK(secp256k1_ec_pubkey_tweak_mul(vrfy, &pubkey, ctmp) == 1); + CHECK(ecount == 3); + CHECK(secp256k1_context_randomize(vrfy, ctmp) == 0); + CHECK(ecount == 4); CHECK(secp256k1_context_randomize(sign, NULL) == 1); - CHECK(ecount2 == 13); + CHECK(ecount2 == 14); secp256k1_context_set_illegal_callback(vrfy, NULL, NULL); secp256k1_context_set_illegal_callback(sign, NULL, NULL); @@ -1879,9 +1891,9 @@ void test_ge(void) { * * When the endomorphism code is compiled in, p5 = lambda*p1 and p6 = lambda^2*p1 are added as well. */ - secp256k1_ge *ge = (secp256k1_ge *)malloc(sizeof(secp256k1_ge) * (1 + 4 * runs)); - secp256k1_gej *gej = (secp256k1_gej *)malloc(sizeof(secp256k1_gej) * (1 + 4 * runs)); - secp256k1_fe *zinv = (secp256k1_fe *)malloc(sizeof(secp256k1_fe) * (1 + 4 * runs)); + secp256k1_ge *ge = (secp256k1_ge *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_ge) * (1 + 4 * runs)); + secp256k1_gej *gej = (secp256k1_gej *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_gej) * (1 + 4 * runs)); + secp256k1_fe *zinv = (secp256k1_fe *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_fe) * (1 + 4 * runs)); secp256k1_fe zf; secp256k1_fe zfi2, zfi3; @@ -1919,7 +1931,7 @@ void test_ge(void) { /* Compute z inverses. */ { - secp256k1_fe *zs = malloc(sizeof(secp256k1_fe) * (1 + 4 * runs)); + secp256k1_fe *zs = checked_malloc(&ctx->error_callback, sizeof(secp256k1_fe) * (1 + 4 * runs)); for (i = 0; i < 4 * runs + 1; i++) { if (i == 0) { /* The point at infinity does not have a meaningful z inverse. Any should do. */ @@ -2020,7 +2032,7 @@ void test_ge(void) { /* Test adding all points together in random order equals infinity. */ { secp256k1_gej sum = SECP256K1_GEJ_CONST_INFINITY; - secp256k1_gej *gej_shuffled = (secp256k1_gej *)malloc((4 * runs + 1) * sizeof(secp256k1_gej)); + secp256k1_gej *gej_shuffled = (secp256k1_gej *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_gej)); for (i = 0; i < 4 * runs + 1; i++) { gej_shuffled[i] = gej[i]; } @@ -2041,9 +2053,9 @@ void test_ge(void) { /* Test batch gej -> ge conversion with and without known z ratios. */ { - secp256k1_fe *zr = (secp256k1_fe *)malloc((4 * runs + 1) * sizeof(secp256k1_fe)); - secp256k1_ge *ge_set_table = (secp256k1_ge *)malloc((4 * runs + 1) * sizeof(secp256k1_ge)); - secp256k1_ge *ge_set_all = (secp256k1_ge *)malloc((4 * runs + 1) * sizeof(secp256k1_ge)); + secp256k1_fe *zr = (secp256k1_fe *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_fe)); + secp256k1_ge *ge_set_table = (secp256k1_ge *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_ge)); + secp256k1_ge *ge_set_all = (secp256k1_ge *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_ge)); for (i = 0; i < 4 * runs + 1; i++) { /* Compute gej[i + 1].z / gez[i].z (with gej[n].z taken to be 1). */ if (i < 4 * runs) { @@ -3436,6 +3448,7 @@ void test_ecdsa_end_to_end(void) { unsigned char pubkeyc[65]; size_t pubkeyclen = 65; secp256k1_pubkey pubkey; + secp256k1_pubkey pubkey_tmp; unsigned char seckey[300]; size_t seckeylen = 300; @@ -3457,6 +3470,13 @@ void test_ecdsa_end_to_end(void) { memset(&pubkey, 0, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(ctx, &pubkey, pubkeyc, pubkeyclen) == 1); + /* Verify negation changes the key and changes it back */ + memcpy(&pubkey_tmp, &pubkey, sizeof(pubkey)); + CHECK(secp256k1_ec_pubkey_negate(ctx, &pubkey_tmp) == 1); + CHECK(memcmp(&pubkey_tmp, &pubkey, sizeof(pubkey)) != 0); + CHECK(secp256k1_ec_pubkey_negate(ctx, &pubkey_tmp) == 1); + CHECK(memcmp(&pubkey_tmp, &pubkey, sizeof(pubkey)) == 0); + /* Verify private key import and export. */ CHECK(ec_privkey_export_der(ctx, seckey, &seckeylen, privkey, secp256k1_rand_bits(1) == 1)); CHECK(ec_privkey_import_der(ctx, privkey2, seckey, seckeylen) == 1); @@ -4383,10 +4403,6 @@ void run_ecdsa_openssl(void) { # include "modules/ecdh/tests_impl.h" #endif -#ifdef ENABLE_MODULE_SCHNORR -# include "modules/schnorr/tests_impl.h" -#endif - #ifdef ENABLE_MODULE_RECOVERY # include "modules/recovery/tests_impl.h" #endif @@ -4504,11 +4520,6 @@ int main(int argc, char **argv) { run_ecdsa_openssl(); #endif -#ifdef ENABLE_MODULE_SCHNORR - /* Schnorr tests */ - run_schnorr_tests(); -#endif - #ifdef ENABLE_MODULE_RECOVERY /* ECDSA pubkey recovery tests */ run_recovery_tests(); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index bda6ee475c..b040bb0733 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -26,6 +26,11 @@ #include "secp256k1.c" #include "testrand_impl.h" +#ifdef ENABLE_MODULE_RECOVERY +#include "src/modules/recovery/main_impl.h" +#include "include/secp256k1_recovery.h" +#endif + /** stolen from tests.c */ void ge_equals_ge(const secp256k1_ge *a, const secp256k1_ge *b) { CHECK(a->infinity == b->infinity); @@ -77,7 +82,7 @@ int secp256k1_nonce_function_smallint(unsigned char *nonce32, const unsigned cha * function with an increased `attempt`. So if attempt > 0 this means we * need to change the nonce to avoid an infinite loop. */ if (attempt > 0) { - (*idata)++; + *idata = (*idata + 1) % EXHAUSTIVE_TEST_ORDER; } secp256k1_scalar_set_int(&s, *idata); secp256k1_scalar_get_b32(nonce32, &s); @@ -244,6 +249,7 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou for (i = 1; i < order; i++) { /* message */ for (j = 1; j < order; j++) { /* key */ for (k = 1; k < order; k++) { /* nonce */ + const int starting_k = k; secp256k1_ecdsa_signature sig; secp256k1_scalar sk, msg, r, s, expected_r; unsigned char sk32[32], msg32[32]; @@ -262,6 +268,11 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou CHECK(r == expected_r); CHECK((k * s) % order == (i + r * j) % order || (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + + /* Overflow means we've tried every possible nonce */ + if (k < starting_k) { + break; + } } } } @@ -276,6 +287,130 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou */ } +#ifdef ENABLE_MODULE_RECOVERY +void test_exhaustive_recovery_sign(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + int i, j, k; + + /* Loop */ + for (i = 1; i < order; i++) { /* message */ + for (j = 1; j < order; j++) { /* key */ + for (k = 1; k < order; k++) { /* nonce */ + const int starting_k = k; + secp256k1_fe r_dot_y_normalized; + secp256k1_ecdsa_recoverable_signature rsig; + secp256k1_ecdsa_signature sig; + secp256k1_scalar sk, msg, r, s, expected_r; + unsigned char sk32[32], msg32[32]; + int expected_recid; + int recid; + secp256k1_scalar_set_int(&msg, i); + secp256k1_scalar_set_int(&sk, j); + secp256k1_scalar_get_b32(sk32, &sk); + secp256k1_scalar_get_b32(msg32, &msg); + + secp256k1_ecdsa_sign_recoverable(ctx, &rsig, msg32, sk32, secp256k1_nonce_function_smallint, &k); + + /* Check directly */ + secp256k1_ecdsa_recoverable_signature_load(ctx, &r, &s, &recid, &rsig); + r_from_k(&expected_r, group, k); + CHECK(r == expected_r); + CHECK((k * s) % order == (i + r * j) % order || + (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + /* In computing the recid, there is an overflow condition that is disabled in + * scalar_low_impl.h `secp256k1_scalar_set_b32` because almost every r.y value + * will exceed the group order, and our signing code always holds out for r + * values that don't overflow, so with a proper overflow check the tests would + * loop indefinitely. */ + r_dot_y_normalized = group[k].y; + secp256k1_fe_normalize(&r_dot_y_normalized); + /* Also the recovery id is flipped depending if we hit the low-s branch */ + if ((k * s) % order == (i + r * j) % order) { + expected_recid = secp256k1_fe_is_odd(&r_dot_y_normalized) ? 1 : 0; + } else { + expected_recid = secp256k1_fe_is_odd(&r_dot_y_normalized) ? 0 : 1; + } + CHECK(recid == expected_recid); + + /* Convert to a standard sig then check */ + secp256k1_ecdsa_recoverable_signature_convert(ctx, &sig, &rsig); + secp256k1_ecdsa_signature_load(ctx, &r, &s, &sig); + /* Note that we compute expected_r *after* signing -- this is important + * because our nonce-computing function function might change k during + * signing. */ + r_from_k(&expected_r, group, k); + CHECK(r == expected_r); + CHECK((k * s) % order == (i + r * j) % order || + (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + + /* Overflow means we've tried every possible nonce */ + if (k < starting_k) { + break; + } + } + } + } +} + +void test_exhaustive_recovery_verify(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + /* This is essentially a copy of test_exhaustive_verify, with recovery added */ + int s, r, msg, key; + for (s = 1; s < order; s++) { + for (r = 1; r < order; r++) { + for (msg = 1; msg < order; msg++) { + for (key = 1; key < order; key++) { + secp256k1_ge nonconst_ge; + secp256k1_ecdsa_recoverable_signature rsig; + secp256k1_ecdsa_signature sig; + secp256k1_pubkey pk; + secp256k1_scalar sk_s, msg_s, r_s, s_s; + secp256k1_scalar s_times_k_s, msg_plus_r_times_sk_s; + int recid = 0; + int k, should_verify; + unsigned char msg32[32]; + + secp256k1_scalar_set_int(&s_s, s); + secp256k1_scalar_set_int(&r_s, r); + secp256k1_scalar_set_int(&msg_s, msg); + secp256k1_scalar_set_int(&sk_s, key); + secp256k1_scalar_get_b32(msg32, &msg_s); + + /* Verify by hand */ + /* Run through every k value that gives us this r and check that *one* works. + * Note there could be none, there could be multiple, ECDSA is weird. */ + should_verify = 0; + for (k = 0; k < order; k++) { + secp256k1_scalar check_x_s; + r_from_k(&check_x_s, group, k); + if (r_s == check_x_s) { + secp256k1_scalar_set_int(&s_times_k_s, k); + secp256k1_scalar_mul(&s_times_k_s, &s_times_k_s, &s_s); + secp256k1_scalar_mul(&msg_plus_r_times_sk_s, &r_s, &sk_s); + secp256k1_scalar_add(&msg_plus_r_times_sk_s, &msg_plus_r_times_sk_s, &msg_s); + should_verify |= secp256k1_scalar_eq(&s_times_k_s, &msg_plus_r_times_sk_s); + } + } + /* nb we have a "high s" rule */ + should_verify &= !secp256k1_scalar_is_high(&s_s); + + /* We would like to try recovering the pubkey and checking that it matches, + * but pubkey recovery is impossible in the exhaustive tests (the reason + * being that there are 12 nonzero r values, 12 nonzero points, and no + * overlap between the sets, so there are no valid signatures). */ + + /* Verify by converting to a standard signature and calling verify */ + secp256k1_ecdsa_recoverable_signature_save(&rsig, &r_s, &s_s, recid); + secp256k1_ecdsa_recoverable_signature_convert(ctx, &sig, &rsig); + memcpy(&nonconst_ge, &group[sk_s], sizeof(nonconst_ge)); + secp256k1_pubkey_save(&pk, &nonconst_ge); + CHECK(should_verify == + secp256k1_ecdsa_verify(ctx, &sig, msg32, &pk)); + } + } + } + } +} +#endif + int main(void) { int i; secp256k1_gej groupj[EXHAUSTIVE_TEST_ORDER]; @@ -324,6 +459,12 @@ int main(void) { test_exhaustive_sign(ctx, group, EXHAUSTIVE_TEST_ORDER); test_exhaustive_verify(ctx, group, EXHAUSTIVE_TEST_ORDER); +#ifdef ENABLE_MODULE_RECOVERY + test_exhaustive_recovery_sign(ctx, group, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_recovery_verify(ctx, group, EXHAUSTIVE_TEST_ORDER); +#endif + + secp256k1_context_destroy(ctx); return 0; } diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index 4eef4ded47..4092a86c91 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -57,7 +57,10 @@ static SECP256K1_INLINE void secp256k1_callback_call(const secp256k1_callback * #endif /* Like assert(), but when VERIFY is defined, and side-effect safe. */ -#ifdef VERIFY +#if defined(COVERAGE) +#define VERIFY_CHECK(check) +#define VERIFY_SETUP(stmt) +#elif defined(VERIFY) #define VERIFY_CHECK CHECK #define VERIFY_SETUP(stmt) do { stmt; } while(0) #else diff --git a/src/serialize.h b/src/serialize.h index e4d72d2348..e82ddf2c5a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -336,11 +336,18 @@ I ReadVarInt(Stream& is) I n = 0; while(true) { unsigned char chData = ser_readdata8(is); + if (n > (std::numeric_limits<I>::max() >> 7)) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n = (n << 7) | (chData & 0x7F); - if (chData & 0x80) + if (chData & 0x80) { + if (n == std::numeric_limits<I>::max()) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n++; - else + } else { return n; + } } } diff --git a/src/streams.h b/src/streams.h index 1d3b55c91e..8dc5a19ead 100644 --- a/src/streams.h +++ b/src/streams.h @@ -248,7 +248,8 @@ public: void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -261,7 +262,8 @@ public: void insert(iterator it, const char* first, const char* last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -339,6 +341,8 @@ public: void read(char* pch, size_t nSize) { + if (nSize == 0) return; + // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + nSize; if (nReadPosNext >= vch.size()) @@ -404,8 +408,8 @@ public: return (*this); } - void GetAndClear(CSerializeData &data) { - data.insert(data.end(), begin(), end()); + void GetAndClear(CSerializeData &d) { + d.insert(d.end(), begin(), end()); clear(); } @@ -584,11 +588,11 @@ protected: readNow = nAvail; if (readNow == 0) return false; - size_t read = fread((void*)&vchBuf[pos], 1, readNow, src); - if (read == 0) { + size_t nBytes = fread((void*)&vchBuf[pos], 1, readNow, src); + if (nBytes == 0) { throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed"); } else { - nSrcPos += read; + nSrcPos += nBytes; return true; } } diff --git a/src/support/cleanse.h b/src/support/cleanse.h index 3e02aa8fd1..f020216c73 100644 --- a/src/support/cleanse.h +++ b/src/support/cleanse.h @@ -8,6 +8,7 @@ #include <stdlib.h> +// Attempt to overwrite data in the specified memory span. void memory_cleanse(void *ptr, size_t len); #endif // BITCOIN_SUPPORT_CLEANSE_H diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 01273c9791..98c1581093 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -357,8 +357,8 @@ LockedPool::LockedPageArena::~LockedPageArena() /*******************************************************************************/ // Implementation: LockedPoolManager // -LockedPoolManager::LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator): - LockedPool(std::move(allocator), &LockedPoolManager::LockingFailed) +LockedPoolManager::LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator_in): + LockedPool(std::move(allocator_in), &LockedPoolManager::LockingFailed) { } diff --git a/src/sync.cpp b/src/sync.cpp index fce57f1df9..552682ab67 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -46,8 +46,6 @@ struct CLockLocation { return mutexName + " " + sourceFile + ":" + itostr(sourceLine) + (fTry ? " (TRY)" : ""); } - std::string MutexName() const { return mutexName; } - bool fTry; private: std::string mutexName; diff --git a/src/sync.h b/src/sync.h index 3b29050e0e..9274f50d8b 100644 --- a/src/sync.h +++ b/src/sync.h @@ -97,7 +97,6 @@ public: } }; -typedef CCriticalSection CDynamicCriticalSection; /** Wrapped boost mutex: supports waiting but not recursive locking */ typedef AnnotatedMixin<boost::mutex> CWaitableCriticalSection; diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index c62e6ae838..e6b45a3b5e 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) connman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, "", true); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; @@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, "", true); + CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); dummyNode2.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode2, *connman); dummyNode2.nVersion = 1; @@ -83,7 +83,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) connman->ClearBanned(); ForceSetArg("-banscore", "111"); // because 11 is my favorite number CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, "", true); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; @@ -109,7 +109,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, "", true); + CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, CAddress(), "", true); dummyNode.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode, *connman); dummyNode.nVersion = 1; diff --git a/src/test/README.md b/src/test/README.md index 8f99804e10..eeb04c6ffa 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -50,12 +50,3 @@ examine `uint256_tests.cpp`. For further reading, I found the following website to be helpful in explaining how the boost unit test framework works: [http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/). - -### bitcoin-util-test.py - -The test directory also contains the bitcoin-util-test.py tool, which tests bitcoin utils (currently just bitcoin-tx). This test gets run automatically during the `make check` build process. It is also possible to run the test manually from the src directory: - -``` -test/bitcoin-util-test.py --srcdir=[current directory] - -``` diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 322addc9f6..dc5372a070 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -84,36 +84,47 @@ BOOST_AUTO_TEST_CASE(addrman_simple) CNetAddr source = ResolveIP("252.2.2.2"); - // Test 1: Does Addrman respond correctly when empty. - BOOST_CHECK(addrman.size() == 0); + // Test: Does Addrman respond correctly when empty. + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo addr_null = addrman.Select(); - BOOST_CHECK(addr_null.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_null.ToString(), "[::]:0"); - // Test 2: Does Addrman::Add work as expected. + // Test: Does Addrman::Add work as expected. CService addr1 = ResolveService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret1 = addrman.Select(); - BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); - // Test 3: Does IP address deduplication work correctly. + // Test: Does IP address deduplication work correctly. // Expected dup IP should not be added. CService addr1_dup = ResolveService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1_dup, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK(!addrman.Add(CAddress(addr1_dup, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 1); + + // Test: New table has one addr and we add a diff addr we should + // have at least one addr. + // Note that addrman's size cannot be tested reliably after insertion, as + // hash collisions may occur. But we can always be sure of at least one + // success. - // Test 5: New table has one addr and we add a diff addr we should - // have two addrs. CService addr2 = ResolveService("250.1.1.2", 8333); - addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 2); + BOOST_CHECK(addrman.Add(CAddress(addr2, NODE_NONE), source)); + BOOST_CHECK(addrman.size() >= 1); - // Test 6: AddrMan::Clear() should empty the new table. + // Test: AddrMan::Clear() should empty the new table. addrman.Clear(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo addr_null2 = addrman.Select(); - BOOST_CHECK(addr_null2.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_null2.ToString(), "[::]:0"); + + // Test: AddrMan::Add multiple addresses works as expected + std::vector<CAddress> vAddr; + vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); + vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); + BOOST_CHECK(addrman.Add(vAddr, source)); + BOOST_CHECK(addrman.size() >= 1); } BOOST_AUTO_TEST_CASE(addrman_ports) @@ -125,26 +136,26 @@ BOOST_AUTO_TEST_CASE(addrman_ports) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = ResolveService("250.1.1.1", 8333); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CService addr1_port = ResolveService("250.1.1.1", 8334); addrman.Add(CAddress(addr1_port, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret2 = addrman.Select(); - BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret2.ToString(), "250.1.1.1:8333"); - // Test 8: Add same IP but diff port to tried table, it doesn't get added. + // Test: Add same IP but diff port to tried table, it doesn't get added. // Perhaps this is not ideal behavior but it is the current behavior. addrman.Good(CAddress(addr1_port, NODE_NONE)); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); bool newOnly = true; CAddrInfo addr_ret3 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret3.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); } @@ -157,25 +168,25 @@ BOOST_AUTO_TEST_CASE(addrman_select) CNetAddr source = ResolveIP("252.2.2.2"); - // Test 9: Select from new with 1 addr in new. + // Test: Select from new with 1 addr in new. CService addr1 = ResolveService("250.1.1.1", 8333); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); bool newOnly = true; CAddrInfo addr_ret1 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); - // Test 10: move addr to tried, select from new expected nothing returned. + // Test: move addr to tried, select from new expected nothing returned. addrman.Good(CAddress(addr1, NODE_NONE)); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret2 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret2.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); CAddrInfo addr_ret3 = addrman.Select(); - BOOST_CHECK(addr_ret3.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); // Add three addresses to new table. @@ -199,14 +210,15 @@ BOOST_AUTO_TEST_CASE(addrman_select) addrman.Add(CAddress(addr7, NODE_NONE), ResolveService("250.1.1.3", 8333)); addrman.Good(CAddress(addr7, NODE_NONE)); - // Test 11: 6 addrs + 1 addr from last test = 7. - BOOST_CHECK(addrman.size() == 7); + // Test: 6 addrs + 1 addr from last test = 7. + BOOST_CHECK_EQUAL(addrman.size(), 7); - // Test 12: Select pulls from new and tried regardless of port number. - BOOST_CHECK(addrman.Select().ToString() == "250.4.6.6:8333"); - BOOST_CHECK(addrman.Select().ToString() == "250.3.2.2:9999"); - BOOST_CHECK(addrman.Select().ToString() == "250.3.3.3:9999"); - BOOST_CHECK(addrman.Select().ToString() == "250.4.4.4:8333"); + // Test: Select pulls from new and tried regardless of port number. + std::set<uint16_t> ports; + for (int i = 0; i < 20; ++i) { + ports.insert(addrman.Select().GetPort()); + } + BOOST_CHECK_EQUAL(ports.size(), 3); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) @@ -218,24 +230,24 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); for (unsigned int i = 1; i < 18; i++) { CService addr = ResolveService("250.1.1." + boost::to_string(i)); addrman.Add(CAddress(addr, NODE_NONE), source); - //Test 13: No collision in new table yet. - BOOST_CHECK(addrman.size() == i); + //Test: No collision in new table yet. + BOOST_CHECK_EQUAL(addrman.size(), i); } - //Test 14: new table collision! + //Test: new table collision! CService addr1 = ResolveService("250.1.1.18"); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 17); + BOOST_CHECK_EQUAL(addrman.size(), 17); CService addr2 = ResolveService("250.1.1.19"); addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 18); + BOOST_CHECK_EQUAL(addrman.size(), 18); } BOOST_AUTO_TEST_CASE(addrman_tried_collisions) @@ -247,25 +259,25 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); for (unsigned int i = 1; i < 80; i++) { CService addr = ResolveService("250.1.1." + boost::to_string(i)); addrman.Add(CAddress(addr, NODE_NONE), source); addrman.Good(CAddress(addr, NODE_NONE)); - //Test 15: No collision in tried table yet. + //Test: No collision in tried table yet. BOOST_CHECK_EQUAL(addrman.size(), i); } - //Test 16: tried table collision! + //Test: tried table collision! CService addr1 = ResolveService("250.1.1.80"); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 79); + BOOST_CHECK_EQUAL(addrman.size(), 79); CService addr2 = ResolveService("250.1.1.81"); addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 80); + BOOST_CHECK_EQUAL(addrman.size(), 80); } BOOST_AUTO_TEST_CASE(addrman_find) @@ -275,7 +287,7 @@ BOOST_AUTO_TEST_CASE(addrman_find) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); @@ -288,23 +300,20 @@ BOOST_AUTO_TEST_CASE(addrman_find) addrman.Add(addr2, source2); addrman.Add(addr3, source1); - // Test 17: ensure Find returns an IP matching what we searched on. + // Test: ensure Find returns an IP matching what we searched on. CAddrInfo* info1 = addrman.Find(addr1); - BOOST_CHECK(info1); - if (info1) - BOOST_CHECK(info1->ToString() == "250.1.2.1:8333"); + BOOST_REQUIRE(info1); + BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); // Test 18; Find does not discriminate by port number. CAddrInfo* info2 = addrman.Find(addr2); - BOOST_CHECK(info2); - if (info2 && info1) - BOOST_CHECK(info2->ToString() == info1->ToString()); + BOOST_REQUIRE(info2); + BOOST_CHECK_EQUAL(info2->ToString(), info1->ToString()); - // Test 19: Find returns another IP matching what we searched on. + // Test: Find returns another IP matching what we searched on. CAddrInfo* info3 = addrman.Find(addr3); - BOOST_CHECK(info3); - if (info3) - BOOST_CHECK(info3->ToString() == "251.255.2.1:8333"); + BOOST_REQUIRE(info3); + BOOST_CHECK_EQUAL(info3->ToString(), "251.255.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_create) @@ -314,7 +323,7 @@ BOOST_AUTO_TEST_CASE(addrman_create) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); @@ -322,11 +331,11 @@ BOOST_AUTO_TEST_CASE(addrman_create) int nId; CAddrInfo* pinfo = addrman.Create(addr1, source1, &nId); - // Test 20: The result should be the same as the input addr. - BOOST_CHECK(pinfo->ToString() == "250.1.2.1:8333"); + // Test: The result should be the same as the input addr. + BOOST_CHECK_EQUAL(pinfo->ToString(), "250.1.2.1:8333"); CAddrInfo* info2 = addrman.Find(addr1); - BOOST_CHECK(info2->ToString() == "250.1.2.1:8333"); + BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:8333"); } @@ -337,7 +346,7 @@ BOOST_AUTO_TEST_CASE(addrman_delete) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); @@ -345,10 +354,10 @@ BOOST_AUTO_TEST_CASE(addrman_delete) int nId; addrman.Create(addr1, source1, &nId); - // Test 21: Delete should actually delete the addr. - BOOST_CHECK(addrman.size() == 1); + // Test: Delete should actually delete the addr. + BOOST_CHECK_EQUAL(addrman.size(), 1); addrman.Delete(nId); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK(info2 == NULL); } @@ -360,11 +369,11 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - // Test 22: Sanity check, GetAddr should never return anything if addrman + // Test: Sanity check, GetAddr should never return anything if addrman // is empty. - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); std::vector<CAddress> vAddr1 = addrman.GetAddr(); - BOOST_CHECK(vAddr1.size() == 0); + BOOST_CHECK_EQUAL(vAddr1.size(), 0); CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); addr1.nTime = GetAdjustedTime(); // Set time so isTerrible = false @@ -379,7 +388,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) CNetAddr source1 = ResolveIP("250.1.2.1"); CNetAddr source2 = ResolveIP("250.2.3.3"); - // Test 23: Ensure GetAddr works with new addresses. + // Test: Ensure GetAddr works with new addresses. addrman.Add(addr1, source1); addrman.Add(addr2, source2); addrman.Add(addr3, source1); @@ -387,21 +396,20 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) addrman.Add(addr5, source1); // GetAddr returns 23% of addresses, 23% of 5 is 1 rounded down. - BOOST_CHECK(addrman.GetAddr().size() == 1); + BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1); - // Test 24: Ensure GetAddr works with new and tried addresses. + // Test: Ensure GetAddr works with new and tried addresses. addrman.Good(CAddress(addr1, NODE_NONE)); addrman.Good(CAddress(addr2, NODE_NONE)); - BOOST_CHECK(addrman.GetAddr().size() == 1); + BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1); - // Test 25: Ensure GetAddr still returns 23% when addrman has many addrs. + // Test: Ensure GetAddr still returns 23% when addrman has many addrs. for (unsigned int i = 1; i < (8 * 256); i++) { int octet1 = i % 256; - int octet2 = (i / 256) % 256; - int octet3 = (i / (256 * 2)) % 256; - std::string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + "." + boost::to_string(octet3) + ".23"; + int octet2 = i >> 8 % 256; + std::string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + ".1.23"; CAddress addr = CAddress(ResolveService(strAddr), NODE_NONE); - + // Ensure that for all addrs in addrman, isTerrible == false. addr.nTime = GetAdjustedTime(); addrman.Add(addr, ResolveIP(strAddr)); @@ -411,10 +419,10 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) std::vector<CAddress> vAddr = addrman.GetAddr(); size_t percent23 = (addrman.size() * 23) / 100; - BOOST_CHECK(vAddr.size() == percent23); - BOOST_CHECK(vAddr.size() == 461); - // (Addrman.size() < number of addresses added) due to address collisons. - BOOST_CHECK(addrman.size() == 2007); + BOOST_CHECK_EQUAL(vAddr.size(), percent23); + BOOST_CHECK_EQUAL(vAddr.size(), 461); + // (Addrman.size() < number of addresses added) due to address collisions. + BOOST_CHECK_EQUAL(addrman.size(), 2006); } @@ -437,13 +445,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetTriedBucket(nKey1) == 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); - // Test 26: Make sure key actually randomizes bucket placement. A fail on + // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); - // Test 27: Two addresses with same IP but different ports can map to + // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); @@ -458,9 +466,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) int bucket = infoi.GetTriedBucket(nKey1); buckets.insert(bucket); } - // Test 28: IP addresses in the same group (\16 prefix for IPv4) should + // Test: IP addresses in the same group (\16 prefix for IPv4) should // never get more than 8 buckets - BOOST_CHECK(buckets.size() == 8); + BOOST_CHECK_EQUAL(buckets.size(), 8); buckets.clear(); for (int j = 0; j < 255; j++) { @@ -470,9 +478,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) int bucket = infoj.GetTriedBucket(nKey1); buckets.insert(bucket); } - // Test 29: IP addresses in the different groups should map to more than + // Test: IP addresses in the different groups should map to more than // 8 buckets. - BOOST_CHECK(buckets.size() == 160); + BOOST_CHECK_EQUAL(buckets.size(), 160); } BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) @@ -492,16 +500,18 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetNewBucket(nKey1) == 786); + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); - // Test 30: Make sure key actually randomizes bucket placement. A fail on + // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); - // Test 31: Ports should not effect bucket placement in the addr + // Test: Ports should not effect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetNewBucket(nKey1) == info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); std::set<int> buckets; for (int i = 0; i < 255; i++) { @@ -511,9 +521,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoi.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 32: IP addresses in the same group (\16 prefix for IPv4) should + // Test: IP addresses in the same group (\16 prefix for IPv4) should // always map to the same bucket. - BOOST_CHECK(buckets.size() == 1); + BOOST_CHECK_EQUAL(buckets.size(), 1); buckets.clear(); for (int j = 0; j < 4 * 255; j++) { @@ -524,7 +534,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 33: IP addresses in the same source groups should map to no more + // Test: IP addresses in the same source groups should map to no more // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); @@ -536,7 +546,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 34: IP addresses in the different source groups should map to more + // Test: IP addresses in the different source groups should map to more // than 64 buckets. BOOST_CHECK(buckets.size() > 64); } diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index fd6f88b366..952cf901f0 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -3,15 +3,23 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "amount.h" +#include "policy/feerate.h" #include "test/test_bitcoin.h" #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(amount_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(MoneyRangeTest) +{ + BOOST_CHECK_EQUAL(MoneyRange(CAmount(-1)), false); + BOOST_CHECK_EQUAL(MoneyRange(MAX_MONEY + CAmount(1)), false); + BOOST_CHECK_EQUAL(MoneyRange(CAmount(1)), true); +} + BOOST_AUTO_TEST_CASE(GetFeeTest) { - CFeeRate feeRate; + CFeeRate feeRate, altFeeRate; feeRate = CFeeRate(0); // Must always return 0 @@ -53,6 +61,11 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) BOOST_CHECK_EQUAL(feeRate.GetFee(8), -1); // Special case: returns -1 instead of 0 BOOST_CHECK_EQUAL(feeRate.GetFee(9), -1); + // check alternate constructor + feeRate = CFeeRate(1000); + altFeeRate = CFeeRate(feeRate); + BOOST_CHECK_EQUAL(feeRate.GetFee(100), altFeeRate.GetFee(100)); + // Check full constructor // default value BOOST_CHECK(CFeeRate(CAmount(-1), 1000) == CFeeRate(-1)); @@ -68,4 +81,28 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) CFeeRate(MAX_MONEY, std::numeric_limits<size_t>::max() >> 1).GetFeePerK(); } +BOOST_AUTO_TEST_CASE(BinaryOperatorTest) +{ + CFeeRate a, b; + a = CFeeRate(1); + b = CFeeRate(2); + BOOST_CHECK(a < b); + BOOST_CHECK(b > a); + BOOST_CHECK(a == a); + BOOST_CHECK(a <= b); + BOOST_CHECK(a <= a); + BOOST_CHECK(b >= a); + BOOST_CHECK(b >= b); + // a should be 0.00000002 BTC/kB now + a += a; + BOOST_CHECK(a == b); +} + +BOOST_AUTO_TEST_CASE(ToStringTest) +{ + CFeeRate feeRate; + feeRate = CFeeRate(1); + BOOST_CHECK_EQUAL(feeRate.ToString(), "0.00000001 BTC/kB"); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bitcoin-util-test.py b/src/test/bitcoin-util-test.py deleted file mode 100755 index e2087187aa..0000000000 --- a/src/test/bitcoin-util-test.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# Copyright 2014 BitPay Inc. -# Copyright 2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -from __future__ import division,print_function,unicode_literals -import os -import bctest -import buildenv -import argparse -import logging - -help_text="""Test framework for bitcoin utils. - -Runs automatically during `make check`. - -Can also be run manually from the src directory by specifying the source directory: - -test/bitcoin-util-test.py --srcdir='srcdir' [--verbose] -""" - -if __name__ == '__main__': - # Try to get the source directory from the environment variables. This will - # be set for `make check` automated runs. If environment variable is not set, - # then get the source directory from command line args. - try: - srcdir = os.environ["srcdir"] - verbose = False - except: - parser = argparse.ArgumentParser(description=help_text) - parser.add_argument('-s', '--srcdir') - parser.add_argument('-v', '--verbose', action='store_true') - args = parser.parse_args() - srcdir = args.srcdir - verbose = args.verbose - - if verbose: - level = logging.DEBUG - else: - level = logging.ERROR - formatter = '%(asctime)s - %(levelname)s - %(message)s' - # Add the format/level to the logger - logging.basicConfig(format = formatter, level=level) - - bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv) diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 311ac024f3..9e4a56919d 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -57,7 +57,7 @@ static CBlock BuildBlockTestCase() { BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { - CTxMemPool pool(CFeeRate(0)); + CTxMemPool pool; TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -156,7 +156,7 @@ public: BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) { - CTxMemPool pool(CFeeRate(0)); + CTxMemPool pool; TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -222,7 +222,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) { - CTxMemPool pool(CFeeRate(0)); + CTxMemPool pool; TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -272,7 +272,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) { - CTxMemPool pool(CFeeRate(0)); + CTxMemPool pool; CMutableTransaction coinbase; coinbase.vin.resize(1); coinbase.vin[0].scriptSig.resize(10); diff --git a/src/test/buildenv.py.in b/src/test/buildenv.py.in deleted file mode 100644 index 153f34a3db..0000000000 --- a/src/test/buildenv.py.in +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -exeext="@EXEEXT@" diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp new file mode 100644 index 0000000000..287395c6c6 --- /dev/null +++ b/src/test/checkqueue_tests.cpp @@ -0,0 +1,442 @@ +// Copyright (c) 2012-2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "util.h" +#include "utiltime.h" +#include "validation.h" + +#include "test/test_bitcoin.h" +#include "checkqueue.h" +#include <boost/test/unit_test.hpp> +#include <boost/thread.hpp> +#include <atomic> +#include <thread> +#include <vector> +#include <mutex> +#include <condition_variable> + +#include <unordered_set> +#include <memory> +#include "random.h" + +// BasicTestingSetup not sufficient because nScriptCheckThreads is not set +// otherwise. +BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) + +static const int QUEUE_BATCH_SIZE = 128; + +struct FakeCheck { + bool operator()() + { + return true; + } + void swap(FakeCheck& x){}; +}; + +struct FakeCheckCheckCompletion { + static std::atomic<size_t> n_calls; + bool operator()() + { + ++n_calls; + return true; + } + void swap(FakeCheckCheckCompletion& x){}; +}; + +struct FailingCheck { + bool fails; + FailingCheck(bool _fails) : fails(_fails){}; + FailingCheck() : fails(true){}; + bool operator()() + { + return !fails; + } + void swap(FailingCheck& x) + { + std::swap(fails, x.fails); + }; +}; + +struct UniqueCheck { + static std::mutex m; + static std::unordered_multiset<size_t> results; + size_t check_id; + UniqueCheck(size_t check_id_in) : check_id(check_id_in){}; + UniqueCheck() : check_id(0){}; + bool operator()() + { + std::lock_guard<std::mutex> l(m); + results.insert(check_id); + return true; + } + void swap(UniqueCheck& x) { std::swap(x.check_id, check_id); }; +}; + + +struct MemoryCheck { + static std::atomic<size_t> fake_allocated_memory; + bool b {false}; + bool operator()() + { + return true; + } + MemoryCheck(){}; + MemoryCheck(const MemoryCheck& x) + { + // We have to do this to make sure that destructor calls are paired + // + // Really, copy constructor should be deletable, but CCheckQueue breaks + // if it is deleted because of internal push_back. + fake_allocated_memory += b; + }; + MemoryCheck(bool b_) : b(b_) + { + fake_allocated_memory += b; + }; + ~MemoryCheck(){ + fake_allocated_memory -= b; + + }; + void swap(MemoryCheck& x) { std::swap(b, x.b); }; +}; + +struct FrozenCleanupCheck { + static std::atomic<uint64_t> nFrozen; + static std::condition_variable cv; + static std::mutex m; + // Freezing can't be the default initialized behavior given how the queue + // swaps in default initialized Checks. + bool should_freeze {false}; + bool operator()() + { + return true; + } + FrozenCleanupCheck() {} + ~FrozenCleanupCheck() + { + if (should_freeze) { + std::unique_lock<std::mutex> l(m); + nFrozen = 1; + cv.notify_one(); + cv.wait(l, []{ return nFrozen == 0;}); + } + } + void swap(FrozenCleanupCheck& x){std::swap(should_freeze, x.should_freeze);}; +}; + +// Static Allocations +std::mutex FrozenCleanupCheck::m{}; +std::atomic<uint64_t> FrozenCleanupCheck::nFrozen{0}; +std::condition_variable FrozenCleanupCheck::cv{}; +std::mutex UniqueCheck::m; +std::unordered_multiset<size_t> UniqueCheck::results; +std::atomic<size_t> FakeCheckCheckCompletion::n_calls{0}; +std::atomic<size_t> MemoryCheck::fake_allocated_memory{0}; + +// Queue Typedefs +typedef CCheckQueue<FakeCheckCheckCompletion> Correct_Queue; +typedef CCheckQueue<FakeCheck> Standard_Queue; +typedef CCheckQueue<FailingCheck> Failing_Queue; +typedef CCheckQueue<UniqueCheck> Unique_Queue; +typedef CCheckQueue<MemoryCheck> Memory_Queue; +typedef CCheckQueue<FrozenCleanupCheck> FrozenCleanup_Queue; + + +/** This test case checks that the CCheckQueue works properly + * with each specified size_t Checks pushed. + */ +void Correct_Queue_range(std::vector<size_t> range) +{ + auto small_queue = std::unique_ptr<Correct_Queue>(new Correct_Queue {QUEUE_BATCH_SIZE}); + boost::thread_group tg; + for (auto x = 0; x < nScriptCheckThreads; ++x) { + tg.create_thread([&]{small_queue->Thread();}); + } + // Make vChecks here to save on malloc (this test can be slow...) + std::vector<FakeCheckCheckCompletion> vChecks; + for (auto i : range) { + size_t total = i; + FakeCheckCheckCompletion::n_calls = 0; + CCheckQueueControl<FakeCheckCheckCompletion> control(small_queue.get()); + while (total) { + vChecks.resize(std::min(total, (size_t) GetRand(10))); + total -= vChecks.size(); + control.Add(vChecks); + } + BOOST_REQUIRE(control.Wait()); + if (FakeCheckCheckCompletion::n_calls != i) { + BOOST_REQUIRE_EQUAL(FakeCheckCheckCompletion::n_calls, i); + BOOST_TEST_MESSAGE("Failure on trial " << i << " expected, got " << FakeCheckCheckCompletion::n_calls); + } + } + tg.interrupt_all(); + tg.join_all(); +} + +/** Test that 0 checks is correct + */ +BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_Zero) +{ + std::vector<size_t> range; + range.push_back((size_t)0); + Correct_Queue_range(range); +} +/** Test that 1 check is correct + */ +BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_One) +{ + std::vector<size_t> range; + range.push_back((size_t)1); + Correct_Queue_range(range); +} +/** Test that MAX check is correct + */ +BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_Max) +{ + std::vector<size_t> range; + range.push_back(100000); + Correct_Queue_range(range); +} +/** Test that random numbers of checks are correct + */ +BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_Random) +{ + std::vector<size_t> range; + range.reserve(100000/1000); + for (size_t i = 2; i < 100000; i += std::max((size_t)1, (size_t)GetRand(std::min((size_t)1000, ((size_t)100000) - i)))) + range.push_back(i); + Correct_Queue_range(range); +} + + +/** Test that failing checks are caught */ +BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) +{ + auto fail_queue = std::unique_ptr<Failing_Queue>(new Failing_Queue {QUEUE_BATCH_SIZE}); + + boost::thread_group tg; + for (auto x = 0; x < nScriptCheckThreads; ++x) { + tg.create_thread([&]{fail_queue->Thread();}); + } + + for (size_t i = 0; i < 1001; ++i) { + CCheckQueueControl<FailingCheck> control(fail_queue.get()); + size_t remaining = i; + while (remaining) { + size_t r = GetRand(10); + + std::vector<FailingCheck> vChecks; + vChecks.reserve(r); + for (size_t k = 0; k < r && remaining; k++, remaining--) + vChecks.emplace_back(remaining == 1); + control.Add(vChecks); + } + bool success = control.Wait(); + if (i > 0) { + BOOST_REQUIRE(!success); + } else if (i == 0) { + BOOST_REQUIRE(success); + } + } + tg.interrupt_all(); + tg.join_all(); +} +// Test that a block validation which fails does not interfere with +// future blocks, ie, the bad state is cleared. +BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) +{ + auto fail_queue = std::unique_ptr<Failing_Queue>(new Failing_Queue {QUEUE_BATCH_SIZE}); + boost::thread_group tg; + for (auto x = 0; x < nScriptCheckThreads; ++x) { + tg.create_thread([&]{fail_queue->Thread();}); + } + + for (auto times = 0; times < 10; ++times) { + for (bool end_fails : {true, false}) { + CCheckQueueControl<FailingCheck> control(fail_queue.get()); + { + std::vector<FailingCheck> vChecks; + vChecks.resize(100, false); + vChecks[99] = end_fails; + control.Add(vChecks); + } + bool r =control.Wait(); + BOOST_REQUIRE(r || end_fails); + } + } + tg.interrupt_all(); + tg.join_all(); +} + +// Test that unique checks are actually all called individually, rather than +// just one check being called repeatedly. Test that checks are not called +// more than once as well +BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) +{ + auto queue = std::unique_ptr<Unique_Queue>(new Unique_Queue {QUEUE_BATCH_SIZE}); + boost::thread_group tg; + for (auto x = 0; x < nScriptCheckThreads; ++x) { + tg.create_thread([&]{queue->Thread();}); + + } + + size_t COUNT = 100000; + size_t total = COUNT; + { + CCheckQueueControl<UniqueCheck> control(queue.get()); + while (total) { + size_t r = GetRand(10); + std::vector<UniqueCheck> vChecks; + for (size_t k = 0; k < r && total; k++) + vChecks.emplace_back(--total); + control.Add(vChecks); + } + } + bool r = true; + BOOST_REQUIRE_EQUAL(UniqueCheck::results.size(), COUNT); + for (size_t i = 0; i < COUNT; ++i) + r = r && UniqueCheck::results.count(i) == 1; + BOOST_REQUIRE(r); + tg.interrupt_all(); + tg.join_all(); +} + + +// Test that blocks which might allocate lots of memory free their memory aggressively. +// +// This test attempts to catch a pathological case where by lazily freeing +// checks might mean leaving a check un-swapped out, and decreasing by 1 each +// time could leave the data hanging across a sequence of blocks. +BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) +{ + auto queue = std::unique_ptr<Memory_Queue>(new Memory_Queue {QUEUE_BATCH_SIZE}); + boost::thread_group tg; + for (auto x = 0; x < nScriptCheckThreads; ++x) { + tg.create_thread([&]{queue->Thread();}); + } + for (size_t i = 0; i < 1000; ++i) { + size_t total = i; + { + CCheckQueueControl<MemoryCheck> control(queue.get()); + while (total) { + size_t r = GetRand(10); + std::vector<MemoryCheck> vChecks; + for (size_t k = 0; k < r && total; k++) { + total--; + // Each iteration leaves data at the front, back, and middle + // to catch any sort of deallocation failure + vChecks.emplace_back(total == 0 || total == i || total == i/2); + } + control.Add(vChecks); + } + } + BOOST_REQUIRE_EQUAL(MemoryCheck::fake_allocated_memory, 0); + } + tg.interrupt_all(); + tg.join_all(); +} + +// Test that a new verification cannot occur until all checks +// have been destructed +BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) +{ + auto queue = std::unique_ptr<FrozenCleanup_Queue>(new FrozenCleanup_Queue {QUEUE_BATCH_SIZE}); + boost::thread_group tg; + bool fails = false; + for (auto x = 0; x < nScriptCheckThreads; ++x) { + tg.create_thread([&]{queue->Thread();}); + } + std::thread t0([&]() { + CCheckQueueControl<FrozenCleanupCheck> control(queue.get()); + std::vector<FrozenCleanupCheck> vChecks(1); + // Freezing can't be the default initialized behavior given how the queue + // swaps in default initialized Checks (otherwise freezing destructor + // would get called twice). + vChecks[0].should_freeze = true; + control.Add(vChecks); + control.Wait(); // Hangs here + }); + { + std::unique_lock<std::mutex> l(FrozenCleanupCheck::m); + // Wait until the queue has finished all jobs and frozen + FrozenCleanupCheck::cv.wait(l, [](){return FrozenCleanupCheck::nFrozen == 1;}); + // Try to get control of the queue a bunch of times + for (auto x = 0; x < 100 && !fails; ++x) { + fails = queue->ControlMutex.try_lock(); + } + // Unfreeze + FrozenCleanupCheck::nFrozen = 0; + } + // Awaken frozen destructor + FrozenCleanupCheck::cv.notify_one(); + // Wait for control to finish + t0.join(); + tg.interrupt_all(); + tg.join_all(); + BOOST_REQUIRE(!fails); +} + + +/** Test that CCheckQueueControl is threadsafe */ +BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) +{ + auto queue = std::unique_ptr<Standard_Queue>(new Standard_Queue{QUEUE_BATCH_SIZE}); + { + boost::thread_group tg; + std::atomic<int> nThreads {0}; + std::atomic<int> fails {0}; + for (size_t i = 0; i < 3; ++i) { + tg.create_thread( + [&]{ + CCheckQueueControl<FakeCheck> control(queue.get()); + // While sleeping, no other thread should execute to this point + auto observed = ++nThreads; + MilliSleep(10); + fails += observed != nThreads; + }); + } + tg.join_all(); + BOOST_REQUIRE_EQUAL(fails, 0); + } + { + boost::thread_group tg; + std::mutex m; + bool has_lock {false}; + bool has_tried {false}; + bool done {false}; + bool done_ack {false}; + std::condition_variable cv; + { + std::unique_lock<std::mutex> l(m); + tg.create_thread([&]{ + CCheckQueueControl<FakeCheck> control(queue.get()); + std::unique_lock<std::mutex> ll(m); + has_lock = true; + cv.notify_one(); + cv.wait(ll, [&]{return has_tried;}); + done = true; + cv.notify_one(); + // Wait until the done is acknowledged + // + cv.wait(ll, [&]{return done_ack;}); + }); + // Wait for thread to get the lock + cv.wait(l, [&](){return has_lock;}); + bool fails = false; + for (auto x = 0; x < 100 && !fails; ++x) { + fails = queue->ControlMutex.try_lock(); + } + has_tried = true; + cv.notify_one(); + cv.wait(l, [&](){return done;}); + // Acknowledge the done + done_ack = true; + cv.notify_one(); + BOOST_REQUIRE(!fails); + } + tg.join_all(); + } +} +BOOST_AUTO_TEST_SUITE_END() + diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index b25c7ccc51..0ed71b96fc 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -17,35 +17,44 @@ #include <boost/test/unit_test.hpp> -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out); +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); namespace { +//! equality test +bool operator==(const Coin &a, const Coin &b) { + // Empty Coin objects are always equal. + if (a.IsSpent() && b.IsSpent()) return true; + return a.fCoinBase == b.fCoinBase && + a.nHeight == b.nHeight && + a.out == b.out; +} + class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; - std::map<uint256, CCoins> map_; + std::map<COutPoint, Coin> map_; public: - bool GetCoins(const uint256& txid, CCoins& coins) const + bool GetCoin(const COutPoint& outpoint, Coin& coin) const { - std::map<uint256, CCoins>::const_iterator it = map_.find(txid); + std::map<COutPoint, Coin>::const_iterator it = map_.find(outpoint); if (it == map_.end()) { return false; } - coins = it->second; - if (coins.IsPruned() && insecure_rand() % 2 == 0) { + coin = it->second; + if (coin.IsSpent() && insecure_rand() % 2 == 0) { // Randomly return false in case of an empty entry. return false; } return true; } - bool HaveCoins(const uint256& txid) const + bool HaveCoin(const COutPoint& outpoint) const { - CCoins coins; - return GetCoins(txid, coins); + Coin coin; + return GetCoin(outpoint, coin); } uint256 GetBestBlock() const { return hashBestBlock_; } @@ -55,8 +64,8 @@ public: for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty entries. - map_[it->first] = it->second.coins; - if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) { + map_[it->first] = it->second.coin; + if (it->second.coin.IsSpent() && insecure_rand() % 3 == 0) { // Randomly delete empty entries on write. map_.erase(it->first); } @@ -72,15 +81,18 @@ public: class CCoinsViewCacheTest : public CCoinsViewCache { public: - CCoinsViewCacheTest(CCoinsView* base) : CCoinsViewCache(base) {} + CCoinsViewCacheTest(CCoinsView* _base) : CCoinsViewCache(_base) {} void SelfTest() const { // Manually recompute the dynamic usage of the whole data, and compare it. size_t ret = memusage::DynamicUsage(cacheCoins); + size_t count = 0; for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coins.DynamicMemoryUsage(); + ret += it->second.coin.DynamicMemoryUsage(); + ++count; } + BOOST_CHECK_EQUAL(GetCacheSize(), count); BOOST_CHECK_EQUAL(DynamicMemoryUsage(), ret); } @@ -97,7 +109,7 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // This is a large randomized insert/remove simulation test on a variable-size // stack of caches on top of CCoinsViewTest. // -// It will randomly create/update/delete CCoins entries to a tip of caches, with +// It will randomly create/update/delete Coin entries to a tip of caches, with // txids picked from a limited list of random 256-bit hashes. Occasionally, a // new tip is added to the stack of caches, or the tip is flushed and removed. // @@ -109,13 +121,15 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) bool removed_all_caches = false; bool reached_4_caches = false; bool added_an_entry = false; + bool added_an_unspendable_entry = false; bool removed_an_entry = false; bool updated_an_entry = false; bool found_an_entry = false; bool missed_an_entry = false; + bool uncached_an_entry = false; // A simple map to track what we expect the cache stack to represent. - std::map<uint256, CCoins> result; + std::map<COutPoint, Coin> result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -133,36 +147,51 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Do a random modification. { uint256 txid = txids[insecure_rand() % txids.size()]; // txid we're going to modify in this iteration. - CCoins& coins = result[txid]; - CCoinsModifier entry = stack.back()->ModifyCoins(txid); - BOOST_CHECK(coins == *entry); - if (insecure_rand() % 5 == 0 || coins.IsPruned()) { - if (coins.IsPruned()) { - added_an_entry = true; + Coin& coin = result[COutPoint(txid, 0)]; + const Coin& entry = (insecure_rand() % 500 == 0) ? AccessByTxid(*stack.back(), txid) : stack.back()->AccessCoin(COutPoint(txid, 0)); + BOOST_CHECK(coin == entry); + + if (insecure_rand() % 5 == 0 || coin.IsSpent()) { + Coin newcoin; + newcoin.out.nValue = insecure_rand(); + newcoin.nHeight = 1; + if (insecure_rand() % 16 == 0 && coin.IsSpent()) { + newcoin.out.scriptPubKey.assign(1 + (insecure_rand() & 0x3F), OP_RETURN); + BOOST_CHECK(newcoin.out.scriptPubKey.IsUnspendable()); + added_an_unspendable_entry = true; } else { - updated_an_entry = true; + newcoin.out.scriptPubKey.assign(insecure_rand() & 0x3F, 0); // Random sizes so we can test memory usage accounting + (coin.IsSpent() ? added_an_entry : updated_an_entry) = true; + coin = newcoin; } - coins.nVersion = insecure_rand(); - coins.vout.resize(1); - coins.vout[0].nValue = insecure_rand(); - *entry = coins; + stack.back()->AddCoin(COutPoint(txid, 0), std::move(newcoin), !coin.IsSpent() || insecure_rand() & 1); } else { - coins.Clear(); - entry->Clear(); removed_an_entry = true; + coin.Clear(); + stack.back()->SpendCoin(COutPoint(txid, 0)); } } + // One every 10 iterations, remove a random entry from the cache + if (insecure_rand() % 10 == 0) { + COutPoint out(txids[insecure_rand() % txids.size()], 0); + int cacheid = insecure_rand() % stack.size(); + stack[cacheid]->Uncache(out); + uncached_an_entry |= !stack[cacheid]->HaveCoinInCache(out); + } + // Once every 1000 iterations and at the end, verify the full cache. if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - found_an_entry = true; - } else { - BOOST_CHECK(it->second.IsPruned()); + for (auto it = result.begin(); it != result.end(); it++) { + bool have = stack.back()->HaveCoin(it->first); + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); + if (coin.IsSpent()) { missed_an_entry = true; + } else { + BOOST_CHECK(stack.back()->HaveCoinInCache(it->first)); + found_an_entry = true; } } BOOST_FOREACH(const CCoinsViewCacheTest *test, stack) { @@ -211,25 +240,27 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(removed_all_caches); BOOST_CHECK(reached_4_caches); BOOST_CHECK(added_an_entry); + BOOST_CHECK(added_an_unspendable_entry); BOOST_CHECK(removed_an_entry); BOOST_CHECK(updated_an_entry); BOOST_CHECK(found_an_entry); BOOST_CHECK(missed_an_entry); + BOOST_CHECK(uncached_an_entry); } -typedef std::tuple<CTransaction,CTxUndo,CCoins> TxData; // Store of all necessary tx and undo data for next test -std::map<uint256, TxData> alltxs; - -TxData &FindRandomFrom(const std::set<uint256> &txidset) { - assert(txidset.size()); - std::set<uint256>::iterator txIt = txidset.lower_bound(GetRandHash()); - if (txIt == txidset.end()) { - txIt = txidset.begin(); +typedef std::map<COutPoint, std::tuple<CTransaction,CTxUndo,Coin>> UtxoData; +UtxoData utxoData; + +UtxoData::iterator FindRandomFrom(const std::set<COutPoint> &utxoSet) { + assert(utxoSet.size()); + auto utxoSetIt = utxoSet.lower_bound(COutPoint(GetRandHash(), 0)); + if (utxoSetIt == utxoSet.end()) { + utxoSetIt = utxoSet.begin(); } - std::map<uint256, TxData>::iterator txdit = alltxs.find(*txIt); - assert(txdit != alltxs.end()); - return txdit->second; + auto utxoDataIt = utxoData.find(*utxoSetIt); + assert(utxoDataIt != utxoData.end()); + return utxoDataIt; } @@ -242,7 +273,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) { bool spent_a_duplicate_coinbase = false; // A simple map to track what we expect the cache stack to represent. - std::map<uint256, CCoins> result; + std::map<COutPoint, Coin> result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -250,10 +281,10 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. // Track the txids we've used in various sets - std::set<uint256> coinbaseids; - std::set<uint256> disconnectedids; - std::set<uint256> duplicateids; - std::set<uint256> utxoset; + std::set<COutPoint> coinbase_coins; + std::set<COutPoint> disconnected_coins; + std::set<COutPoint> duplicate_coins; + std::set<COutPoint> utxoset; for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { uint32_t randiter = insecure_rand(); @@ -264,23 +295,24 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) tx.vin.resize(1); tx.vout.resize(1); tx.vout[0].nValue = i; //Keep txs unique unless intended to duplicate + tx.vout[0].scriptPubKey.assign(insecure_rand() & 0x3F, 0); // Random sizes so we can test memory usage accounting unsigned int height = insecure_rand(); - CCoins oldcoins; + Coin old_coin; // 2/20 times create a new coinbase - if (randiter % 20 < 2 || coinbaseids.size() < 10) { + if (randiter % 20 < 2 || coinbase_coins.size() < 10) { // 1/10 of those times create a duplicate coinbase - if (insecure_rand() % 10 == 0 && coinbaseids.size()) { - TxData &txd = FindRandomFrom(coinbaseids); + if (insecure_rand() % 10 == 0 && coinbase_coins.size()) { + auto utxod = FindRandomFrom(coinbase_coins); // Reuse the exact same coinbase - tx = std::get<0>(txd); + tx = std::get<0>(utxod->second); // shouldn't be available for reconnection if its been duplicated - disconnectedids.erase(tx.GetHash()); + disconnected_coins.erase(utxod->first); - duplicateids.insert(tx.GetHash()); + duplicate_coins.insert(utxod->first); } else { - coinbaseids.insert(tx.GetHash()); + coinbase_coins.insert(COutPoint(tx.GetHash(), 0)); } assert(CTransaction(tx).IsCoinBase()); } @@ -288,114 +320,118 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // 17/20 times reconnect previous or add a regular tx else { - uint256 prevouthash; + COutPoint prevout; // 1/20 times reconnect a previously disconnected tx - if (randiter % 20 == 2 && disconnectedids.size()) { - TxData &txd = FindRandomFrom(disconnectedids); - tx = std::get<0>(txd); - prevouthash = tx.vin[0].prevout.hash; - if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevouthash)) { - disconnectedids.erase(tx.GetHash()); + if (randiter % 20 == 2 && disconnected_coins.size()) { + auto utxod = FindRandomFrom(disconnected_coins); + tx = std::get<0>(utxod->second); + prevout = tx.vin[0].prevout; + if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevout)) { + disconnected_coins.erase(utxod->first); continue; } // If this tx is already IN the UTXO, then it must be a coinbase, and it must be a duplicate - if (utxoset.count(tx.GetHash())) { + if (utxoset.count(utxod->first)) { assert(CTransaction(tx).IsCoinBase()); - assert(duplicateids.count(tx.GetHash())); + assert(duplicate_coins.count(utxod->first)); } - disconnectedids.erase(tx.GetHash()); + disconnected_coins.erase(utxod->first); } // 16/20 times create a regular tx else { - TxData &txd = FindRandomFrom(utxoset); - prevouthash = std::get<0>(txd).GetHash(); + auto utxod = FindRandomFrom(utxoset); + prevout = utxod->first; // Construct the tx to spend the coins of prevouthash - tx.vin[0].prevout.hash = prevouthash; - tx.vin[0].prevout.n = 0; + tx.vin[0].prevout = prevout; assert(!CTransaction(tx).IsCoinBase()); } // In this simple test coins only have two states, spent or unspent, save the unspent state to restore - oldcoins = result[prevouthash]; + old_coin = result[prevout]; // Update the expected result of prevouthash to know these coins are spent - result[prevouthash].Clear(); + result[prevout].Clear(); - utxoset.erase(prevouthash); + utxoset.erase(prevout); // The test is designed to ensure spending a duplicate coinbase will work properly // if that ever happens and not resurrect the previously overwritten coinbase - if (duplicateids.count(prevouthash)) + if (duplicate_coins.count(prevout)) { spent_a_duplicate_coinbase = true; + } } // Update the expected result to know about the new output coins - result[tx.GetHash()].FromTx(tx, height); + assert(tx.vout.size() == 1); + const COutPoint outpoint(tx.GetHash(), 0); + result[outpoint] = Coin(tx.vout[0], height, CTransaction(tx).IsCoinBase()); // Call UpdateCoins on the top cache CTxUndo undo; UpdateCoins(tx, *(stack.back()), undo, height); // Update the utxo set for future spends - utxoset.insert(tx.GetHash()); + utxoset.insert(outpoint); // Track this tx and undo info to use later - alltxs.insert(std::make_pair(tx.GetHash(),std::make_tuple(tx,undo,oldcoins))); - } - - //1/20 times undo a previous transaction - else if (utxoset.size()) { - TxData &txd = FindRandomFrom(utxoset); + utxoData.emplace(outpoint, std::make_tuple(tx,undo,old_coin)); + } else if (utxoset.size()) { + //1/20 times undo a previous transaction + auto utxod = FindRandomFrom(utxoset); - CTransaction &tx = std::get<0>(txd); - CTxUndo &undo = std::get<1>(txd); - CCoins &origcoins = std::get<2>(txd); - - uint256 undohash = tx.GetHash(); + CTransaction &tx = std::get<0>(utxod->second); + CTxUndo &undo = std::get<1>(utxod->second); + Coin &orig_coin = std::get<2>(utxod->second); // Update the expected result // Remove new outputs - result[undohash].Clear(); + result[utxod->first].Clear(); // If not coinbase restore prevout if (!tx.IsCoinBase()) { - result[tx.vin[0].prevout.hash] = origcoins; + result[tx.vin[0].prevout] = orig_coin; } // Disconnect the tx from the current UTXO // See code in DisconnectBlock // remove outputs - { - CCoinsModifier outs = stack.back()->ModifyCoins(undohash); - outs->Clear(); - } + stack.back()->SpendCoin(utxod->first); // restore inputs if (!tx.IsCoinBase()) { const COutPoint &out = tx.vin[0].prevout; - const CTxInUndo &undoin = undo.vprevout[0]; - ApplyTxInUndo(undoin, *(stack.back()), out); + Coin coin = undo.vprevout[0]; + ApplyTxInUndo(std::move(coin), *(stack.back()), out); } // Store as a candidate for reconnection - disconnectedids.insert(undohash); + disconnected_coins.insert(utxod->first); // Update the utxoset - utxoset.erase(undohash); + utxoset.erase(utxod->first); if (!tx.IsCoinBase()) - utxoset.insert(tx.vin[0].prevout.hash); + utxoset.insert(tx.vin[0].prevout); } // Once every 1000 iterations and at the end, verify the full cache. if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - } else { - BOOST_CHECK(it->second.IsPruned()); - } + for (auto it = result.begin(); it != result.end(); it++) { + bool have = stack.back()->HaveCoin(it->first); + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); } } + // One every 10 iterations, remove a random entry from the cache + if (utxoset.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(utxoset)->first); + } + if (disconnected_coins.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(disconnected_coins)->first); + } + if (duplicate_coins.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(duplicate_coins)->first); + } + if (insecure_rand() % 100 == 0) { // Every 100 iterations, flush an intermediate cache if (stack.size() > 1 && insecure_rand() % 2 == 0) { @@ -433,53 +469,36 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_AUTO_TEST_CASE(ccoins_serialization) { // Good example - CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); - CCoins cc1; + CDataStream ss1(ParseHex("97f23c835800816115944e077fe7c803cfa57f29b36bf87c1d35"), SER_DISK, CLIENT_VERSION); + Coin cc1; ss1 >> cc1; - BOOST_CHECK_EQUAL(cc1.nVersion, 1); BOOST_CHECK_EQUAL(cc1.fCoinBase, false); BOOST_CHECK_EQUAL(cc1.nHeight, 203998); - BOOST_CHECK_EQUAL(cc1.vout.size(), 2); - BOOST_CHECK_EQUAL(cc1.IsAvailable(0), false); - BOOST_CHECK_EQUAL(cc1.IsAvailable(1), true); - BOOST_CHECK_EQUAL(cc1.vout[1].nValue, 60000000000ULL); - BOOST_CHECK_EQUAL(HexStr(cc1.vout[1].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); + BOOST_CHECK_EQUAL(cc1.out.nValue, 60000000000ULL); + BOOST_CHECK_EQUAL(HexStr(cc1.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); // Good example - CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); - CCoins cc2; + CDataStream ss2(ParseHex("8ddf77bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4"), SER_DISK, CLIENT_VERSION); + Coin cc2; ss2 >> cc2; - BOOST_CHECK_EQUAL(cc2.nVersion, 1); BOOST_CHECK_EQUAL(cc2.fCoinBase, true); BOOST_CHECK_EQUAL(cc2.nHeight, 120891); - BOOST_CHECK_EQUAL(cc2.vout.size(), 17); - for (int i = 0; i < 17; i++) { - BOOST_CHECK_EQUAL(cc2.IsAvailable(i), i == 4 || i == 16); - } - BOOST_CHECK_EQUAL(cc2.vout[4].nValue, 234925952); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[4].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("61b01caab50f1b8e9c50a5057eb43c2d9563a4ee")))))); - BOOST_CHECK_EQUAL(cc2.vout[16].nValue, 110397); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[16].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); + BOOST_CHECK_EQUAL(cc2.out.nValue, 110397); + BOOST_CHECK_EQUAL(HexStr(cc2.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); // Smallest possible example - CDataStream ssx(SER_DISK, CLIENT_VERSION); - BOOST_CHECK_EQUAL(HexStr(ssx.begin(), ssx.end()), ""); - - CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); - CCoins cc3; + CDataStream ss3(ParseHex("000006"), SER_DISK, CLIENT_VERSION); + Coin cc3; ss3 >> cc3; - BOOST_CHECK_EQUAL(cc3.nVersion, 0); BOOST_CHECK_EQUAL(cc3.fCoinBase, false); BOOST_CHECK_EQUAL(cc3.nHeight, 0); - BOOST_CHECK_EQUAL(cc3.vout.size(), 1); - BOOST_CHECK_EQUAL(cc3.IsAvailable(0), true); - BOOST_CHECK_EQUAL(cc3.vout[0].nValue, 0); - BOOST_CHECK_EQUAL(cc3.vout[0].scriptPubKey.size(), 0); + BOOST_CHECK_EQUAL(cc3.out.nValue, 0); + BOOST_CHECK_EQUAL(cc3.out.scriptPubKey.size(), 0); // scriptPubKey that ends beyond the end of the stream - CDataStream ss4(ParseHex("0002000800"), SER_DISK, CLIENT_VERSION); + CDataStream ss4(ParseHex("000007"), SER_DISK, CLIENT_VERSION); try { - CCoins cc4; + Coin cc4; ss4 >> cc4; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { @@ -490,16 +509,16 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) uint64_t x = 3000000000ULL; tmp << VARINT(x); BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); - CDataStream ss5(ParseHex("0002008a95c0bb0000"), SER_DISK, CLIENT_VERSION); + CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { - CCoins cc5; + Coin cc5; ss5 >> cc5; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { } } -const static uint256 TXID; +const static COutPoint OUTPOINT; const static CAmount PRUNED = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; @@ -514,15 +533,15 @@ const static auto FLAGS = {char(0), FRESH, DIRTY, char(DIRTY | FRESH)}; const static auto CLEAN_FLAGS = {char(0), FRESH}; const static auto ABSENT_FLAGS = {NO_ENTRY}; -void SetCoinsValue(CAmount value, CCoins& coins) +void SetCoinsValue(CAmount value, Coin& coin) { assert(value != ABSENT); - coins.Clear(); - assert(coins.IsPruned()); + coin.Clear(); + assert(coin.IsSpent()); if (value != PRUNED) { - coins.vout.emplace_back(); - coins.vout.back().nValue = value; - assert(!coins.IsPruned()); + coin.out.nValue = value; + coin.nHeight = 1; + assert(!coin.IsSpent()); } } @@ -535,25 +554,23 @@ size_t InsertCoinsMapEntry(CCoinsMap& map, CAmount value, char flags) assert(flags != NO_ENTRY); CCoinsCacheEntry entry; entry.flags = flags; - SetCoinsValue(value, entry.coins); - auto inserted = map.emplace(TXID, std::move(entry)); + SetCoinsValue(value, entry.coin); + auto inserted = map.emplace(OUTPOINT, std::move(entry)); assert(inserted.second); - return inserted.first->second.coins.DynamicMemoryUsage(); + return inserted.first->second.coin.DynamicMemoryUsage(); } void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) { - auto it = map.find(TXID); + auto it = map.find(OUTPOINT); if (it == map.end()) { value = ABSENT; flags = NO_ENTRY; } else { - if (it->second.coins.IsPruned()) { - assert(it->second.coins.vout.size() == 0); + if (it->second.coin.IsSpent()) { value = PRUNED; } else { - assert(it->second.coins.vout.size() == 1); - value = it->second.coins.vout[0].nValue; + value = it->second.coin.out.nValue; } flags = it->second.flags; assert(flags != NO_ENTRY); @@ -581,10 +598,10 @@ public: CCoinsViewCacheTest cache{&base}; }; -void CheckAccessCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckAccessCoin(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - test.cache.AccessCoins(TXID); + test.cache.AccessCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -603,39 +620,39 @@ BOOST_AUTO_TEST_CASE(ccoins_access) * Base Cache Result Cache Result * Value Value Value Flags Flags */ - CheckAccessCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); } -void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckSpendCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - SetCoinsValue(modify_value, *test.cache.ModifyCoins(TXID)); + test.cache.SpendCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -645,79 +662,55 @@ void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_va BOOST_CHECK_EQUAL(result_flags, expected_flags); }; -BOOST_AUTO_TEST_CASE(ccoins_modify) +BOOST_AUTO_TEST_CASE(ccoins_spend) { - /* Check ModifyCoin behavior, requesting a coin from a cache view layered on - * top of a base view, writing a modification to the coin, and then checking + /* Check SpendCoin behavior, requesting a coin from a cache view layered on + * top of a base view, spending, and then checking * the resulting entry in the cache after the modification. * - * Base Cache Write Result Cache Result - * Value Value Value Value Flags Flags + * Base Cache Result Cache Result + * Value Value Value Flags Flags */ - CheckModifyCoins(ABSENT, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(ABSENT, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(PRUNED, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); + CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, ABSENT, PRUNED, NO_ENTRY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); } -void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) +void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); CAmount result_value; char result_flags; try { - SetCoinsValue(modify_value, *test.cache.ModifyNewCoins(TXID, coinbase)); + CTxOut output; + output.nValue = modify_value; + test.cache.AddCoin(OUTPOINT, Coin(std::move(output), 1, coinbase), coinbase); + test.cache.SelfTest(); GetCoinsMapEntry(test.cache.map(), result_value, result_flags); } catch (std::logic_error& e) { result_value = FAIL; @@ -728,64 +721,46 @@ void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount mo BOOST_CHECK_EQUAL(result_flags, expected_flags); } -// Simple wrapper for CheckModifyNewCoinsBase function above that loops through +// Simple wrapper for CheckAddCoinBase function above that loops through // different possible base_values, making sure each one gives the same results. -// This wrapper lets the modify_new test below be shorter and less repetitive, -// while still verifying that the CoinsViewCache::ModifyNewCoins implementation +// This wrapper lets the coins_add test below be shorter and less repetitive, +// while still verifying that the CoinsViewCache::AddCoin implementation // ignores base values. template <typename... Args> -void CheckModifyNewCoins(Args&&... args) +void CheckAddCoin(Args&&... args) { for (CAmount base_value : {ABSENT, PRUNED, VALUE1}) - CheckModifyNewCoinsBase(base_value, std::forward<Args>(args)...); + CheckAddCoinBase(base_value, std::forward<Args>(args)...); } -BOOST_AUTO_TEST_CASE(ccoins_modify_new) +BOOST_AUTO_TEST_CASE(ccoins_add) { - /* Check ModifyNewCoin behavior, requesting a new coin from a cache view, + /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * with the ModifyNewCoin coinbase argument set to false, and to true. + * with the AddCoin potential_overwrite argument set to false, and to true. * - * Cache Write Result Cache Result Coinbase - * Value Value Value Flags Flags + * Cache Write Result Cache Result potential_overwrite + * Value Value Value Flags Flags */ - CheckModifyNewCoins(ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY , false); - CheckModifyNewCoins(ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, 0 , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); } void CheckWriteCoins(CAmount parent_value, CAmount child_value, CAmount expected_value, char parent_flags, char child_flags, char expected_flags) diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 4d17417179..72e562808a 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -3,12 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "crypto/aes.h" +#include "crypto/chacha20.h" #include "crypto/ripemd160.h" #include "crypto/sha1.h" #include "crypto/sha256.h" #include "crypto/sha512.h" #include "crypto/hmac_sha256.h" #include "crypto/hmac_sha512.h" +#include "random.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" #include "test/test_random.h" @@ -187,6 +189,19 @@ void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, bool pad } } +void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t seek, const std::string& hexout) +{ + std::vector<unsigned char> key = ParseHex(hexkey); + ChaCha20 rng(key.data(), key.size()); + rng.SetIV(nonce); + rng.Seek(seek); + std::vector<unsigned char> out = ParseHex(hexout); + std::vector<unsigned char> outres; + outres.resize(out.size()); + rng.Output(outres.data(), outres.size()); + BOOST_CHECK(out == outres); +} + std::string LongTestString(void) { std::string ret; for (int i=0; i<200000; i++) { @@ -439,4 +454,57 @@ BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { "b2eb05e2c39be9fcda6c19078c6a9d1b3f461796d6b0d6b2e0c2a72b4d80e644"); } + +BOOST_AUTO_TEST_CASE(chacha20_testvector) +{ + // Test vector from RFC 7539 + TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, + "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf788a3b0aa372600a92b57974cded2b9334794cb" + "a40c63e34cdea212c4cf07d41b769a6749f3f630f4122cafe28ec4dc47e26d4346d70b98c73f3e9c53ac40c5945398b6eda1a" + "832c89c167eacd901d7e2bf363"); + + // Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0, 0, + "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b" + "8f41518a11cc387b669b2ee6586"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000001", 0, 0, + "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d79" + "2b1c43fea817e9ad275ae546963"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0x0100000000000000ULL, 0, + "de9cba7bf3d69ef5e786dc63973f653a0b49e015adbff7134fcb7df137821031e85a050278a7084527214f73efc7fa5b52770" + "62eb7a0433e445f41e3"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 1, 0, + "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32111e4caf237ee53ca8ad6426194a88545ddc4" + "97a0b466e7d6bbdb0041b2f586b"); + TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x0706050403020100ULL, 0, + "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3b" + "e59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc1" + "18be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5" + "a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5" + "360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb3ab78" + "fab78c9"); +} + +BOOST_AUTO_TEST_CASE(countbits_tests) +{ + FastRandomContext ctx; + for (int i = 0; i <= 64; ++i) { + if (i == 0) { + // Check handling of zero. + BOOST_CHECK_EQUAL(CountBits(0), 0); + } else if (i < 10) { + for (uint64_t j = 1 << (i - 1); (j >> i) == 0; ++j) { + // Exhaustively test up to 10 bits + BOOST_CHECK_EQUAL(CountBits(j), i); + } + } else { + for (int k = 0; k < 1000; k++) { + // Randomly test 1000 samples of each length above 10 bits. + uint64_t j = ((uint64_t)1) << (i - 1) | ctx.randbits(i - 1); + BOOST_CHECK_EQUAL(CountBits(j), i); + } + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 00446aa11e..8cae4e66e8 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> #include "cuckoocache.h" +#include "script/sigcache.h" #include "test/test_bitcoin.h" #include "random.h" #include <thread> @@ -36,20 +37,6 @@ void insecure_GetRandHash(uint256& t) *(ptr++) = insecure_rand.rand32(); } -/** Definition copied from /src/script/sigcache.cpp - */ -class uint256Hasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin() + 4 * hash_select, 4); - return u; - } -}; /* Test that no values not inserted into the cache are read out of it. @@ -59,8 +46,9 @@ public: BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { insecure_rand = FastRandomContext(true); - CuckooCache::cache<uint256, uint256Hasher> cc{}; - cc.setup_bytes(32 << 20); + CuckooCache::cache<uint256, SignatureCacheHasher> cc{}; + size_t megabytes = 4; + cc.setup_bytes(megabytes << 20); uint256 v; for (int x = 0; x < 100000; ++x) { insecure_GetRandHash(v); @@ -135,9 +123,9 @@ BOOST_AUTO_TEST_CASE(cuckoocache_hit_rate_ok) * as a lower bound on performance. */ double HitRateThresh = 0.98; - size_t megabytes = 32; + size_t megabytes = 4; for (double load = 0.1; load < 2; load *= 2) { - double hits = test_cache<CuckooCache::cache<uint256, uint256Hasher>>(megabytes, load); + double hits = test_cache<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes, load); BOOST_CHECK(normalize_hit_rate(hits, load) > HitRateThresh); } } @@ -204,8 +192,8 @@ void test_cache_erase(size_t megabytes) BOOST_AUTO_TEST_CASE(cuckoocache_erase_ok) { - size_t megabytes = 32; - test_cache_erase<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); + size_t megabytes = 4; + test_cache_erase<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes); } template <typename Cache> @@ -291,8 +279,8 @@ void test_cache_erase_parallel(size_t megabytes) } BOOST_AUTO_TEST_CASE(cuckoocache_erase_parallel_ok) { - size_t megabytes = 32; - test_cache_erase_parallel<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); + size_t megabytes = 4; + test_cache_erase_parallel<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes); } @@ -342,13 +330,13 @@ void test_cache_generations() } }; - const uint32_t BLOCK_SIZE = 10000; + const uint32_t BLOCK_SIZE = 1000; // We expect window size 60 to perform reasonably given that each epoch // stores 45% of the cache size (~472k). const uint32_t WINDOW_SIZE = 60; const uint32_t POP_AMOUNT = (BLOCK_SIZE / WINDOW_SIZE) / 2; const double load = 10; - const size_t megabytes = 32; + const size_t megabytes = 4; const size_t bytes = megabytes * (1 << 20); const uint32_t n_insert = static_cast<uint32_t>(load * (bytes / sizeof(uint256))); @@ -379,7 +367,7 @@ void test_cache_generations() // Loose Check that hit rate is above min_hit_rate BOOST_CHECK(hit > min_hit_rate); // Tighter check, count number of times we are less than tight_hit_rate - // (and implicityly, greater than min_hit_rate) + // (and implicitly, greater than min_hit_rate) out_of_tight_tolerance += hit < tight_hit_rate; } // Check that being out of tolerance happens less than @@ -388,7 +376,7 @@ void test_cache_generations() } BOOST_AUTO_TEST_CASE(cuckoocache_generations) { - test_cache_generations<CuckooCache::cache<uint256, uint256Hasher>>(); + test_cache_generations<CuckooCache::cache<uint256, SignatureCacheHasher>>(); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index 5c054ed3e8..e35a7ce569 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -349,7 +349,7 @@ ["2147483647", "0x04 0xFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], ["2147483648", "0x05 0x0000008000 EQUAL", "P2SH,STRICTENC", "OK"], ["549755813887", "0x05 0xFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], -["549755813888", "0x06 0xFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], +["549755813888", "0x06 0x000000008000 EQUAL", "P2SH,STRICTENC", "OK"], ["9223372036854775807", "0x08 0xFFFFFFFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], ["-1", "0x01 0x81 EQUAL", "P2SH,STRICTENC", "OK", "Numbers are little-endian with the MSB being a sign bit"], ["-127", "0x01 0xFF EQUAL", "P2SH,STRICTENC", "OK"], diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 22c90bd95b..c9d9849ada 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -28,7 +28,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper) // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); char key = 'k'; uint256 in = GetRandHash(); @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_batch) // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); char key = 'i'; @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_iterator) // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); // The two keys are intentionally chosen for ordering @@ -125,8 +125,8 @@ BOOST_AUTO_TEST_CASE(dbwrapper_iterator) // Test that we do not obfuscation if there is existing data. BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) { - // We're going to share this boost::filesystem::path between two wrappers - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + // We're going to share this fs::path between two wrappers + fs::path ph = fs::temp_directory_path() / fs::unique_path(); create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. @@ -141,6 +141,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) // Call the destructor to free leveldb LOCK delete dbw; + dbw = nullptr; // Now, set up another wrapper that wants to obfuscate the same directory CDBWrapper odbw(ph, (1 << 10), false, false, true); @@ -166,8 +167,8 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) // Ensure that we start obfuscating during a reindex. BOOST_AUTO_TEST_CASE(existing_data_reindex) { - // We're going to share this boost::filesystem::path between two wrappers - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + // We're going to share this fs::path between two wrappers + fs::path ph = fs::temp_directory_path() / fs::unique_path(); create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. @@ -182,6 +183,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) // Call the destructor to free leveldb LOCK delete dbw; + dbw = nullptr; // Simulate a -reindex by wiping the existing data store CDBWrapper odbw(ph, (1 << 10), false, true, true); @@ -202,7 +204,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) BOOST_AUTO_TEST_CASE(iterator_ordering) { - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, false); for (int x=0x00; x<256; ++x) { uint8_t key = x; @@ -273,7 +275,7 @@ BOOST_AUTO_TEST_CASE(iterator_string_ordering) { char buf[10]; - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, false); for (int x=0x00; x<10; ++x) { for (int y = 0; y < 10; y++) { diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index d8de765db1..bb7e473248 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -128,6 +128,23 @@ BOOST_AUTO_TEST_CASE(siphash) tx.nVersion = 1; ss << tx; BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); + + // Check consistency between CSipHasher and SipHashUint256[Extra]. + FastRandomContext ctx; + for (int i = 0; i < 16; ++i) { + uint64_t k1 = ctx.rand64(); + uint64_t k2 = ctx.rand64(); + uint256 x = GetRandHash(); + uint32_t n = ctx.rand32(); + uint8_t nb[4]; + WriteLE32(nb, n); + CSipHasher sip256(k1, k2); + sip256.Write(x.begin(), 32); + CSipHasher sip288 = sip256; + sip288.Write(nb, 4); + BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize()); + BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize()); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 40a7fdf11d..559b3caf1c 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -29,34 +29,6 @@ static const CBitcoinAddress addr2C("1CRj2HyM1CXWzHAXLQtiGLyggNT9WQqsDs"); static const std::string strAddressBad("1HV9Lc3sNHZxwj4Zk6fB38tEmBryq2cBiF"); -#ifdef KEY_TESTS_DUMPINFO -void dumpKeyInfo(uint256 privkey) -{ - CKey key; - key.resize(32); - memcpy(&secret[0], &privkey, 32); - std::vector<unsigned char> sec; - sec.resize(32); - memcpy(&sec[0], &secret[0], 32); - printf(" * secret (hex): %s\n", HexStr(sec).c_str()); - - for (int nCompressed=0; nCompressed<2; nCompressed++) - { - bool fCompressed = nCompressed == 1; - printf(" * %s:\n", fCompressed ? "compressed" : "uncompressed"); - CBitcoinSecret bsecret; - bsecret.SetSecret(secret, fCompressed); - printf(" * secret (base58): %s\n", bsecret.ToString().c_str()); - CKey key; - key.SetSecret(secret, fCompressed); - std::vector<unsigned char> vchPubKey = key.GetPubKey(); - printf(" * pubkey (hex): %s\n", HexStr(vchPubKey).c_str()); - printf(" * address (base58): %s\n", CBitcoinAddress(vchPubKey).ToString().c_str()); - } -} -#endif - - BOOST_FIXTURE_TEST_SUITE(key_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(key_test1) diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index d52104b4cc..656aec606b 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -39,17 +39,18 @@ static void TestBlockSubsidyHalvings(int nSubsidyHalvingInterval) BOOST_AUTO_TEST_CASE(block_subsidy_test) { - TestBlockSubsidyHalvings(Params(CBaseChainParams::MAIN).GetConsensus()); // As in main + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + TestBlockSubsidyHalvings(chainParams->GetConsensus()); // As in main TestBlockSubsidyHalvings(150); // As in regtest TestBlockSubsidyHalvings(1000); // Just another interval } BOOST_AUTO_TEST_CASE(subsidy_limit_test) { - const Consensus::Params& consensusParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); CAmount nSum = 0; for (int nHeight = 0; nHeight < 14000000; nHeight += 1000) { - CAmount nSubsidy = GetBlockSubsidy(nHeight, consensusParams); + CAmount nSubsidy = GetBlockSubsidy(nHeight, chainParams->GetConsensus()); BOOST_CHECK(nSubsidy <= 50 * COIN); nSum += nSubsidy * 1000; BOOST_CHECK(MoneyRange(nSum)); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 91f549fe48..51b28d09fa 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) } - CTxMemPool testPool(CFeeRate(0)); + CTxMemPool testPool; // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); @@ -118,7 +118,7 @@ void CheckSort(CTxMemPool &pool, std::vector<std::string> &sortedOrder) BOOST_AUTO_TEST_CASE(MempoolIndexingTest) { - CTxMemPool pool(CFeeRate(0)); + CTxMemPool pool; TestMemPoolEntryHelper entry; /* 3rd highest fee */ @@ -126,28 +126,28 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; - pool.addUnchecked(tx1.GetHash(), entry.Fee(10000LL).Priority(10.0).FromTx(tx1)); + pool.addUnchecked(tx1.GetHash(), entry.Fee(10000LL).FromTx(tx1)); /* highest fee */ CMutableTransaction tx2 = CMutableTransaction(); tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; - pool.addUnchecked(tx2.GetHash(), entry.Fee(20000LL).Priority(9.0).FromTx(tx2)); + pool.addUnchecked(tx2.GetHash(), entry.Fee(20000LL).FromTx(tx2)); /* lowest fee */ CMutableTransaction tx3 = CMutableTransaction(); tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx3.vout[0].nValue = 5 * COIN; - pool.addUnchecked(tx3.GetHash(), entry.Fee(0LL).Priority(100.0).FromTx(tx3)); + pool.addUnchecked(tx3.GetHash(), entry.Fee(0LL).FromTx(tx3)); /* 2nd highest fee */ CMutableTransaction tx4 = CMutableTransaction(); tx4.vout.resize(1); tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx4.vout[0].nValue = 6 * COIN; - pool.addUnchecked(tx4.GetHash(), entry.Fee(15000LL).Priority(1.0).FromTx(tx4)); + pool.addUnchecked(tx4.GetHash(), entry.Fee(15000LL).FromTx(tx4)); /* equal fee rate to tx1, but newer */ CMutableTransaction tx5 = CMutableTransaction(); @@ -155,7 +155,6 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx5.vout[0].nValue = 11 * COIN; entry.nTime = 1; - entry.dPriority = 10.0; pool.addUnchecked(tx5.GetHash(), entry.Fee(10000LL).FromTx(tx5)); BOOST_CHECK_EQUAL(pool.size(), 5); @@ -320,7 +319,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) { - CTxMemPool pool(CFeeRate(0)); + CTxMemPool pool; TestMemPoolEntryHelper entry; /* 3rd highest fee */ @@ -328,14 +327,14 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; - pool.addUnchecked(tx1.GetHash(), entry.Fee(10000LL).Priority(10.0).FromTx(tx1)); + pool.addUnchecked(tx1.GetHash(), entry.Fee(10000LL).FromTx(tx1)); /* highest fee */ CMutableTransaction tx2 = CMutableTransaction(); tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; - pool.addUnchecked(tx2.GetHash(), entry.Fee(20000LL).Priority(9.0).FromTx(tx2)); + pool.addUnchecked(tx2.GetHash(), entry.Fee(20000LL).FromTx(tx2)); uint64_t tx2Size = GetVirtualTransactionSize(tx2); /* lowest fee */ @@ -343,14 +342,14 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx3.vout[0].nValue = 5 * COIN; - pool.addUnchecked(tx3.GetHash(), entry.Fee(0LL).Priority(100.0).FromTx(tx3)); + pool.addUnchecked(tx3.GetHash(), entry.Fee(0LL).FromTx(tx3)); /* 2nd highest fee */ CMutableTransaction tx4 = CMutableTransaction(); tx4.vout.resize(1); tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx4.vout[0].nValue = 6 * COIN; - pool.addUnchecked(tx4.GetHash(), entry.Fee(15000LL).Priority(1.0).FromTx(tx4)); + pool.addUnchecked(tx4.GetHash(), entry.Fee(15000LL).FromTx(tx4)); /* equal fee rate to tx1, but newer */ CMutableTransaction tx5 = CMutableTransaction(); @@ -408,7 +407,6 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) /* set the fee to just below tx2's feerate when including ancestor */ CAmount fee = (20000/tx2Size)*(tx7Size + tx6Size) - 1; - //CTxMemPoolEntry entry7(tx7, fee, 2, 10.0, 1, true); pool.addUnchecked(tx7.GetHash(), entry.Fee(fee).FromTx(tx7)); BOOST_CHECK_EQUAL(pool.size(), 7); sortedOrder.insert(sortedOrder.begin()+1, tx7.GetHash().ToString()); @@ -432,9 +430,8 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) { - CTxMemPool pool(CFeeRate(1000)); + CTxMemPool pool; TestMemPoolEntryHelper entry; - entry.dPriority = 10.0; CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); @@ -442,7 +439,7 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; - pool.addUnchecked(tx1.GetHash(), entry.Fee(10000LL).FromTx(tx1, &pool)); + pool.addUnchecked(tx1.GetHash(), entry.Fee(10000LL).FromTx(tx1)); CMutableTransaction tx2 = CMutableTransaction(); tx2.vin.resize(1); @@ -450,7 +447,7 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; tx2.vout[0].nValue = 10 * COIN; - pool.addUnchecked(tx2.GetHash(), entry.Fee(5000LL).FromTx(tx2, &pool)); + pool.addUnchecked(tx2.GetHash(), entry.Fee(5000LL).FromTx(tx2)); pool.TrimToSize(pool.DynamicMemoryUsage()); // should do nothing BOOST_CHECK(pool.exists(tx1.GetHash())); @@ -460,7 +457,7 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) BOOST_CHECK(pool.exists(tx1.GetHash())); BOOST_CHECK(!pool.exists(tx2.GetHash())); - pool.addUnchecked(tx2.GetHash(), entry.FromTx(tx2, &pool)); + pool.addUnchecked(tx2.GetHash(), entry.FromTx(tx2)); CMutableTransaction tx3 = CMutableTransaction(); tx3.vin.resize(1); tx3.vin[0].prevout = COutPoint(tx2.GetHash(), 0); @@ -468,7 +465,7 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; tx3.vout[0].nValue = 10 * COIN; - pool.addUnchecked(tx3.GetHash(), entry.Fee(20000LL).FromTx(tx3, &pool)); + pool.addUnchecked(tx3.GetHash(), entry.Fee(20000LL).FromTx(tx3)); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // tx3 should pay for tx2 (CPFP) BOOST_CHECK(!pool.exists(tx1.GetHash())); @@ -531,10 +528,10 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; - pool.addUnchecked(tx4.GetHash(), entry.Fee(7000LL).FromTx(tx4, &pool)); - pool.addUnchecked(tx5.GetHash(), entry.Fee(1000LL).FromTx(tx5, &pool)); - pool.addUnchecked(tx6.GetHash(), entry.Fee(1100LL).FromTx(tx6, &pool)); - pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7, &pool)); + pool.addUnchecked(tx4.GetHash(), entry.Fee(7000LL).FromTx(tx4)); + pool.addUnchecked(tx5.GetHash(), entry.Fee(1000LL).FromTx(tx5)); + pool.addUnchecked(tx6.GetHash(), entry.Fee(1100LL).FromTx(tx6)); + pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7)); // we only require this remove, at max, 2 txn, because its not clear what we're really optimizing for aside from that pool.TrimToSize(pool.DynamicMemoryUsage() - 1); @@ -543,8 +540,8 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) BOOST_CHECK(!pool.exists(tx7.GetHash())); if (!pool.exists(tx5.GetHash())) - pool.addUnchecked(tx5.GetHash(), entry.Fee(1000LL).FromTx(tx5, &pool)); - pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7, &pool)); + pool.addUnchecked(tx5.GetHash(), entry.Fee(1000LL).FromTx(tx5)); + pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7)); pool.TrimToSize(pool.DynamicMemoryUsage() / 2); // should maximize mempool size by only removing 5/7 BOOST_CHECK(pool.exists(tx4.GetHash())); @@ -552,8 +549,8 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) BOOST_CHECK(pool.exists(tx6.GetHash())); BOOST_CHECK(!pool.exists(tx7.GetHash())); - pool.addUnchecked(tx5.GetHash(), entry.Fee(1000LL).FromTx(tx5, &pool)); - pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7, &pool)); + pool.addUnchecked(tx5.GetHash(), entry.Fee(1000LL).FromTx(tx5)); + pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7)); std::vector<CTransactionRef> vtx; SetMockTime(42); diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index af02d67f74..f2f06fa8e2 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -118,7 +118,7 @@ BOOST_AUTO_TEST_CASE(merkle_test) // If no mutation was done (once for every ntx value), try up to 16 branches. if (mutate == 0) { for (int loop = 0; loop < std::min(ntx, 16); loop++) { - // If ntx <= 16, try all branches. Otherise, try 16 random ones. + // If ntx <= 16, try all branches. Otherwise, try 16 random ones. int mtx = loop; if (ntx > 16) { mtx = insecure_rand() % ntx; diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index f856d8a91a..a40060e657 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -6,6 +6,7 @@ #include "coins.h" #include "consensus/consensus.h" #include "consensus/merkle.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "validation.h" #include "miner.h" @@ -27,6 +28,15 @@ BOOST_FIXTURE_TEST_SUITE(miner_tests, TestingSetup) static CFeeRate blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); +static BlockAssembler AssemblerForTest(const CChainParams& params) { + BlockAssembler::Options options; + + options.nBlockMaxWeight = MAX_BLOCK_WEIGHT; + options.nBlockMaxSize = MAX_BLOCK_SERIALIZED_SIZE; + options.blockMinFeeRate = blockMinFeeRate; + return BlockAssembler(params, options); +} + static struct { unsigned char extranonce; @@ -79,7 +89,6 @@ bool TestSequenceLocks(const CTransaction &tx, int flags) // Test suite for ancestor feerate transaction selection. // Implemented as an additional function, rather than a separate test case, // to allow reusing the blockchain created in CreateNewBlock_validity. -// Note that this test assumes blockprioritysize is 0. void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, std::vector<CTransactionRef>& txFirst) { // Test the ancestor feerate transaction selection. @@ -110,7 +119,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, uint256 hashHighFeeTx = tx.GetHash(); mempool.addUnchecked(hashHighFeeTx, entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashParentTx); BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashHighFeeTx); BOOST_CHECK(pblocktemplate->block.vtx[3]->GetHash() == hashMediumFeeTx); @@ -130,7 +139,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, tx.vout[0].nValue = 5000000000LL - 1000 - 50000 - feeToUse; uint256 hashLowFeeTx = tx.GetHash(); mempool.addUnchecked(hashLowFeeTx, entry.Fee(feeToUse).FromTx(tx)); - pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); // Verify that the free tx and the low fee tx didn't get selected for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeTx); @@ -144,7 +153,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee hashLowFeeTx = tx.GetHash(); mempool.addUnchecked(hashLowFeeTx, entry.Fee(feeToUse+2).FromTx(tx)); - pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashFreeTx); BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashLowFeeTx); @@ -165,7 +174,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, tx.vout[0].nValue = 5000000000LL - 100000000 - feeToUse; uint256 hashLowFeeTx2 = tx.GetHash(); mempool.addUnchecked(hashLowFeeTx2, entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); - pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); // Verify that this tx isn't selected. for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { @@ -178,7 +187,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, tx.vin[0].prevout.n = 1; tx.vout[0].nValue = 100000000 - 10000; // 10k satoshi fee mempool.addUnchecked(tx.GetHash(), entry.Fee(10000).FromTx(tx)); - pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); } @@ -186,7 +195,8 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) { // Note that by default, these tests run with size accounting enabled. - const CChainParams& chainparams = Params(CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const CChainParams& chainparams = *chainParams; CScript scriptPubKey = CScript() << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f") << OP_CHECKSIG; std::unique_ptr<CBlockTemplate> pblocktemplate; CMutableTransaction tx,tx2; @@ -194,14 +204,13 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) uint256 hash; TestMemPoolEntryHelper entry; entry.nFee = 11; - entry.dPriority = 111.0; entry.nHeight = 11; LOCK(cs_main); fCheckpointsEnabled = false; // Simple block creation, nothing special yet: - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // We can't make transactions until we have inputs // Therefore, load 100 blocks :) @@ -232,7 +241,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) } // Just to make sure we can still make simple blocks - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); const CAmount BLOCKSUBSIDY = 50*COIN; const CAmount LOWFEE = CENT; @@ -256,7 +265,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); tx.vin[0].prevout.hash = txFirst[0]->GetHash(); @@ -270,7 +279,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); mempool.clear(); // block size > limit @@ -290,16 +299,16 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); mempool.clear(); // orphan in mempool, template creation fails hash = tx.GetHash(); mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); - BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); - // child with higher priority than parent + // child with higher feerate than parent tx.vin[0].scriptSig = CScript() << OP_1; tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = BLOCKSUBSIDY-HIGHFEE; @@ -313,7 +322,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].nValue = tx.vout[0].nValue+BLOCKSUBSIDY-HIGHERFEE; //First txn output + fresh coinbase - new txn fee hash = tx.GetHash(); mempool.addUnchecked(hash, entry.Fee(HIGHERFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); mempool.clear(); // coinbase in mempool, template creation fails @@ -324,7 +333,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) hash = tx.GetHash(); // give it a fee so it'll get mined mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); // invalid (pre-p2sh) txn in mempool, template creation fails @@ -341,7 +350,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); // double spend txn pair in mempool, template creation fails @@ -354,7 +363,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].scriptPubKey = CScript() << OP_2; hash = tx.GetHash(); mempool.addUnchecked(hash, entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); // subsidy changing @@ -370,7 +379,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) next->BuildSkip(); chainActive.SetTip(next); } - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // Extend to a 210000-long block chain. while (chainActive.Tip()->nHeight < 210000) { CBlockIndex* prev = chainActive.Tip(); @@ -382,7 +391,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) next->BuildSkip(); chainActive.SetTip(next); } - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // Delete the dummy blocks again. while (chainActive.Tip()->nHeight > nHeight) { CBlockIndex* del = chainActive.Tip(); @@ -468,7 +477,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | 1; BOOST_CHECK(!TestSequenceLocks(tx, flags)); // Sequence locks fail - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // None of the of the absolute height/time locked tx should have made // it into the template because we still check IsFinalTx in CreateNewBlock, @@ -481,7 +490,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) chainActive.Tip()->nHeight++; SetMockTime(chainActive.Tip()->GetMedianTimePast() + 1); - BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 5); chainActive.Tip()->nHeight--; diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b9ed4952bb..66354699b2 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -11,6 +11,7 @@ #include "net.h" #include "netbase.h" #include "chainparams.h" +#include "util.h" class CAddrManSerializationMock : public CAddrMan { @@ -72,6 +73,18 @@ CDataStream AddrmanToStream(CAddrManSerializationMock& _addrman) BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(cnode_listen_port) +{ + // test default + unsigned short port = GetListenPort(); + BOOST_CHECK(port == Params().GetDefaultPort()); + // test set port + unsigned short altPort = 12345; + SoftSetArg("-port", std::to_string(altPort)); + port = GetListenPort(); + BOOST_CHECK(port == altPort); +} + BOOST_AUTO_TEST_CASE(caddrdb_read) { CAddrManUncorrupted addrmanUncorrupted; @@ -162,12 +175,12 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) bool fInboundIn = false; // Test that fFeeler is false by default. - std::unique_ptr<CNode> pnode1(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, pszDest, fInboundIn)); + std::unique_ptr<CNode> pnode1(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, CAddress(), pszDest, fInboundIn)); BOOST_CHECK(pnode1->fInbound == false); BOOST_CHECK(pnode1->fFeeler == false); fInboundIn = true; - std::unique_ptr<CNode> pnode2(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, pszDest, fInboundIn)); + std::unique_ptr<CNode> pnode2(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, CAddress(), pszDest, fInboundIn)); BOOST_CHECK(pnode2->fInbound == true); BOOST_CHECK(pnode2->fFeeler == false); } diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 0c060801bc..6bfd315647 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -16,7 +16,8 @@ BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { - CTxMemPool mpool(CFeeRate(1000)); + CBlockPolicyEstimator feeEst; + CTxMemPool mpool(&feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); @@ -49,14 +50,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) int blocknum = 0; // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold + // At a decay .9952 and 4 fee transactions per block + // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold while (blocknum < 200) { for (int j = 0; j < 10; j++) { // For each fee for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; // make transaction unique uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Height(blocknum).FromTx(tx)); txHashes[j].push_back(hash); } } @@ -74,20 +75,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, ++blocknum); block.clear(); - if (blocknum == 30) { - // At this point we should need to combine 5 buckets to get enough data points - // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around - // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(2) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(3) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); - int answerFound; - BOOST_CHECK(mpool.estimateSmartFee(1, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(3, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(4, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(8, &answerFound) == mpool.estimateFee(8) && answerFound == 8); + // Check after just a few txs that combining buckets works as expected + if (blocknum == 3) { + // At this point we should need to combine 3 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() < 9*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() > 9*baseRate.GetFeePerK() - deltaFee); } } @@ -99,18 +94,19 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Second highest feerate has 100% chance of being included by 2 blocks, // so estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10;i++) { - origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); if (i > 2) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } int mult = 11-i; - if (i > 1) { + if (i % 2 == 0) { //At scale 2, test logic is only correct for even targets BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); } - else { - BOOST_CHECK(origFeeEst[i-1] == CFeeRate(0).GetFeePerK()); - } + } + // Fill out rest of the original estimates + for (int i = 10; i <= 48; i++) { + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -118,10 +114,10 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) while (blocknum < 250) mpool.removeForBlock(block, ++blocknum); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } @@ -132,17 +128,15 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Height(blocknum).FromTx(tx)); txHashes[j].push_back(hash); } } mpool.removeForBlock(block, ++blocknum); } - int answerFound; for (int i = 1; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(0) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimateSmartFee(i, &answerFound).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } // Mine all those transactions @@ -155,21 +149,21 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265); + mpool.removeForBlock(block, 266); block.clear(); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } - // Mine 200 more blocks where everything is mined every block + // Mine 400 more blocks where everything is mined every block // Estimates should be below original estimates - while (blocknum < 465) { + while (blocknum < 665) { for (int j = 0; j < 10; j++) { // For each fee multiple for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Height(blocknum).FromTx(tx)); CTransactionRef ptx = mpool.get(hash); if (ptx) block.push_back(ptx); @@ -179,21 +173,19 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); block.clear(); } - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); - for (int i = 2; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); + for (int i = 2; i < 9; i++) { // At 9, the original estimate was already at the bottom (b/c scale = 2) + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee - // and that estimateSmartPriority returns essentially an infinite value - mpool.addUnchecked(tx.GetHash(), entry.Fee(feeV[5]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(tx.GetHash(), entry.Fee(feeV[5]).Time(GetTime()).Height(blocknum).FromTx(tx)); // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[5] mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartPriority(i) == INF_PRIORITY); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateRawFee(i, 0.85, FeeEstimateHorizon::MED_HALFLIFE).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 4ca6f1caf0..3b79f8000d 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -16,69 +16,59 @@ BOOST_FIXTURE_TEST_SUITE(pow_tests, BasicTestingSetup) /* Test calculation of next difficulty target with no constraints applying */ BOOST_AUTO_TEST_CASE(get_next_work) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1261130161; // Block #30240 CBlockIndex pindexLast; pindexLast.nHeight = 32255; pindexLast.nTime = 1262152739; // Block #32255 pindexLast.nBits = 0x1d00ffff; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00d86a); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00d86a); } /* Test the constraint on the upper bound for next work */ BOOST_AUTO_TEST_CASE(get_next_work_pow_limit) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1231006505; // Block #0 CBlockIndex pindexLast; pindexLast.nHeight = 2015; pindexLast.nTime = 1233061996; // Block #2015 pindexLast.nBits = 0x1d00ffff; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00ffff); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00ffff); } /* Test the constraint on the lower bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_lower_limit_actual) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1279008237; // Block #66528 CBlockIndex pindexLast; pindexLast.nHeight = 68543; pindexLast.nTime = 1279297671; // Block #68543 pindexLast.nBits = 0x1c05a3f4; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1c0168fd); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1c0168fd); } /* Test the constraint on the upper bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_upper_limit_actual) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1263163443; // NOTE: Not an actual block time CBlockIndex pindexLast; pindexLast.nHeight = 46367; pindexLast.nTime = 1269211443; // Block #46367 pindexLast.nBits = 0x1c387f6f; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00e1fd); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00e1fd); } BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); std::vector<CBlockIndex> blocks(10000); for (int i = 0; i < 10000; i++) { blocks[i].pprev = i ? &blocks[i - 1] : NULL; blocks[i].nHeight = i; - blocks[i].nTime = 1269211443 + i * params.nPowTargetSpacing; + blocks[i].nTime = 1269211443 + i * chainParams->GetConsensus().nPowTargetSpacing; blocks[i].nBits = 0x207fffff; /* target 0x7fffff000... */ blocks[i].nChainWork = i ? blocks[i - 1].nChainWork + GetBlockProof(blocks[i - 1]) : arith_uint256(0); } @@ -88,7 +78,7 @@ BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) CBlockIndex *p2 = &blocks[GetRand(10000)]; CBlockIndex *p3 = &blocks[GetRand(10000)]; - int64_t tdiff = GetBlockProofEquivalentTime(*p1, *p2, *p3, params); + int64_t tdiff = GetBlockProofEquivalentTime(*p1, *p2, *p3, chainParams->GetConsensus()); BOOST_CHECK_EQUAL(tdiff, p1->GetBlockTime() - p2->GetBlockTime()); } } diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index bd8a7819a4..cfed5e347e 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -28,6 +28,7 @@ class prevector_tester { typedef typename pretype::size_type Size; bool passed = true; FastRandomContext rand_cache; + uint256 rand_seed; template <typename A, typename B> @@ -183,13 +184,12 @@ public: } ~prevector_tester() { - BOOST_CHECK_MESSAGE(passed, "insecure_rand_Rz: " - << rand_cache.Rz - << ", insecure_rand_Rw: " - << rand_cache.Rw); + BOOST_CHECK_MESSAGE(passed, "insecure_rand: " + rand_seed.ToString()); } + prevector_tester() { seed_insecure_rand(); + rand_seed = insecure_rand_seed; rand_cache = insecure_rand_ctx; } }; diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp new file mode 100644 index 0000000000..8596734226 --- /dev/null +++ b/src/test/random_tests.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "random.h" + +#include "test/test_bitcoin.h" + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(random_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(osrandom_tests) +{ + BOOST_CHECK(Random_SanityCheck()); +} + +BOOST_AUTO_TEST_CASE(fastrandom_tests) +{ + // Check that deterministic FastRandomContexts are deterministic + FastRandomContext ctx1(true); + FastRandomContext ctx2(true); + + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + BOOST_CHECK_EQUAL(ctx1.randbits(7), ctx2.randbits(7)); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + + // Check that a nondeterministic ones are not + FastRandomContext ctx3; + FastRandomContext ctx4; + BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal +} + +BOOST_AUTO_TEST_CASE(fastrandom_randbits) +{ + FastRandomContext ctx1; + FastRandomContext ctx2; + for (int bits = 0; bits < 63; ++bits) { + for (int j = 0; j < 1000; ++j) { + uint64_t rangebits = ctx1.randbits(bits); + BOOST_CHECK_EQUAL(rangebits >> bits, 0); + uint64_t range = ((uint64_t)1) << bits | rangebits; + uint64_t rand = ctx2.randrange(range); + BOOST_CHECK(rand < range); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 399bdbc811..7cf2a744ea 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -276,7 +276,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK(banned_until.get_int64() > now); BOOST_CHECK(banned_until.get_int64()-now <= 200); - // must throw an exception because 127.0.0.1 is in already banned suubnet range + // must throw an exception because 127.0.0.1 is in already banned subnet range BOOST_CHECK_THROW(r = CallRPC(std::string("setban 127.0.0.1 add")), std::runtime_error); BOOST_CHECK_NO_THROW(CallRPC(std::string("setban 127.0.0.0/24 remove"))); diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index f8fd8cc30c..0789b2e80c 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "consensus/tx_verify.h" #include "core_io.h" #include "key.h" #include "keystore.h" @@ -111,7 +112,8 @@ BOOST_AUTO_TEST_CASE(sign) { CScript sigSave = txTo[i].vin[0].scriptSig; txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; - bool sigOK = CScriptCheck(CCoins(txFrom, 0), txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); + const CTxOut& output = txFrom.vout[txTo[i].vin[0].prevout.n]; + bool sigOK = CScriptCheck(output.scriptPubKey, output.nValue, txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); if (i == j) BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); else @@ -315,7 +317,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txFrom.vout[6].scriptPubKey = GetScriptForDestination(CScriptID(twentySigops)); txFrom.vout[6].nValue = 6000; - coins.ModifyCoins(txFrom.GetHash())->FromTx(txFrom, 0); + AddCoins(coins, txFrom, 0); CMutableTransaction txTo; txTo.vout.resize(1); diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 32184165f7..343c645cb1 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -32,7 +32,7 @@ // Uncomment if you want to output updated JSON tests. // #define UPDATE_JSON_TESTS -static const unsigned int flags = SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC; +static const unsigned int gFlags = SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC; unsigned int ParseScriptFlags(std::string strFlags); std::string FormatScriptFlags(unsigned int flags); @@ -455,11 +455,6 @@ public: { return comment; } - - const CScript& GetScriptPubKey() - { - return creditTx->vout[0].scriptPubKey; - } }; std::string JSONPrettyPrint(const UniValue& univalue) @@ -1070,18 +1065,18 @@ BOOST_AUTO_TEST_CASE(script_CHECKMULTISIG12) CMutableTransaction txTo12 = BuildSpendingTransaction(CScript(), CScriptWitness(), txFrom12); CScript goodsig1 = sign_multisig(scriptPubKey12, key1, txTo12); - BOOST_CHECK(VerifyScript(goodsig1, scriptPubKey12, NULL, flags, MutableTransactionSignatureChecker(&txTo12, 0, txFrom12.vout[0].nValue), &err)); + BOOST_CHECK(VerifyScript(goodsig1, scriptPubKey12, NULL, gFlags, MutableTransactionSignatureChecker(&txTo12, 0, txFrom12.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err)); txTo12.vout[0].nValue = 2; - BOOST_CHECK(!VerifyScript(goodsig1, scriptPubKey12, NULL, flags, MutableTransactionSignatureChecker(&txTo12, 0, txFrom12.vout[0].nValue), &err)); + BOOST_CHECK(!VerifyScript(goodsig1, scriptPubKey12, NULL, gFlags, MutableTransactionSignatureChecker(&txTo12, 0, txFrom12.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EVAL_FALSE, ScriptErrorString(err)); CScript goodsig2 = sign_multisig(scriptPubKey12, key2, txTo12); - BOOST_CHECK(VerifyScript(goodsig2, scriptPubKey12, NULL, flags, MutableTransactionSignatureChecker(&txTo12, 0, txFrom12.vout[0].nValue), &err)); + BOOST_CHECK(VerifyScript(goodsig2, scriptPubKey12, NULL, gFlags, MutableTransactionSignatureChecker(&txTo12, 0, txFrom12.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err)); CScript badsig1 = sign_multisig(scriptPubKey12, key3, txTo12); - BOOST_CHECK(!VerifyScript(badsig1, scriptPubKey12, NULL, flags, MutableTransactionSignatureChecker(&txTo12, 0, txFrom12.vout[0].nValue), &err)); + BOOST_CHECK(!VerifyScript(badsig1, scriptPubKey12, NULL, gFlags, MutableTransactionSignatureChecker(&txTo12, 0, txFrom12.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EVAL_FALSE, ScriptErrorString(err)); } @@ -1103,54 +1098,54 @@ BOOST_AUTO_TEST_CASE(script_CHECKMULTISIG23) std::vector<CKey> keys; keys.push_back(key1); keys.push_back(key2); CScript goodsig1 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(VerifyScript(goodsig1, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(VerifyScript(goodsig1, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err)); keys.clear(); keys.push_back(key1); keys.push_back(key3); CScript goodsig2 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(VerifyScript(goodsig2, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(VerifyScript(goodsig2, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err)); keys.clear(); keys.push_back(key2); keys.push_back(key3); CScript goodsig3 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(VerifyScript(goodsig3, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(VerifyScript(goodsig3, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err)); keys.clear(); keys.push_back(key2); keys.push_back(key2); // Can't re-use sig CScript badsig1 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(!VerifyScript(badsig1, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(!VerifyScript(badsig1, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EVAL_FALSE, ScriptErrorString(err)); keys.clear(); keys.push_back(key2); keys.push_back(key1); // sigs must be in correct order CScript badsig2 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(!VerifyScript(badsig2, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(!VerifyScript(badsig2, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EVAL_FALSE, ScriptErrorString(err)); keys.clear(); keys.push_back(key3); keys.push_back(key2); // sigs must be in correct order CScript badsig3 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(!VerifyScript(badsig3, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(!VerifyScript(badsig3, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EVAL_FALSE, ScriptErrorString(err)); keys.clear(); keys.push_back(key4); keys.push_back(key2); // sigs must match pubkeys CScript badsig4 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(!VerifyScript(badsig4, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(!VerifyScript(badsig4, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EVAL_FALSE, ScriptErrorString(err)); keys.clear(); keys.push_back(key1); keys.push_back(key4); // sigs must match pubkeys CScript badsig5 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(!VerifyScript(badsig5, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(!VerifyScript(badsig5, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EVAL_FALSE, ScriptErrorString(err)); keys.clear(); // Must have signatures CScript badsig6 = sign_multisig(scriptPubKey23, keys, txTo23); - BOOST_CHECK(!VerifyScript(badsig6, scriptPubKey23, NULL, flags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); + BOOST_CHECK(!VerifyScript(badsig6, scriptPubKey23, NULL, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_INVALID_STACK_OPERATION, ScriptErrorString(err)); } diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 5279cb243b..2f7c22084e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "data/sighash.json.h" #include "hash.h" -#include "validation.h" // For CheckTransaction #include "script/interpreter.h" #include "script/script.h" #include "serialize.h" diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 13d8911f03..4e117448fe 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "validation.h" +#include "consensus/tx_verify.h" #include "pubkey.h" #include "key.h" #include "script/script.h" @@ -102,7 +102,7 @@ void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CMutableT spendingTx.vout[0].nValue = 1; spendingTx.vout[0].scriptPubKey = CScript(); - coins.ModifyCoins(creationTx.GetHash())->FromTx(creationTx, 0); + AddCoins(coins, creationTx, 0); } BOOST_AUTO_TEST_CASE(GetTxSigOpCost) diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 51fc6ae0ba..cb625bda11 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -2,13 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#define BOOST_TEST_MODULE Bitcoin Test Suite - #include "test_bitcoin.h" #include "chainparams.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "fs.h" #include "key.h" #include "validation.h" #include "miner.h" @@ -26,12 +25,10 @@ #include <memory> -#include <boost/filesystem.hpp> -#include <boost/test/unit_test.hpp> #include <boost/thread.hpp> -std::unique_ptr<CConnman> g_connman; -FastRandomContext insecure_rand_ctx(true); +uint256 insecure_rand_seed = GetRandHash(); +FastRandomContext insecure_rand_ctx(insecure_rand_seed); extern bool fPrintToConsole; extern void noui_connect(); @@ -63,17 +60,20 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha RegisterAllCoreRPCCommands(tableRPC); ClearDatadirCache(); pathTemp = GetTempPath() / strprintf("test_bitcoin_%lu_%i", (unsigned long)GetTime(), (int)(GetRand(100000))); - boost::filesystem::create_directories(pathTemp); + fs::create_directories(pathTemp); ForceSetArg("-datadir", pathTemp.string()); mempool.setSanityCheck(1.0); pblocktree = new CBlockTreeDB(1 << 20, true); pcoinsdbview = new CCoinsViewDB(1 << 23, true); pcoinsTip = new CCoinsViewCache(pcoinsdbview); - InitBlockIndex(chainparams); + if (!InitBlockIndex(chainparams)) { + throw std::runtime_error("InitBlockIndex failed."); + } { CValidationState state; - bool ok = ActivateBestChain(state, chainparams); - BOOST_CHECK(ok); + if (!ActivateBestChain(state, chainparams)) { + throw std::runtime_error("ActivateBestChain failed."); + } } nScriptCheckThreads = 3; for (int i=0; i < nScriptCheckThreads-1; i++) @@ -92,7 +92,7 @@ TestingSetup::~TestingSetup() delete pcoinsTip; delete pcoinsdbview; delete pblocktree; - boost::filesystem::remove_all(pathTemp); + fs::remove_all(pathTemp); } TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) @@ -141,30 +141,12 @@ TestChain100Setup::~TestChain100Setup() } -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction &tx, CTxMemPool *pool) { +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction &tx) { CTransaction txn(tx); - return FromTx(txn, pool); -} - -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransaction &txn, CTxMemPool *pool) { - // Hack to assume either it's completely dependent on other mempool txs or not at all - CAmount inChainValue = pool && pool->HasNoInputsOf(txn) ? txn.GetValueOut() : 0; - - return CTxMemPoolEntry(MakeTransactionRef(txn), nFee, nTime, dPriority, nHeight, - inChainValue, spendsCoinbase, sigOpCost, lp); -} - -void Shutdown(void* parg) -{ - exit(EXIT_SUCCESS); -} - -void StartShutdown() -{ - exit(EXIT_SUCCESS); + return FromTx(txn); } -bool ShutdownRequested() -{ - return false; +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransaction &txn) { + return CTxMemPoolEntry(MakeTransactionRef(txn), nFee, nTime, nHeight, + spendsCoinbase, sigOpCost, lp); } diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index 5ef6fa764f..60a86d8c48 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -6,12 +6,12 @@ #define BITCOIN_TEST_TEST_BITCOIN_H #include "chainparamsbase.h" +#include "fs.h" #include "key.h" #include "pubkey.h" #include "txdb.h" #include "txmempool.h" -#include <boost/filesystem.hpp> #include <boost/thread.hpp> /** Basic testing setup. @@ -30,7 +30,7 @@ struct BasicTestingSetup { class CConnman; struct TestingSetup: public BasicTestingSetup { CCoinsViewDB *pcoinsdbview; - boost::filesystem::path pathTemp; + fs::path pathTemp; boost::thread_group threadGroup; CConnman* connman; @@ -61,30 +61,27 @@ struct TestChain100Setup : public TestingSetup { }; class CTxMemPoolEntry; -class CTxMemPool; struct TestMemPoolEntryHelper { // Default values CAmount nFee; int64_t nTime; - double dPriority; unsigned int nHeight; bool spendsCoinbase; unsigned int sigOpCost; LockPoints lp; TestMemPoolEntryHelper() : - nFee(0), nTime(0), dPriority(0.0), nHeight(1), + nFee(0), nTime(0), nHeight(1), spendsCoinbase(false), sigOpCost(4) { } - CTxMemPoolEntry FromTx(const CMutableTransaction &tx, CTxMemPool *pool = NULL); - CTxMemPoolEntry FromTx(const CTransaction &tx, CTxMemPool *pool = NULL); + CTxMemPoolEntry FromTx(const CMutableTransaction &tx); + CTxMemPoolEntry FromTx(const CTransaction &tx); // Change the default value TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } TestMemPoolEntryHelper &Time(int64_t _time) { nTime = _time; return *this; } - TestMemPoolEntryHelper &Priority(double _priority) { dPriority = _priority; return *this; } TestMemPoolEntryHelper &Height(unsigned int _height) { nHeight = _height; return *this; } TestMemPoolEntryHelper &SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } TestMemPoolEntryHelper &SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index c4983f6f5c..de14251601 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -59,9 +59,8 @@ bool read_stdin(std::vector<char> &data) { return length==0; } -int main(int argc, char **argv) +int do_fuzz() { - ECCVerifyHandle globalVerifyHandle; std::vector<char> buffer; if (!read_stdin(buffer)) return 0; @@ -169,8 +168,8 @@ int main(int argc, char **argv) { try { - CCoins block; - ds >> block; + Coin coin; + ds >> coin; } catch (const std::ios_base::failure& e) {return 0;} break; } @@ -256,3 +255,23 @@ int main(int argc, char **argv) return 0; } +int main(int argc, char **argv) +{ + ECCVerifyHandle globalVerifyHandle; +#ifdef __AFL_INIT + // Enable AFL deferred forkserver mode. Requires compilation using + // afl-clang-fast++. See fuzzing.md for details. + __AFL_INIT(); +#endif + +#ifdef __AFL_LOOP + // Enable AFL persistent mode. Requires compilation using afl-clang-fast++. + // See fuzzing.md for details. + while (__AFL_LOOP(1000)) { + do_fuzz(); + } + return 0; +#else + return do_fuzz(); +#endif +} diff --git a/src/test/test_bitcoin_main.cpp b/src/test/test_bitcoin_main.cpp new file mode 100644 index 0000000000..34beef5539 --- /dev/null +++ b/src/test/test_bitcoin_main.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2011-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#define BOOST_TEST_MODULE Bitcoin Test Suite + +#include "net.h" + +#include <boost/test/unit_test.hpp> + +std::unique_ptr<CConnman> g_connman; + +void Shutdown(void* parg) +{ + exit(EXIT_SUCCESS); +} + +void StartShutdown() +{ + exit(EXIT_SUCCESS); +} + +bool ShutdownRequested() +{ + return false; +} diff --git a/src/test/test_random.h b/src/test/test_random.h index 4a1637ac72..318c44df4d 100644 --- a/src/test/test_random.h +++ b/src/test/test_random.h @@ -8,11 +8,17 @@ #include "random.h" +extern uint256 insecure_rand_seed; extern FastRandomContext insecure_rand_ctx; static inline void seed_insecure_rand(bool fDeterministic = false) { - insecure_rand_ctx = FastRandomContext(fDeterministic); + if (fDeterministic) { + insecure_rand_seed = uint256(); + } else { + insecure_rand_seed = GetRandHash(); + } + insecure_rand_ctx = FastRandomContext(insecure_rand_seed); } static inline uint32_t insecure_rand(void) diff --git a/src/test/testutil.cpp b/src/test/testutil.cpp index e6d8622979..591d0bf302 100644 --- a/src/test/testutil.cpp +++ b/src/test/testutil.cpp @@ -8,8 +8,8 @@ #include <shlobj.h> #endif -#include <boost/filesystem.hpp> +#include "fs.h" -boost::filesystem::path GetTempPath() { - return boost::filesystem::temp_directory_path(); +fs::path GetTempPath() { + return fs::temp_directory_path(); } diff --git a/src/test/testutil.h b/src/test/testutil.h index 5875dc50e6..cbe784d640 100644 --- a/src/test/testutil.h +++ b/src/test/testutil.h @@ -8,8 +8,8 @@ #ifndef BITCOIN_TEST_TESTUTIL_H #define BITCOIN_TEST_TESTUTIL_H -#include <boost/filesystem/path.hpp> +#include "fs.h" -boost::filesystem::path GetTempPath(); +fs::path GetTempPath(); #endif // BITCOIN_TEST_TESTUTIL_H diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp new file mode 100644 index 0000000000..b7affaacde --- /dev/null +++ b/src/test/torcontrol_tests.cpp @@ -0,0 +1,199 @@ +// Copyright (c) 2017 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include "test/test_bitcoin.h" +#include "torcontrol.cpp" + +#include <boost/test/unit_test.hpp> + + +BOOST_FIXTURE_TEST_SUITE(torcontrol_tests, BasicTestingSetup) + +void CheckSplitTorReplyLine(std::string input, std::string command, std::string args) +{ + BOOST_TEST_MESSAGE(std::string("CheckSplitTorReplyLine(") + input + ")"); + auto ret = SplitTorReplyLine(input); + BOOST_CHECK_EQUAL(ret.first, command); + BOOST_CHECK_EQUAL(ret.second, args); +} + +BOOST_AUTO_TEST_CASE(util_SplitTorReplyLine) +{ + // Data we should receive during normal usage + CheckSplitTorReplyLine( + "PROTOCOLINFO PIVERSION", + "PROTOCOLINFO", "PIVERSION"); + CheckSplitTorReplyLine( + "AUTH METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\"", + "AUTH", "METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\""); + CheckSplitTorReplyLine( + "AUTH METHODS=NULL", + "AUTH", "METHODS=NULL"); + CheckSplitTorReplyLine( + "AUTH METHODS=HASHEDPASSWORD", + "AUTH", "METHODS=HASHEDPASSWORD"); + CheckSplitTorReplyLine( + "VERSION Tor=\"0.2.9.8 (git-a0df013ea241b026)\"", + "VERSION", "Tor=\"0.2.9.8 (git-a0df013ea241b026)\""); + CheckSplitTorReplyLine( + "AUTHCHALLENGE SERVERHASH=aaaa SERVERNONCE=bbbb", + "AUTHCHALLENGE", "SERVERHASH=aaaa SERVERNONCE=bbbb"); + + // Other valid inputs + CheckSplitTorReplyLine("COMMAND", "COMMAND", ""); + CheckSplitTorReplyLine("COMMAND SOME ARGS", "COMMAND", "SOME ARGS"); + + // These inputs are valid because PROTOCOLINFO accepts an OtherLine that is + // just an OptArguments, which enables multiple spaces to be present + // between the command and arguments. + CheckSplitTorReplyLine("COMMAND ARGS", "COMMAND", " ARGS"); + CheckSplitTorReplyLine("COMMAND EVEN+more ARGS", "COMMAND", " EVEN+more ARGS"); +} + +void CheckParseTorReplyMapping(std::string input, std::map<std::string,std::string> expected) +{ + BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(") + input + ")"); + auto ret = ParseTorReplyMapping(input); + BOOST_CHECK_EQUAL(ret.size(), expected.size()); + auto r_it = ret.begin(); + auto e_it = expected.begin(); + while (r_it != ret.end() && e_it != expected.end()) { + BOOST_CHECK_EQUAL(r_it->first, e_it->first); + BOOST_CHECK_EQUAL(r_it->second, e_it->second); + r_it++; + e_it++; + } +} + +BOOST_AUTO_TEST_CASE(util_ParseTorReplyMapping) +{ + // Data we should receive during normal usage + CheckParseTorReplyMapping( + "METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\"", { + {"METHODS", "COOKIE,SAFECOOKIE"}, + {"COOKIEFILE", "/home/x/.tor/control_auth_cookie"}, + }); + CheckParseTorReplyMapping( + "METHODS=NULL", { + {"METHODS", "NULL"}, + }); + CheckParseTorReplyMapping( + "METHODS=HASHEDPASSWORD", { + {"METHODS", "HASHEDPASSWORD"}, + }); + CheckParseTorReplyMapping( + "Tor=\"0.2.9.8 (git-a0df013ea241b026)\"", { + {"Tor", "0.2.9.8 (git-a0df013ea241b026)"}, + }); + CheckParseTorReplyMapping( + "SERVERHASH=aaaa SERVERNONCE=bbbb", { + {"SERVERHASH", "aaaa"}, + {"SERVERNONCE", "bbbb"}, + }); + CheckParseTorReplyMapping( + "ServiceID=exampleonion1234", { + {"ServiceID", "exampleonion1234"}, + }); + CheckParseTorReplyMapping( + "PrivateKey=RSA1024:BLOB", { + {"PrivateKey", "RSA1024:BLOB"}, + }); + CheckParseTorReplyMapping( + "ClientAuth=bob:BLOB", { + {"ClientAuth", "bob:BLOB"}, + }); + + // Other valid inputs + CheckParseTorReplyMapping( + "Foo=Bar=Baz Spam=Eggs", { + {"Foo", "Bar=Baz"}, + {"Spam", "Eggs"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar=Baz\"", { + {"Foo", "Bar=Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar Baz\"", { + {"Foo", "Bar Baz"}, + }); + + // Escapes + CheckParseTorReplyMapping( + "Foo=\"Bar\\ Baz\"", { + {"Foo", "Bar Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\Baz\"", { + {"Foo", "BarBaz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\@Baz\"", { + {"Foo", "Bar@Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\\"Baz\" Spam=\"\\\"Eggs\\\"\"", { + {"Foo", "Bar\"Baz"}, + {"Spam", "\"Eggs\""}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\\\Baz\"", { + {"Foo", "Bar\\Baz"}, + }); + + // C escapes + CheckParseTorReplyMapping( + "Foo=\"Bar\\nBaz\\t\" Spam=\"\\rEggs\" Octals=\"\\1a\\11\\17\\18\\81\\377\\378\\400\\2222\" Final=Check", { + {"Foo", "Bar\nBaz\t"}, + {"Spam", "\rEggs"}, + {"Octals", "\1a\11\17\1" "881\377\37" "8\40" "0\222" "2"}, + {"Final", "Check"}, + }); + CheckParseTorReplyMapping( + "Valid=Mapping Escaped=\"Escape\\\\\"", { + {"Valid", "Mapping"}, + {"Escaped", "Escape\\"}, + }); + CheckParseTorReplyMapping( + "Valid=Mapping Bare=\"Escape\\\"", {}); + CheckParseTorReplyMapping( + "OneOctal=\"OneEnd\\1\" TwoOctal=\"TwoEnd\\11\"", { + {"OneOctal", "OneEnd\1"}, + {"TwoOctal", "TwoEnd\11"}, + }); + + // Special handling for null case + // (needed because string comparison reads the null as end-of-string) + BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(Null=\"\\0\")")); + auto ret = ParseTorReplyMapping("Null=\"\\0\""); + BOOST_CHECK_EQUAL(ret.size(), 1); + auto r_it = ret.begin(); + BOOST_CHECK_EQUAL(r_it->first, "Null"); + BOOST_CHECK_EQUAL(r_it->second.size(), 1); + BOOST_CHECK_EQUAL(r_it->second[0], '\0'); + + // A more complex valid grammar. PROTOCOLINFO accepts a VersionLine that + // takes a key=value pair followed by an OptArguments, making this valid. + // Because an OptArguments contains no semantic data, there is no point in + // parsing it. + CheckParseTorReplyMapping( + "SOME=args,here MORE optional=arguments here", { + {"SOME", "args,here"}, + }); + + // Inputs that are effectively invalid under the target grammar. + // PROTOCOLINFO accepts an OtherLine that is just an OptArguments, which + // would make these inputs valid. However, + // - This parser is never used in that situation, because the + // SplitTorReplyLine parser enables OtherLine to be skipped. + // - Even if these were valid, an OptArguments contains no semantic data, + // so there is no point in parsing it. + CheckParseTorReplyMapping("ARGS", {}); + CheckParseTorReplyMapping("MORE ARGS", {}); + CheckParseTorReplyMapping("MORE ARGS", {}); + CheckParseTorReplyMapping("EVEN more=ARGS", {}); + CheckParseTorReplyMapping("EVEN+more ARGS", {}); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 374423179c..5c7516fbf1 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -8,11 +8,12 @@ #include "clientversion.h" #include "checkqueue.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "core_io.h" #include "key.h" #include "keystore.h" -#include "validation.h" // For CheckTransaction +#include "validation.h" #include "policy/policy.h" #include "script/script.h" #include "script/sign.h" @@ -189,7 +190,9 @@ BOOST_AUTO_TEST_CASE(tx_invalid) // verifyFlags is a comma separated list of script verification flags to apply, or "NONE" UniValue tests = read_json(std::string(json_tests::tx_invalid, json_tests::tx_invalid + sizeof(json_tests::tx_invalid))); - ScriptError err; + // Initialize to SCRIPT_ERR_OK. The tests expect err to be changed to a + // value other than SCRIPT_ERR_OK. + ScriptError err = SCRIPT_ERR_OK; for (unsigned int idx = 0; idx < tests.size(); idx++) { UniValue test = tests[idx]; std::string strTest = test.write(); @@ -304,14 +307,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50*CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21*CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22*CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } @@ -467,19 +470,20 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { for (int i=0; i<20; i++) threadGroup.create_thread(boost::bind(&CCheckQueue<CScriptCheck>::Thread, boost::ref(scriptcheckqueue))); - CCoins coins; - coins.nVersion = 1; - coins.fCoinBase = false; + std::vector<Coin> coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { - CTxOut txout; - txout.nValue = 1000; - txout.scriptPubKey = scriptPubKey; - coins.vout.push_back(txout); + Coin coin; + coin.nHeight = 1; + coin.fCoinBase = false; + coin.out.nValue = 1000; + coin.out.scriptPubKey = scriptPubKey; + coins.emplace_back(std::move(coin)); } for(uint32_t i = 0; i < mtx.vin.size(); i++) { std::vector<CScriptCheck> vChecks; - CScriptCheck check(coins, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); + const CTxOut& output = coins[tx.vin[i].prevout.n].out; + CScriptCheck check(output.scriptPubKey, output.nValue, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); vChecks.push_back(CScriptCheck()); check.swap(vChecks.back()); control.Add(vChecks); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 641655621c..10330c0c23 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -17,8 +17,6 @@ #include <boost/test/unit_test.hpp> -extern std::map<std::string, std::string> mapArgs; - BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(util_criticalsection) @@ -100,52 +98,67 @@ BOOST_AUTO_TEST_CASE(util_DateTimeStrFormat) BOOST_CHECK_EQUAL(DateTimeStrFormat("%a, %d %b %Y %H:%M:%S +0000", 1317425777), "Fri, 30 Sep 2011 23:36:17 +0000"); } +class TestArgsManager : public ArgsManager +{ +public: + std::map<std::string, std::string>& GetMapArgs() + { + return mapArgs; + }; + const std::map<std::string, std::vector<std::string> >& GetMapMultiArgs() + { + return mapMultiArgs; + }; +}; + BOOST_AUTO_TEST_CASE(util_ParseParameters) { + TestArgsManager testArgs; const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; - ParseParameters(0, (char**)argv_test); - BOOST_CHECK(mapArgs.empty() && mapMultiArgs.empty()); + testArgs.ParseParameters(0, (char**)argv_test); + BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - ParseParameters(1, (char**)argv_test); - BOOST_CHECK(mapArgs.empty() && mapMultiArgs.empty()); + testArgs.ParseParameters(1, (char**)argv_test); + BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - ParseParameters(5, (char**)argv_test); + testArgs.ParseParameters(5, (char**)argv_test); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) - BOOST_CHECK(mapArgs.size() == 3 && mapMultiArgs.size() == 3); - BOOST_CHECK(IsArgSet("-a") && IsArgSet("-b") && IsArgSet("-ccc") - && !IsArgSet("f") && !IsArgSet("-d")); - BOOST_CHECK(mapMultiArgs.count("-a") && mapMultiArgs.count("-b") && mapMultiArgs.count("-ccc") - && !mapMultiArgs.count("f") && !mapMultiArgs.count("-d")); - - BOOST_CHECK(mapArgs["-a"] == "" && mapArgs["-ccc"] == "multiple"); - BOOST_CHECK(mapMultiArgs.at("-ccc").size() == 2); + BOOST_CHECK(testArgs.GetMapArgs().size() == 3 && testArgs.GetMapMultiArgs().size() == 3); + BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") + && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); + BOOST_CHECK(testArgs.GetMapMultiArgs().count("-a") && testArgs.GetMapMultiArgs().count("-b") && testArgs.GetMapMultiArgs().count("-ccc") + && !testArgs.GetMapMultiArgs().count("f") && !testArgs.GetMapMultiArgs().count("-d")); + + BOOST_CHECK(testArgs.GetMapArgs()["-a"] == "" && testArgs.GetMapArgs()["-ccc"] == "multiple"); + BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); } BOOST_AUTO_TEST_CASE(util_GetArg) { - mapArgs.clear(); - mapArgs["strtest1"] = "string..."; + TestArgsManager testArgs; + testArgs.GetMapArgs().clear(); + testArgs.GetMapArgs()["strtest1"] = "string..."; // strtest2 undefined on purpose - mapArgs["inttest1"] = "12345"; - mapArgs["inttest2"] = "81985529216486895"; + testArgs.GetMapArgs()["inttest1"] = "12345"; + testArgs.GetMapArgs()["inttest2"] = "81985529216486895"; // inttest3 undefined on purpose - mapArgs["booltest1"] = ""; + testArgs.GetMapArgs()["booltest1"] = ""; // booltest2 undefined on purpose - mapArgs["booltest3"] = "0"; - mapArgs["booltest4"] = "1"; - - BOOST_CHECK_EQUAL(GetArg("strtest1", "default"), "string..."); - BOOST_CHECK_EQUAL(GetArg("strtest2", "default"), "default"); - BOOST_CHECK_EQUAL(GetArg("inttest1", -1), 12345); - BOOST_CHECK_EQUAL(GetArg("inttest2", -1), 81985529216486895LL); - BOOST_CHECK_EQUAL(GetArg("inttest3", -1), -1); - BOOST_CHECK_EQUAL(GetBoolArg("booltest1", false), true); - BOOST_CHECK_EQUAL(GetBoolArg("booltest2", false), false); - BOOST_CHECK_EQUAL(GetBoolArg("booltest3", false), false); - BOOST_CHECK_EQUAL(GetBoolArg("booltest4", false), true); + testArgs.GetMapArgs()["booltest3"] = "0"; + testArgs.GetMapArgs()["booltest4"] = "1"; + + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest1", -1), 12345); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest2", -1), 81985529216486895LL); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest3", -1), -1); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest1", false), true); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest2", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest3", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest4", false), true); } BOOST_AUTO_TEST_CASE(util_FormatMoney) @@ -247,7 +260,7 @@ BOOST_AUTO_TEST_CASE(util_seed_insecure_rand) for (int mod=2;mod<11;mod++) { int mask = 1; - // Really rough binomal confidence approximation. + // Really rough binomial confidence approximation. int err = 30*10000./mod*sqrt((1./mod*(1-1./mod))/10000.); //mask is 2^ceil(log2(mod))-1 while(mask<mod-1)mask=(mask<<1)+1; @@ -321,7 +334,7 @@ BOOST_AUTO_TEST_CASE(test_ParseInt32) BOOST_CHECK(ParseInt32("1234", &n) && n == 1234); BOOST_CHECK(ParseInt32("01234", &n) && n == 1234); // no octal BOOST_CHECK(ParseInt32("2147483647", &n) && n == 2147483647); - BOOST_CHECK(ParseInt32("-2147483648", &n) && n == -2147483648); + BOOST_CHECK(ParseInt32("-2147483648", &n) && n == (-2147483647 - 1)); // (-2147483647 - 1) equals INT_MIN BOOST_CHECK(ParseInt32("-1234", &n) && n == -1234); // Invalid values BOOST_CHECK(!ParseInt32("", &n)); diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index e2b5573abd..79405ec4d1 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -209,7 +209,8 @@ BOOST_AUTO_TEST_CASE(versionbits_test) } // Sanity checks of version bit deployments - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const Consensus::Params &mainnetParams = chainParams->GetConsensus(); for (int i=0; i<(int) Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { uint32_t bitmask = VersionBitsMask(mainnetParams, (Consensus::DeploymentPos)i); // Make sure that no deployment tries to set an invalid bit. @@ -235,7 +236,8 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) { // Check that ComputeBlockVersion will set the appropriate bit correctly // on mainnet. - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const Consensus::Params &mainnetParams = chainParams->GetConsensus(); // Use the TESTDUMMY deployment for testing purposes. int64_t bit = mainnetParams.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].bit; diff --git a/src/timedata.cpp b/src/timedata.cpp index 2ff6437c73..ec74912703 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -58,7 +58,7 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) // Add data static CMedianFilter<int64_t> vTimeOffsets(BITCOIN_TIMEDATA_MAX_SAMPLES, 0); vTimeOffsets.input(nOffsetSample); - LogPrint("net","added time data, samples %d, offset %+d (%+d minutes)\n", vTimeOffsets.size(), nOffsetSample, nOffsetSample/60); + LogPrint(BCLog::NET,"added time data, samples %d, offset %+d (%+d minutes)\n", vTimeOffsets.size(), nOffsetSample, nOffsetSample/60); // There is a known issue here (see issue #4521): // @@ -108,11 +108,14 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) } } } - - BOOST_FOREACH(int64_t n, vSorted) - LogPrint("net", "%+d ", n); - LogPrint("net", "| "); - - LogPrint("net", "nTimeOffset = %+d (%+d minutes)\n", nTimeOffset, nTimeOffset/60); + + if (LogAcceptCategory(BCLog::NET)) { + BOOST_FOREACH(int64_t n, vSorted) { + LogPrint(BCLog::NET, "%+d ", n); + } + LogPrint(BCLog::NET, "| "); + + LogPrint(BCLog::NET, "nTimeOffset = %+d (%+d minutes)\n", nTimeOffset, nTimeOffset/60); + } } } diff --git a/src/timedata.h b/src/timedata.h index 4a2eab3487..bc5451b19b 100644 --- a/src/timedata.h +++ b/src/timedata.h @@ -27,9 +27,9 @@ private: unsigned int nSize; public: - CMedianFilter(unsigned int size, T initial_value) : nSize(size) + CMedianFilter(unsigned int _size, T initial_value) : nSize(_size) { - vValues.reserve(size); + vValues.reserve(_size); vValues.push_back(initial_value); vSorted = vValues; } @@ -48,14 +48,14 @@ public: T median() const { - int size = vSorted.size(); - assert(size > 0); - if (size & 1) // Odd number of elements + int vSortedSize = vSorted.size(); + assert(vSortedSize > 0); + if (vSortedSize & 1) // Odd number of elements { - return vSorted[size / 2]; + return vSorted[vSortedSize / 2]; } else // Even number of elements { - return (vSorted[size / 2 - 1] + vSorted[size / 2]) / 2; + return (vSorted[vSortedSize / 2 - 1] + vSorted[vSortedSize / 2]) / 2; } } diff --git a/src/tinyformat.h b/src/tinyformat.h index 17f0360c42..5022d46809 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -123,7 +123,7 @@ namespace tinyformat {} namespace tfm = tinyformat; // Error handling; calls assert() by default. -#define TINYFORMAT_ERROR(reasonString) throw std::runtime_error(reasonString) +#define TINYFORMAT_ERROR(reasonString) throw tinyformat::format_error(reasonString) // Define for C++11 variadic templates which make the code shorter & more // general. If you don't define this, C++11 support is autodetected below. @@ -164,6 +164,13 @@ namespace tfm = tinyformat; namespace tinyformat { +class format_error: public std::runtime_error +{ +public: + format_error(const std::string &what): std::runtime_error(what) { + } +}; + //------------------------------------------------------------------------------ namespace detail { diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index c49c5d9eb2..8a37139f1d 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,7 +15,6 @@ #include <set> #include <stdlib.h> -#include <boost/function.hpp> #include <boost/bind.hpp> #include <boost/signals2/signal.hpp> #include <boost/foreach.hpp> @@ -73,8 +73,8 @@ public: class TorControlConnection { public: - typedef boost::function<void(TorControlConnection&)> ConnectionCB; - typedef boost::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB; + typedef std::function<void(TorControlConnection&)> ConnectionCB; + typedef std::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB; /** Create a new TorControlConnection. */ @@ -105,9 +105,9 @@ public: boost::signals2::signal<void(TorControlConnection &,const TorControlReply &)> async_handler; private: /** Callback when ready for use */ - boost::function<void(TorControlConnection&)> connected; + std::function<void(TorControlConnection&)> connected; /** Callback when connection lost */ - boost::function<void(TorControlConnection&)> disconnected; + std::function<void(TorControlConnection&)> disconnected; /** Libevent event base */ struct event_base *base; /** Connection to control socket */ @@ -163,7 +163,7 @@ void TorControlConnection::readcb(struct bufferevent *bev, void *ctx) self->reply_handlers.front()(*self, self->message); self->reply_handlers.pop_front(); } else { - LogPrint("tor", "tor: Received unexpected sync reply %i\n", self->message.code); + LogPrint(BCLog::TOR, "tor: Received unexpected sync reply %i\n", self->message.code); } } self->message.Clear(); @@ -182,13 +182,14 @@ void TorControlConnection::eventcb(struct bufferevent *bev, short what, void *ct { TorControlConnection *self = (TorControlConnection*)ctx; if (what & BEV_EVENT_CONNECTED) { - LogPrint("tor", "tor: Successfully connected!\n"); + LogPrint(BCLog::TOR, "tor: Successfully connected!\n"); self->connected(*self); } else if (what & (BEV_EVENT_EOF|BEV_EVENT_ERROR)) { - if (what & BEV_EVENT_ERROR) - LogPrint("tor", "tor: Error connecting to Tor control socket\n"); - else - LogPrint("tor", "tor: End of stream\n"); + if (what & BEV_EVENT_ERROR) { + LogPrint(BCLog::TOR, "tor: Error connecting to Tor control socket\n"); + } else { + LogPrint(BCLog::TOR, "tor: End of stream\n"); + } self->Disconnect(); self->disconnected(*self); } @@ -249,6 +250,8 @@ bool TorControlConnection::Command(const std::string &cmd, const ReplyHandlerCB& /* Split reply line in the form 'AUTH METHODS=...' into a type * 'AUTH' and arguments 'METHODS=...'. + * Grammar is implicitly defined in https://spec.torproject.org/control-spec by + * the server reply formats for PROTOCOLINFO (S3.21) and AUTHCHALLENGE (S3.24). */ static std::pair<std::string,std::string> SplitTorReplyLine(const std::string &s) { @@ -264,6 +267,10 @@ static std::pair<std::string,std::string> SplitTorReplyLine(const std::string &s } /** Parse reply arguments in the form 'METHODS=COOKIE,SAFECOOKIE COOKIEFILE=".../control_auth_cookie"'. + * Returns a map of keys to values, or an empty map if there was an error. + * Grammar is implicitly defined in https://spec.torproject.org/control-spec by + * the server reply formats for PROTOCOLINFO (S3.21), AUTHCHALLENGE (S3.24), + * and ADD_ONION (S3.27). See also sections 2.1 and 2.3. */ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) { @@ -271,28 +278,74 @@ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string size_t ptr=0; while (ptr < s.size()) { std::string key, value; - while (ptr < s.size() && s[ptr] != '=') { + while (ptr < s.size() && s[ptr] != '=' && s[ptr] != ' ') { key.push_back(s[ptr]); ++ptr; } if (ptr == s.size()) // unexpected end of line return std::map<std::string,std::string>(); + if (s[ptr] == ' ') // The remaining string is an OptArguments + break; ++ptr; // skip '=' if (ptr < s.size() && s[ptr] == '"') { // Quoted string - ++ptr; // skip '=' + ++ptr; // skip opening '"' bool escape_next = false; - while (ptr < s.size() && (!escape_next && s[ptr] != '"')) { - escape_next = (s[ptr] == '\\'); + while (ptr < s.size() && (escape_next || s[ptr] != '"')) { + // Repeated backslashes must be interpreted as pairs + escape_next = (s[ptr] == '\\' && !escape_next); value.push_back(s[ptr]); ++ptr; } if (ptr == s.size()) // unexpected end of line return std::map<std::string,std::string>(); ++ptr; // skip closing '"' - /* TODO: unescape value - according to the spec this depends on the - * context, some strings use C-LogPrintf style escape codes, some - * don't. So may be better handled at the call site. + /** + * Unescape value. Per https://spec.torproject.org/control-spec section 2.1.1: + * + * For future-proofing, controller implementors MAY use the following + * rules to be compatible with buggy Tor implementations and with + * future ones that implement the spec as intended: + * + * Read \n \t \r and \0 ... \377 as C escapes. + * Treat a backslash followed by any other character as that character. */ + std::string escaped_value; + for (size_t i = 0; i < value.size(); ++i) { + if (value[i] == '\\') { + // This will always be valid, because if the QuotedString + // ended in an odd number of backslashes, then the parser + // would already have returned above, due to a missing + // terminating double-quote. + ++i; + if (value[i] == 'n') { + escaped_value.push_back('\n'); + } else if (value[i] == 't') { + escaped_value.push_back('\t'); + } else if (value[i] == 'r') { + escaped_value.push_back('\r'); + } else if ('0' <= value[i] && value[i] <= '7') { + size_t j; + // Octal escape sequences have a limit of three octal digits, + // but terminate at the first character that is not a valid + // octal digit if encountered sooner. + for (j = 1; j < 3 && (i+j) < value.size() && '0' <= value[i+j] && value[i+j] <= '7'; ++j) {} + // Tor restricts first digit to 0-3 for three-digit octals. + // A leading digit of 4-7 would therefore be interpreted as + // a two-digit octal. + if (j == 3 && value[i] > '3') { + j--; + } + escaped_value.push_back(strtol(value.substr(i, j).c_str(), NULL, 8)); + // Account for automatic incrementing at loop end + i += j - 1; + } else { + escaped_value.push_back(value[i]); + } + } else { + escaped_value.push_back(value[i]); + } + } + value = escaped_value; } else { // Unquoted value. Note that values can contain '=' at will, just no spaces while (ptr < s.size() && s[ptr] != ' ') { value.push_back(s[ptr]); @@ -313,15 +366,19 @@ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string * @param maxsize Puts a maximum size limit on the file that is read. If the file is larger than this, truncated data * (with len > maxsize) will be returned. */ -static std::pair<bool,std::string> ReadBinaryFile(const std::string &filename, size_t maxsize=std::numeric_limits<size_t>::max()) +static std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxsize=std::numeric_limits<size_t>::max()) { - FILE *f = fopen(filename.c_str(), "rb"); + FILE *f = fsbridge::fopen(filename, "rb"); if (f == NULL) return std::make_pair(false,""); std::string retval; char buffer[128]; size_t n; while ((n=fread(buffer, 1, sizeof(buffer), f)) > 0) { + // Check for reading errors so we don't return any data if we couldn't + // read the entire file (or up to maxsize) + if (ferror(f)) + return std::make_pair(false,""); retval.append(buffer, buffer+n); if (retval.size() > maxsize) break; @@ -333,9 +390,9 @@ static std::pair<bool,std::string> ReadBinaryFile(const std::string &filename, s /** Write contents of std::string to a file. * @return true on success. */ -static bool WriteBinaryFile(const std::string &filename, const std::string &data) +static bool WriteBinaryFile(const fs::path &filename, const std::string &data) { - FILE *f = fopen(filename.c_str(), "wb"); + FILE *f = fsbridge::fopen(filename, "wb"); if (f == NULL) return false; if (fwrite(data.data(), 1, data.size(), f) != data.size()) { @@ -358,7 +415,7 @@ public: ~TorController(); /** Get name fo file to store private key in */ - std::string GetPrivateKeyFile(); + fs::path GetPrivateKeyFile(); /** Reconnect, after getting disconnected */ void Reconnect(); @@ -410,7 +467,7 @@ TorController::TorController(struct event_base* _base, const std::string& _targe // Read service private key if cached std::pair<bool,std::string> pkf = ReadBinaryFile(GetPrivateKeyFile()); if (pkf.first) { - LogPrint("tor", "tor: Reading cached private key from %s\n", GetPrivateKeyFile()); + LogPrint(BCLog::TOR, "tor: Reading cached private key from %s\n", GetPrivateKeyFile().string()); private_key = pkf.second; } } @@ -429,7 +486,7 @@ TorController::~TorController() void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlReply& reply) { if (reply.code == 250) { - LogPrint("tor", "tor: ADD_ONION successful\n"); + LogPrint(BCLog::TOR, "tor: ADD_ONION successful\n"); BOOST_FOREACH(const std::string &s, reply.lines) { std::map<std::string,std::string> m = ParseTorReplyMapping(s); std::map<std::string,std::string>::iterator i; @@ -438,12 +495,19 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe if ((i = m.find("PrivateKey")) != m.end()) private_key = i->second; } + if (service_id.empty()) { + LogPrintf("tor: Error parsing ADD_ONION parameters:\n"); + for (const std::string &s : reply.lines) { + LogPrintf(" %s\n", SanitizeString(s)); + } + return; + } service = LookupNumeric(std::string(service_id+".onion").c_str(), GetListenPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { - LogPrint("tor", "tor: Cached service private key to %s\n", GetPrivateKeyFile()); + LogPrint(BCLog::TOR, "tor: Cached service private key to %s\n", GetPrivateKeyFile().string()); } else { - LogPrintf("tor: Error writing service private key to %s\n", GetPrivateKeyFile()); + LogPrintf("tor: Error writing service private key to %s\n", GetPrivateKeyFile().string()); } AddLocal(service, LOCAL_MANUAL); // ... onion requested - keep connection open @@ -457,7 +521,7 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply& reply) { if (reply.code == 250) { - LogPrint("tor", "tor: Authentication successful\n"); + LogPrint(BCLog::TOR, "tor: Authentication successful\n"); // Now that we know Tor is running setup the proxy for onion addresses // if -onion isn't set to something else. @@ -511,13 +575,17 @@ static std::vector<uint8_t> ComputeResponse(const std::string &key, const std::v void TorController::authchallenge_cb(TorControlConnection& _conn, const TorControlReply& reply) { if (reply.code == 250) { - LogPrint("tor", "tor: SAFECOOKIE authentication challenge successful\n"); + LogPrint(BCLog::TOR, "tor: SAFECOOKIE authentication challenge successful\n"); std::pair<std::string,std::string> l = SplitTorReplyLine(reply.lines[0]); if (l.first == "AUTHCHALLENGE") { std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); + if (m.empty()) { + LogPrintf("tor: Error parsing AUTHCHALLENGE parameters: %s\n", SanitizeString(l.second)); + return; + } std::vector<uint8_t> serverHash = ParseHex(m["SERVERHASH"]); std::vector<uint8_t> serverNonce = ParseHex(m["SERVERNONCE"]); - LogPrint("tor", "tor: AUTHCHALLENGE ServerHash %s ServerNonce %s\n", HexStr(serverHash), HexStr(serverNonce)); + LogPrint(BCLog::TOR, "tor: AUTHCHALLENGE ServerHash %s ServerNonce %s\n", HexStr(serverHash), HexStr(serverNonce)); if (serverNonce.size() != 32) { LogPrintf("tor: ServerNonce is not 32 bytes, as required by spec\n"); return; @@ -562,12 +630,12 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); std::map<std::string,std::string>::iterator i; if ((i = m.find("Tor")) != m.end()) { - LogPrint("tor", "tor: Connected to Tor version %s\n", i->second); + LogPrint(BCLog::TOR, "tor: Connected to Tor version %s\n", i->second); } } } BOOST_FOREACH(const std::string &s, methods) { - LogPrint("tor", "tor: Supported authentication method: %s\n", s); + LogPrint(BCLog::TOR, "tor: Supported authentication method: %s\n", s); } // Prefer NULL, otherwise SAFECOOKIE. If a password is provided, use HASHEDPASSWORD /* Authentication: @@ -577,18 +645,18 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro std::string torpassword = GetArg("-torpassword", ""); if (!torpassword.empty()) { if (methods.count("HASHEDPASSWORD")) { - LogPrint("tor", "tor: Using HASHEDPASSWORD authentication\n"); + LogPrint(BCLog::TOR, "tor: Using HASHEDPASSWORD authentication\n"); boost::replace_all(torpassword, "\"", "\\\""); _conn.Command("AUTHENTICATE \"" + torpassword + "\"", boost::bind(&TorController::auth_cb, this, _1, _2)); } else { LogPrintf("tor: Password provided with -torpassword, but HASHEDPASSWORD authentication is not available\n"); } } else if (methods.count("NULL")) { - LogPrint("tor", "tor: Using NULL authentication\n"); + LogPrint(BCLog::TOR, "tor: Using NULL authentication\n"); _conn.Command("AUTHENTICATE", boost::bind(&TorController::auth_cb, this, _1, _2)); } else if (methods.count("SAFECOOKIE")) { // Cookie: hexdump -e '32/1 "%02x""\n"' ~/.tor/control_auth_cookie - LogPrint("tor", "tor: Using SAFECOOKIE authentication, reading cookie authentication from %s\n", cookiefile); + LogPrint(BCLog::TOR, "tor: Using SAFECOOKIE authentication, reading cookie authentication from %s\n", cookiefile); std::pair<bool,std::string> status_cookie = ReadBinaryFile(cookiefile, TOR_COOKIE_SIZE); if (status_cookie.first && status_cookie.second.size() == TOR_COOKIE_SIZE) { // _conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), boost::bind(&TorController::auth_cb, this, _1, _2)); @@ -630,7 +698,7 @@ void TorController::disconnected_cb(TorControlConnection& _conn) if (!reconnect) return; - LogPrint("tor", "tor: Not connected to Tor control port %s, trying to reconnect\n", target); + LogPrint(BCLog::TOR, "tor: Not connected to Tor control port %s, trying to reconnect\n", target); // Single-shot timer for reconnect. Use exponential backoff. struct timeval time = MillisToTimeval(int64_t(reconnect_timeout * 1000.0)); @@ -650,9 +718,9 @@ void TorController::Reconnect() } } -std::string TorController::GetPrivateKeyFile() +fs::path TorController::GetPrivateKeyFile() { - return (GetDataDir() / "onion_private_key").string(); + return GetDataDir() / "onion_private_key"; } void TorController::reconnect_cb(evutil_socket_t fd, short what, void *arg) @@ -662,26 +730,26 @@ void TorController::reconnect_cb(evutil_socket_t fd, short what, void *arg) } /****** Thread ********/ -struct event_base *base; -boost::thread torControlThread; +static struct event_base *gBase; +static boost::thread torControlThread; static void TorControlThread() { - TorController ctrl(base, GetArg("-torcontrol", DEFAULT_TOR_CONTROL)); + TorController ctrl(gBase, GetArg("-torcontrol", DEFAULT_TOR_CONTROL)); - event_base_dispatch(base); + event_base_dispatch(gBase); } void StartTorControl(boost::thread_group& threadGroup, CScheduler& scheduler) { - assert(!base); + assert(!gBase); #ifdef WIN32 evthread_use_windows_threads(); #else evthread_use_pthreads(); #endif - base = event_base_new(); - if (!base) { + gBase = event_base_new(); + if (!gBase) { LogPrintf("tor: Unable to create event_base\n"); return; } @@ -691,18 +759,18 @@ void StartTorControl(boost::thread_group& threadGroup, CScheduler& scheduler) void InterruptTorControl() { - if (base) { + if (gBase) { LogPrintf("tor: Thread interrupt\n"); - event_base_loopbreak(base); + event_base_loopbreak(gBase); } } void StopTorControl() { - if (base) { + if (gBase) { torControlThread.join(); - event_base_free(base); - base = 0; + event_base_free(gBase); + gBase = 0; } } diff --git a/src/txdb.cpp b/src/txdb.cpp index 1a30bb58ad..c8f5090293 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -14,6 +14,7 @@ #include <boost/thread.hpp> +static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; @@ -24,17 +25,40 @@ static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; +namespace { + +struct CoinEntry { + COutPoint* outpoint; + char key; + CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {} + + template<typename Stream> + void Serialize(Stream &s) const { + s << key; + s << outpoint->hash; + s << VARINT(outpoint->n); + } + + template<typename Stream> + void Unserialize(Stream& s) { + s >> key; + s >> outpoint->hash; + s >> VARINT(outpoint->n); + } +}; + +} CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { } -bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { - return db.Read(std::make_pair(DB_COINS, txid), coins); +bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { + return db.Read(CoinEntry(&outpoint), coin); } -bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { - return db.Exists(std::make_pair(DB_COINS, txid)); +bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { + return db.Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { @@ -50,10 +74,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - if (it->second.coins.IsPruned()) - batch.Erase(std::make_pair(DB_COINS, it->first)); + CoinEntry entry(&it->first); + if (it->second.coin.IsSpent()) + batch.Erase(entry); else - batch.Write(std::make_pair(DB_COINS, it->first), it->second.coins); + batch.Write(entry, it->second.coin); changed++; } count++; @@ -63,8 +88,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); - return db.WriteBatch(batch); + bool ret = db.WriteBatch(batch); + LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); + return ret; +} + +size_t CCoinsViewDB::EstimateSize() const +{ + return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { @@ -96,25 +127,31 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - i->pcursor->Seek(DB_COINS); + i->pcursor->Seek(DB_COIN); // Cache key of first record - i->pcursor->GetKey(i->keyTmp); + if (i->pcursor->Valid()) { + CoinEntry entry(&i->keyTmp.second); + i->pcursor->GetKey(entry); + i->keyTmp.first = entry.key; + } else { + i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false + } return i; } -bool CCoinsViewDBCursor::GetKey(uint256 &key) const +bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key - if (keyTmp.first == DB_COINS) { + if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } -bool CCoinsViewDBCursor::GetValue(CCoins &coins) const +bool CCoinsViewDBCursor::GetValue(Coin &coin) const { - return pcursor->GetValue(coins); + return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const @@ -124,14 +161,18 @@ unsigned int CCoinsViewDBCursor::GetValueSize() const bool CCoinsViewDBCursor::Valid() const { - return keyTmp.first == DB_COINS; + return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); - if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + CoinEntry entry(&keyTmp.second); + if (!pcursor->Valid() || !pcursor->GetKey(entry)) { keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false + } else { + keyTmp.first = entry.key; + } } bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) { @@ -169,7 +210,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { return true; } -bool CBlockTreeDB::LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256&)> insertBlockIndex) +bool CBlockTreeDB::LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&)> insertBlockIndex) { std::unique_ptr<CDBIterator> pcursor(NewIterator()); @@ -211,3 +252,103 @@ bool CBlockTreeDB::LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256 return true; } + +namespace { + +//! Legacy class to deserialize pre-pertxout database entries without reindex. +class CCoins +{ +public: + //! whether transaction is a coinbase + bool fCoinBase; + + //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped + std::vector<CTxOut> vout; + + //! at which height this transaction was included in the active block chain + int nHeight; + + //! empty constructor + CCoins() : fCoinBase(false), vout(0), nHeight(0) { } + + template<typename Stream> + void Unserialize(Stream &s) { + unsigned int nCode = 0; + // version + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + // header code + ::Unserialize(s, VARINT(nCode)); + fCoinBase = nCode & 1; + std::vector<bool> vAvail(2, false); + vAvail[0] = (nCode & 2) != 0; + vAvail[1] = (nCode & 4) != 0; + unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); + // spentness bitmask + while (nMaskCode > 0) { + unsigned char chAvail = 0; + ::Unserialize(s, chAvail); + for (unsigned int p = 0; p < 8; p++) { + bool f = (chAvail & (1 << p)) != 0; + vAvail.push_back(f); + } + if (chAvail != 0) + nMaskCode--; + } + // txouts themself + vout.assign(vAvail.size(), CTxOut()); + for (unsigned int i = 0; i < vAvail.size(); i++) { + if (vAvail[i]) + ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); + } + // coinbase height + ::Unserialize(s, VARINT(nHeight)); + } +}; + +} + +/** Upgrade the database from older formats. + * + * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. + */ +bool CCoinsViewDB::Upgrade() { + std::unique_ptr<CDBIterator> pcursor(db.NewIterator()); + pcursor->Seek(std::make_pair(DB_COINS, uint256())); + if (!pcursor->Valid()) { + return true; + } + + LogPrintf("Upgrading database...\n"); + size_t batch_size = 1 << 24; + CDBBatch batch(db); + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair<unsigned char, uint256> key; + if (pcursor->GetKey(key) && key.first == DB_COINS) { + CCoins old_coins; + if (!pcursor->GetValue(old_coins)) { + return error("%s: cannot parse CCoins record", __func__); + } + COutPoint outpoint(key.second, 0); + for (size_t i = 0; i < old_coins.vout.size(); ++i) { + if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { + Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); + outpoint.n = i; + CoinEntry entry(&outpoint); + batch.Write(entry, newcoin); + } + } + batch.Erase(key); + if (batch.SizeEstimate() > batch_size) { + db.WriteBatch(batch); + batch.Clear(); + } + pcursor->Next(); + } else { + break; + } + } + db.WriteBatch(batch); + return true; +} diff --git a/src/txdb.h b/src/txdb.h index 7f5cf2b583..974dd4ebe3 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -15,14 +15,16 @@ #include <utility> #include <vector> -#include <boost/function.hpp> - class CBlockIndex; class CCoinsViewDBCursor; class uint256; +//! Compensate for extra memory peak (x1.5-x1.9) at flush time. +static constexpr int DB_PEAK_USAGE_FACTOR = 2; +//! No need to periodic flush if at least this much space still available. +static constexpr int MAX_BLOCK_COINSDB_USAGE = 10 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) -static const int64_t nDefaultDbCache = 300; +static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache (MiB) @@ -69,11 +71,15 @@ protected: public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + + //! Attempt to update from an older database format. Returns whether an error occurred. + bool Upgrade(); + size_t EstimateSize() const override; }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ @@ -82,8 +88,8 @@ class CCoinsViewDBCursor: public CCoinsViewCursor public: ~CCoinsViewDBCursor() {} - bool GetKey(uint256 &key) const; - bool GetValue(CCoins &coins) const; + bool GetKey(COutPoint &key) const; + bool GetValue(Coin &coin) const; unsigned int GetValueSize() const; bool Valid() const; @@ -93,7 +99,7 @@ private: CCoinsViewDBCursor(CDBIterator* pcursorIn, const uint256 &hashBlockIn): CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} std::unique_ptr<CDBIterator> pcursor; - std::pair<char, uint256> keyTmp; + std::pair<char, COutPoint> keyTmp; friend class CCoinsViewDB; }; @@ -116,7 +122,7 @@ public: bool WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> > &list); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); - bool LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256&)> insertBlockIndex); + bool LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&)> insertBlockIndex); }; #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 942a6fcce7..17389db9f0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -5,8 +5,8 @@ #include "txmempool.h" -#include "clientversion.h" #include "consensus/consensus.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "validation.h" #include "policy/policy.h" @@ -16,25 +16,19 @@ #include "util.h" #include "utilmoneystr.h" #include "utiltime.h" -#include "version.h" CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, double _entryPriority, unsigned int _entryHeight, - CAmount _inChainInputValue, + int64_t _nTime, unsigned int _entryHeight, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp): - tx(_tx), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), - inChainInputValue(_inChainInputValue), + tx(_tx), nFee(_nFee), nTime(_nTime), entryHeight(_entryHeight), spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nTxWeight = GetTransactionWeight(*tx); - nModSize = tx->CalculateModifiedSize(GetTxSize()); - nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); + nUsageSize = RecursiveDynamicUsage(tx); nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); nModFeesWithDescendants = nFee; - CAmount nValueIn = tx->GetValueOut()+nFee; - assert(inChainInputValue <= nValueIn); feeDelta = 0; @@ -49,16 +43,6 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other) *this = other; } -double -CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const -{ - double deltaPriority = ((double)(currentHeight-entryHeight)*inChainInputValue)/nModSize; - double dResult = entryPriority + deltaPriority; - if (dResult < 0) // This should only happen if it was called with a height below entry height - dResult = 0; - return dResult; -} - void CTxMemPoolEntry::UpdateFeeDelta(int64_t newFeeDelta) { nModFeesWithDescendants += newFeeDelta - feeDelta; @@ -123,7 +107,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendan // vHashesToUpdate is the set of transaction hashes from a disconnected block // which has been re-added to the mempool. -// for each entry, look for descendants that are outside hashesToUpdate, and +// for each entry, look for descendants that are outside vHashesToUpdate, and // add fee/size information for such descendants to the parent. // for each such descendant, also update the ancestor state to include the parent. void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashesToUpdate) @@ -348,8 +332,8 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : - nTransactionsUpdated(0) +CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), minerPolicyEstimator(estimator) { _clear(); //lock free clear @@ -357,26 +341,12 @@ CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool nCheckFrequency = 0; - - minerPolicyEstimator = new CBlockPolicyEstimator(_minReasonableRelayFee); -} - -CTxMemPool::~CTxMemPool() -{ - delete minerPolicyEstimator; } -void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) +bool CTxMemPool::isSpent(const COutPoint& outpoint) { LOCK(cs); - - auto it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); - - // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx - while (it != mapNextTx.end() && it->first->hash == hashTx) { - coins.Spend(it->first->n); // and remove those outputs from coins - it++; - } + return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const @@ -404,11 +374,11 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, // Update transaction for any feeDelta created by PrioritiseTransaction // TODO: refactor so that the fee delta is calculated before inserting // into mapTx. - std::map<uint256, std::pair<double, CAmount> >::const_iterator pos = mapDeltas.find(hash); + std::map<uint256, CAmount>::const_iterator pos = mapDeltas.find(hash); if (pos != mapDeltas.end()) { - const std::pair<double, CAmount> &deltas = pos->second; - if (deltas.second) { - mapTx.modify(newit, update_fee_delta(deltas.second)); + const CAmount &delta = pos->second; + if (delta) { + mapTx.modify(newit, update_fee_delta(delta)); } } @@ -442,7 +412,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -472,7 +442,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - minerPolicyEstimator->removeTx(hash); + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -554,9 +524,9 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; - const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); - if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < COINBASE_MATURITY)) { + const Coin &coin = pcoins->AccessCoin(txin.prevout); + if (nCheckFrequency != 0) assert(!coin.IsSpent()); + if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { txToRemove.insert(it); break; } @@ -606,7 +576,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne entries.push_back(&*i); } // Before the txs in the new block have been removed from the mempool, update policy estimates - minerPolicyEstimator->processBlock(nBlockHeight, entries); + if (minerPolicyEstimator) {minerPolicyEstimator->processBlock(nBlockHeight, entries);} for (const auto& tx : vtx) { txiter it = mapTx.find(tx->GetHash()); @@ -649,7 +619,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const if (GetRand(std::numeric_limits<uint32_t>::max()) >= nCheckFrequency) return; - LogPrint("mempool", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); + LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; uint64_t innerUsage = 0; @@ -684,8 +654,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const parentSigOpCost += it2->GetSigOpCost(); } } else { - const CCoins* coins = pcoins->AccessCoins(txin.prevout.hash); - assert(coins && coins->IsAvailable(txin.prevout.n)); + assert(pcoins->HaveCoin(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); @@ -865,71 +834,15 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const return GetInfo(i); } -CFeeRate CTxMemPool::estimateFee(int nBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateFee(nBlocks); -} -CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); -} -double CTxMemPool::estimatePriority(int nBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimatePriority(nBlocks); -} -double CTxMemPool::estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateSmartPriority(nBlocks, answerFoundAtBlocks, *this); -} - -bool -CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const -{ - try { - LOCK(cs); - fileout << 139900; // version required to read: 0.13.99 or later - fileout << CLIENT_VERSION; // version that wrote the file - minerPolicyEstimator->Write(fileout); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - -bool -CTxMemPool::ReadFeeEstimates(CAutoFile& filein) -{ - try { - int nVersionRequired, nVersionThatWrote; - filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) - return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - LOCK(cs); - minerPolicyEstimator->Read(filein, nVersionThatWrote); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - -void CTxMemPool::PrioritiseTransaction(const uint256& hash, double dPriorityDelta, const CAmount& nFeeDelta) +void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { LOCK(cs); - std::pair<double, CAmount> &deltas = mapDeltas[hash]; - deltas.first += dPriorityDelta; - deltas.second += nFeeDelta; + CAmount &delta = mapDeltas[hash]; + delta += nFeeDelta; txiter it = mapTx.find(hash); if (it != mapTx.end()) { - mapTx.modify(it, update_fee_delta(deltas.second)); + mapTx.modify(it, update_fee_delta(delta)); // Now update all ancestors' modified fees with descendants setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); @@ -938,20 +851,27 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, double dPriorityDelt BOOST_FOREACH(txiter ancestorIt, setAncestors) { mapTx.modify(ancestorIt, update_descendant_state(0, nFeeDelta, 0)); } + // Now update all descendants' modified fees with ancestors + setEntries setDescendants; + CalculateDescendants(it, setDescendants); + setDescendants.erase(it); + BOOST_FOREACH(txiter descendantIt, setDescendants) { + mapTx.modify(descendantIt, update_ancestor_state(0, nFeeDelta, 0, 0)); + } + ++nTransactionsUpdated; } } - LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", hash.ToString(), dPriorityDelta, FormatMoney(nFeeDelta)); + LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); } -void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const +void CTxMemPool::ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const { LOCK(cs); - std::map<uint256, std::pair<double, CAmount> >::const_iterator pos = mapDeltas.find(hash); + std::map<uint256, CAmount>::const_iterator pos = mapDeltas.find(hash); if (pos == mapDeltas.end()) return; - const std::pair<double, CAmount> &deltas = pos->second; - dPriorityDelta += deltas.first; - nFeeDelta += deltas.second; + const CAmount &delta = pos->second; + nFeeDelta += delta; } void CTxMemPool::ClearPrioritisation(const uint256 hash) @@ -970,20 +890,24 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } -bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { +bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. - CTransactionRef ptx = mempool.get(txid); + CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { - coins = CCoins(*ptx, MEMPOOL_HEIGHT); - return true; + if (outpoint.n < ptx->vout.size()) { + coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); + return true; + } else { + return false; + } } - return (base->GetCoins(txid, coins) && !coins.IsPruned()); + return (base->GetCoin(outpoint, coin) && !coin.IsSpent()); } -bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { - return mempool.exists(txid) || base->HaveCoins(txid); +bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { + return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { @@ -1094,7 +1018,7 @@ void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { } } -void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRemaining) { +void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining) { LOCK(cs); unsigned nTxnRemoved = 0; @@ -1125,18 +1049,18 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRe if (pvNoSpendsRemaining) { BOOST_FOREACH(const CTransaction& tx, txn) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { - if (exists(txin.prevout.hash)) - continue; - auto iter = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); - if (iter == mapNextTx.end() || iter->first->hash != txin.prevout.hash) - pvNoSpendsRemaining->push_back(txin.prevout.hash); + if (exists(txin.prevout.hash)) continue; + if (!mapNextTx.count(txin.prevout)) { + pvNoSpendsRemaining->push_back(txin.prevout); + } } } } } - if (maxFeeRateRemoved > CFeeRate(0)) - LogPrint("mempool", "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + if (maxFeeRateRemoved > CFeeRate(0)) { + LogPrint(BCLog::MEMPOOL, "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + } } bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLimit) const { @@ -1145,3 +1069,5 @@ bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLi return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } + +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index 8a8039ded1..0316b42ba2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -16,6 +16,7 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" +#include "policy/feerate.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" @@ -23,26 +24,15 @@ #include "boost/multi_index_container.hpp" #include "boost/multi_index/ordered_index.hpp" #include "boost/multi_index/hashed_index.hpp" +#include <boost/multi_index/sequenced_index.hpp> #include <boost/signals2/signal.hpp> class CAutoFile; class CBlockIndex; -inline double AllowFreeThreshold() -{ - return COIN * 144 / 250; -} - -inline bool AllowFree(double dPriority) -{ - // Large (in bytes) low-priority (new, small-coin) transactions - // need a fee. - return dPriority > AllowFreeThreshold(); -} - -/** Fake height value used in CCoins to signify they are only in the memory pool (since 0.8) */ -static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; +/** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ +static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { @@ -71,11 +61,6 @@ class CTxMemPool; * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for * all ancestors of the newly added transaction. * - * If updating the descendant state is skipped, we can mark the entry as - * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal nTxSize/ - * nFee+feeDelta. (This can potentially happen during a reorg, where we limit the - * amount of work we're willing to do to avoid consuming too much CPU.) - * */ class CTxMemPoolEntry @@ -84,12 +69,9 @@ private: CTransactionRef tx; CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) - size_t nModSize; //!< ... and modified size for priority size_t nUsageSize; //!< ... and total memory usage int64_t nTime; //!< Local time when entering the mempool - double entryPriority; //!< Priority when entering the mempool unsigned int entryHeight; //!< Chain height when entering the mempool - CAmount inChainInputValue; //!< Sum of all txin values that are already in blockchain bool spendsCoinbase; //!< keep track of transactions that spend a coinbase int64_t sigOpCost; //!< Total sigop cost int64_t feeDelta; //!< Used for determining the priority of the transaction for mining in a block @@ -97,9 +79,7 @@ private: // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these - // descendants as well. if nCountWithDescendants is 0, treat this entry as - // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be - // correct. + // descendants as well. uint64_t nCountWithDescendants; //!< number of descendant transactions uint64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) @@ -112,19 +92,14 @@ private: public: CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, double _entryPriority, unsigned int _entryHeight, - CAmount _inChainInputValue, bool spendsCoinbase, + int64_t _nTime, unsigned int _entryHeight, + bool spendsCoinbase, int64_t nSigOpsCost, LockPoints lp); CTxMemPoolEntry(const CTxMemPoolEntry& other); const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } - /** - * Fast calculation of lower bound of current priority as update - * from entry priority. Only inputs that were originally in-chain will age. - */ - double GetPriority(unsigned int currentHeight) const; const CAmount& GetFee() const { return nFee; } size_t GetTxSize() const; size_t GetTxWeight() const { return nTxWeight; } @@ -135,7 +110,7 @@ public: size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints& GetLockPoints() const { return lockPoints; } - // Adjusts the descendant state, if this entry is not dirty. + // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); @@ -211,7 +186,7 @@ private: const LockPoints& lp; }; -// extracts a TxMemPoolEntry's transaction hash +// extracts a transaction hash from CTxMempoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; @@ -219,6 +194,11 @@ struct mempoolentry_txid { return entry.GetTx().GetHash(); } + + result_type operator() (const CTransactionRef& tx) const + { + return tx->GetHash(); + } }; /** \class CompareTxMemPoolEntryByDescendantScore @@ -347,6 +327,20 @@ enum class MemPoolRemovalReason { REPLACED //! Removed for replacement }; +class SaltedTxidHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -418,20 +412,12 @@ enum class MemPoolRemovalReason { * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * - * Adding transactions from a disconnected block can be very time consuming, - * because we don't have a way to limit the number of in-mempool descendants. - * To bound CPU processing, we limit the amount of work we're willing to do - * to properly update the descendant information for a tx being added from - * a disconnected block. If we would exceed the limit, then we instead mark - * the entry as "dirty", and set the feerate for sorting purposes to be equal - * the feerate of the transaction without any descendants. - * */ class CTxMemPool { private: uint32_t nCheckFrequency; //!< Value n means that n times in 2^32 we check. - unsigned int nTransactionsUpdated; + unsigned int nTransactionsUpdated; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. @@ -512,12 +498,11 @@ private: public: indirectmap<COutPoint, const CTransaction*> mapNextTx; - std::map<uint256, std::pair<double, CAmount> > mapDeltas; + std::map<uint256, CAmount> mapDeltas; /** Create a new CTxMemPool. */ - CTxMemPool(const CFeeRate& _minReasonableRelayFee); - ~CTxMemPool(); + CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is @@ -544,7 +529,7 @@ public: void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); void queryHashes(std::vector<uint256>& vtxid); - void pruneSpent(const uint256& hash, CCoins &coins); + bool isSpent(const COutPoint& outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** @@ -554,8 +539,8 @@ public: bool HasNoInputsOf(const CTransaction& tx) const; /** Affect CreateNewBlock prioritisation of transactions */ - void PrioritiseTransaction(const uint256& hash, double dPriorityDelta, const CAmount& nFeeDelta); - void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const; + void PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta); + void ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const; void ClearPrioritisation(const uint256 hash); public: @@ -572,12 +557,12 @@ public: * new mempool entries may have children in the mempool (which is generally * not the case when otherwise adding transactions). * UpdateTransactionsFromBlock() will find child transactions and update the - * descendant state for each transaction in hashesToUpdate (excluding any - * child transactions present in hashesToUpdate, which are already accounted - * for). Note: hashesToUpdate should be the set of transactions from the + * descendant state for each transaction in vHashesToUpdate (excluding any + * child transactions present in vHashesToUpdate, which are already accounted + * for). Note: vHashesToUpdate should be the set of transactions from the * disconnected block that have been accepted back into the mempool. */ - void UpdateTransactionsFromBlock(const std::vector<uint256> &hashesToUpdate); + void UpdateTransactionsFromBlock(const std::vector<uint256> &vHashesToUpdate); /** Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) @@ -605,10 +590,10 @@ public: CFeeRate GetMinFee(size_t sizelimit) const; /** Remove transactions from the mempool until its dynamic size is <= sizelimit. - * pvNoSpendsRemaining, if set, will be populated with the list of transactions + * pvNoSpendsRemaining, if set, will be populated with the list of outpoints * which are not in mempool which no longer have any spends in this mempool. */ - void TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRemaining=NULL); + void TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining=NULL); /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ int Expire(int64_t time); @@ -634,32 +619,17 @@ public: return (mapTx.count(hash) != 0); } + bool exists(const COutPoint& outpoint) const + { + LOCK(cs); + auto it = mapTx.find(outpoint.hash); + return (it != mapTx.end() && outpoint.n < it->GetTx().vout.size()); + } + CTransactionRef get(const uint256& hash) const; TxMempoolInfo info(const uint256& hash) const; std::vector<TxMempoolInfo> infoAll() const; - /** Estimate fee rate needed to get into the next nBlocks - * If no answer can be given at nBlocks, return an estimate - * at the lowest number of blocks where one can be given - */ - CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = NULL) const; - - /** Estimate fee rate needed to get into the next nBlocks */ - CFeeRate estimateFee(int nBlocks) const; - - /** Estimate priority needed to get into the next nBlocks - * If no answer can be given at nBlocks, return an estimate - * at the lowest number of blocks where one can be given - */ - double estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks = NULL) const; - - /** Estimate priority needed to get into the next nBlocks */ - double estimatePriority(int nBlocks) const; - - /** Write/Read estimates to disk */ - bool WriteFeeEstimates(CAutoFile& fileout) const; - bool ReadFeeEstimates(CAutoFile& filein); - size_t DynamicMemoryUsage() const; boost::signals2::signal<void (CTransactionRef)> NotifyEntryAdded; @@ -715,20 +685,98 @@ protected: public: CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; }; -// We want to sort transactions by coin age priority -typedef std::pair<double, CTxMemPool::txiter> TxCoinAgePriority; +/** + * DisconnectedBlockTransactions + + * During the reorg, it's desirable to re-add previously confirmed transactions + * to the mempool, so that anything not re-confirmed in the new chain is + * available to be mined. However, it's more efficient to wait until the reorg + * is complete and process all still-unconfirmed transactions at that time, + * since we expect most confirmed transactions to (typically) still be + * confirmed in the new chain, and re-accepting to the memory pool is expensive + * (and therefore better to not do in the middle of reorg-processing). + * Instead, store the disconnected transactions (in order!) as we go, remove any + * that are included in blocks in the new chain, and then process the remaining + * still-unconfirmed transactions at the end. + */ -struct TxCoinAgePriorityCompare -{ - bool operator()(const TxCoinAgePriority& a, const TxCoinAgePriority& b) +// multi_index tag names +struct txid_index {}; +struct insertion_order {}; + +struct DisconnectedBlockTransactions { + typedef boost::multi_index_container< + CTransactionRef, + boost::multi_index::indexed_by< + // sorted by txid + boost::multi_index::hashed_unique< + boost::multi_index::tag<txid_index>, + mempoolentry_txid, + SaltedTxidHasher + >, + // sorted by order in the blockchain + boost::multi_index::sequenced< + boost::multi_index::tag<insertion_order> + > + > + > indexed_disconnected_transactions; + + // It's almost certainly a logic bug if we don't clear out queuedTx before + // destruction, as we add to it while disconnecting blocks, and then we + // need to re-process remaining transactions to ensure mempool consistency. + // For now, assert() that we've emptied out this object on destruction. + // This assert() can always be removed if the reorg-processing code were + // to be refactored such that this assumption is no longer true (for + // instance if there was some other way we cleaned up the mempool after a + // reorg, besides draining this object). + ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } + + indexed_disconnected_transactions queuedTx; + uint64_t cachedInnerUsage = 0; + + // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as + // no exact formula for boost::multi_index_contained is implemented. + size_t DynamicMemoryUsage() const { + return memusage::MallocUsage(sizeof(CTransactionRef) + 6 * sizeof(void*)) * queuedTx.size() + cachedInnerUsage; + } + + void addTransaction(const CTransactionRef& tx) + { + queuedTx.insert(tx); + cachedInnerUsage += RecursiveDynamicUsage(tx); + } + + // Remove entries based on txid_index, and update memory usage. + void removeForBlock(const std::vector<CTransactionRef>& vtx) + { + // Short-circuit in the common case of a block being added to the tip + if (queuedTx.empty()) { + return; + } + for (auto const &tx : vtx) { + auto it = queuedTx.find(tx->GetHash()); + if (it != queuedTx.end()) { + cachedInnerUsage -= RecursiveDynamicUsage(*it); + queuedTx.erase(it); + } + } + } + + // Remove an entry by insertion_order index, and update memory usage. + void removeEntry(indexed_disconnected_transactions::index<insertion_order>::type::iterator entry) + { + cachedInnerUsage -= RecursiveDynamicUsage(*entry); + queuedTx.get<insertion_order>().erase(entry); + } + + void clear() { - if (a.first == b.first) - return CompareTxMemPoolEntryByScore()(*(b.second), *(a.second)); //Reverse order to make sort less than - return a.first < b.first; + cachedInnerUsage = 0; + queuedTx.clear(); } }; diff --git a/src/undo.h b/src/undo.h index a94e31f5cc..3749d5d7a8 100644 --- a/src/undo.h +++ b/src/undo.h @@ -7,58 +7,90 @@ #define BITCOIN_UNDO_H #include "compressor.h" +#include "consensus/consensus.h" #include "primitives/transaction.h" #include "serialize.h" /** Undo information for a CTxIn * - * Contains the prevout's CTxOut being spent, and if this was the - * last output of the affected transaction, its metadata as well - * (coinbase or not, height, transaction version) + * Contains the prevout's CTxOut being spent, and its metadata as well + * (coinbase or not, height). The serialization contains a dummy value of + * zero. This is be compatible with older versions which expect to see + * the transaction version there. */ -class CTxInUndo +class TxInUndoSerializer { -public: - CTxOut txout; // the txout data before being spent - bool fCoinBase; // if the outpoint was the last unspent: whether it belonged to a coinbase - unsigned int nHeight; // if the outpoint was the last unspent: its height - int nVersion; // if the outpoint was the last unspent: its version - - CTxInUndo() : txout(), fCoinBase(false), nHeight(0), nVersion(0) {} - CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0, int nVersionIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn), nVersion(nVersionIn) { } + const Coin* txout; +public: template<typename Stream> void Serialize(Stream &s) const { - ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0))); - if (nHeight > 0) - ::Serialize(s, VARINT(this->nVersion)); - ::Serialize(s, CTxOutCompressor(REF(txout))); + ::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1 : 0))); + if (txout->nHeight > 0) { + // Required to maintain compatibility with older undo format. + ::Serialize(s, (unsigned char)0); + } + ::Serialize(s, CTxOutCompressor(REF(txout->out))); } + TxInUndoSerializer(const Coin* coin) : txout(coin) {} +}; + +class TxInUndoDeserializer +{ + Coin* txout; + +public: template<typename Stream> void Unserialize(Stream &s) { unsigned int nCode = 0; ::Unserialize(s, VARINT(nCode)); - nHeight = nCode / 2; - fCoinBase = nCode & 1; - if (nHeight > 0) - ::Unserialize(s, VARINT(this->nVersion)); - ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); + txout->nHeight = nCode / 2; + txout->fCoinBase = nCode & 1; + if (txout->nHeight > 0) { + // Old versions stored the version number for the last spend of + // a transaction's outputs. Non-final spends were indicated with + // height = 0. + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + } + ::Unserialize(s, REF(CTxOutCompressor(REF(txout->out)))); } + + TxInUndoDeserializer(Coin* coin) : txout(coin) {} }; +static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION); + /** Undo information for a CTransaction */ class CTxUndo { public: // undo information for all txins - std::vector<CTxInUndo> vprevout; + std::vector<Coin> vprevout; - ADD_SERIALIZE_METHODS; + template <typename Stream> + void Serialize(Stream& s) const { + // TODO: avoid reimplementing vector serializer + uint64_t count = vprevout.size(); + ::Serialize(s, COMPACTSIZE(REF(count))); + for (const auto& prevout : vprevout) { + ::Serialize(s, REF(TxInUndoSerializer(&prevout))); + } + } - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vprevout); + template <typename Stream> + void Unserialize(Stream& s) { + // TODO: avoid reimplementing vector deserializer + uint64_t count = 0; + ::Unserialize(s, COMPACTSIZE(count)); + if (count > MAX_INPUTS_PER_BLOCK) { + throw std::ios_base::failure("Too many input undo records"); + } + vprevout.resize(count); + for (auto& prevout : vprevout) { + ::Unserialize(s, REF(TxInUndoDeserializer(&prevout))); + } } }; diff --git a/src/util.cpp b/src/util.cpp index 78c353dfe5..653a4f072a 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -10,9 +10,9 @@ #include "util.h" #include "chainparamsbase.h" +#include "fs.h" #include "random.h" #include "serialize.h" -#include "sync.h" #include "utilstrencodings.h" #include "utiltime.h" @@ -72,11 +72,13 @@ #include <sys/prctl.h> #endif +#ifdef HAVE_MALLOPT_ARENA_MAX +#include <malloc.h> +#endif + #include <boost/algorithm/string/case_conv.hpp> // for to_lower() #include <boost/algorithm/string/join.hpp> #include <boost/algorithm/string/predicate.hpp> // for startswith() and endswith() -#include <boost/filesystem.hpp> -#include <boost/filesystem/fstream.hpp> #include <boost/foreach.hpp> #include <boost/program_options/detail/config_file.hpp> #include <boost/program_options/parsers.hpp> @@ -85,28 +87,11 @@ #include <openssl/rand.h> #include <openssl/conf.h> -// Work around clang compilation problem in Boost 1.46: -// /usr/include/boost/program_options/detail/config_file.hpp:163:17: error: call to function 'to_internal' that is neither visible in the template definition nor found by argument-dependent lookup -// See also: http://stackoverflow.com/questions/10020179/compilation-fail-in-boost-librairies-program-options -// http://clang.debian.net/status.php?version=3.0&key=CANNOT_FIND_FUNCTION -namespace boost { - - namespace program_options { - std::string to_internal(const std::string&); - } - -} // namespace boost - -using namespace std; const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; -CCriticalSection cs_args; -map<string, string> mapArgs; -static map<string, vector<string> > _mapMultiArgs; -const map<string, vector<string> >& mapMultiArgs = _mapMultiArgs; -bool fDebug = false; +ArgsManager gArgs; bool fPrintToConsole = false; bool fPrintToDebugLog = true; @@ -116,27 +101,28 @@ bool fLogIPs = DEFAULT_LOGIPS; std::atomic<bool> fReopenDebugLog(false); CTranslationInterface translationInterface; +/** Log categories bitfield. */ +std::atomic<uint32_t> logCategories(0); + /** Init OpenSSL library multithreading support */ -static CCriticalSection** ppmutexOpenSSL; +static std::unique_ptr<CCriticalSection[]> ppmutexOpenSSL; void locking_callback(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS { if (mode & CRYPTO_LOCK) { - ENTER_CRITICAL_SECTION(*ppmutexOpenSSL[i]); + ENTER_CRITICAL_SECTION(ppmutexOpenSSL[i]); } else { - LEAVE_CRITICAL_SECTION(*ppmutexOpenSSL[i]); + LEAVE_CRITICAL_SECTION(ppmutexOpenSSL[i]); } } -// Init +// Singleton for wrapping OpenSSL setup/teardown. class CInit { public: CInit() { // Init OpenSSL library multithreading support - ppmutexOpenSSL = (CCriticalSection**)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(CCriticalSection*)); - for (int i = 0; i < CRYPTO_num_locks(); i++) - ppmutexOpenSSL[i] = new CCriticalSection(); + ppmutexOpenSSL.reset(new CCriticalSection[CRYPTO_num_locks()]); CRYPTO_set_locking_callback(locking_callback); // OpenSSL can optionally load a config file which lists optional loadable modules and engines. @@ -160,9 +146,8 @@ public: RAND_cleanup(); // Shutdown OpenSSL library multithreading support CRYPTO_set_locking_callback(NULL); - for (int i = 0; i < CRYPTO_num_locks(); i++) - delete ppmutexOpenSSL[i]; - OPENSSL_free(ppmutexOpenSSL); + // Clear the set of locks now to maintain symmetry with the constructor. + ppmutexOpenSSL.reset(); } } instance_of_cinit; @@ -191,7 +176,7 @@ static boost::once_flag debugPrintInitFlag = BOOST_ONCE_INIT; */ static FILE* fileout = NULL; static boost::mutex* mutexDebugLog = NULL; -static list<string> *vMsgsBeforeOpenLog; +static std::list<std::string>* vMsgsBeforeOpenLog; static int FileWriteStr(const std::string &str, FILE *fp) { @@ -202,7 +187,7 @@ static void DebugPrintInit() { assert(mutexDebugLog == NULL); mutexDebugLog = new boost::mutex(); - vMsgsBeforeOpenLog = new list<string>; + vMsgsBeforeOpenLog = new std::list<std::string>; } void OpenDebugLog() @@ -212,8 +197,8 @@ void OpenDebugLog() assert(fileout == NULL); assert(vMsgsBeforeOpenLog); - boost::filesystem::path pathDebug = GetDataDir() / "debug.log"; - fileout = fopen(pathDebug.string().c_str(), "a"); + fs::path pathDebug = GetDataDir() / "debug.log"; + fileout = fsbridge::fopen(pathDebug, "a"); if (fileout) { setbuf(fileout, NULL); // unbuffered // dump buffered messages from before we opened the log @@ -227,36 +212,85 @@ void OpenDebugLog() vMsgsBeforeOpenLog = NULL; } -bool LogAcceptCategory(const char* category) +struct CLogCategoryDesc +{ + uint32_t flag; + std::string category; +}; + +const CLogCategoryDesc LogCategories[] = +{ + {BCLog::NONE, "0"}, + {BCLog::NET, "net"}, + {BCLog::TOR, "tor"}, + {BCLog::MEMPOOL, "mempool"}, + {BCLog::HTTP, "http"}, + {BCLog::BENCH, "bench"}, + {BCLog::ZMQ, "zmq"}, + {BCLog::DB, "db"}, + {BCLog::RPC, "rpc"}, + {BCLog::ESTIMATEFEE, "estimatefee"}, + {BCLog::ADDRMAN, "addrman"}, + {BCLog::SELECTCOINS, "selectcoins"}, + {BCLog::REINDEX, "reindex"}, + {BCLog::CMPCTBLOCK, "cmpctblock"}, + {BCLog::RAND, "rand"}, + {BCLog::PRUNE, "prune"}, + {BCLog::PROXY, "proxy"}, + {BCLog::MEMPOOLREJ, "mempoolrej"}, + {BCLog::LIBEVENT, "libevent"}, + {BCLog::COINDB, "coindb"}, + {BCLog::QT, "qt"}, + {BCLog::LEVELDB, "leveldb"}, + {BCLog::ALL, "1"}, + {BCLog::ALL, "all"}, +}; + +bool GetLogCategory(uint32_t *f, const std::string *str) +{ + if (f && str) { + if (*str == "") { + *f = BCLog::ALL; + return true; + } + for (unsigned int i = 0; i < ARRAYLEN(LogCategories); i++) { + if (LogCategories[i].category == *str) { + *f = LogCategories[i].flag; + return true; + } + } + } + return false; +} + +std::string ListLogCategories() { - if (category != NULL) - { - if (!fDebug) - return false; - - // Give each thread quick access to -debug settings. - // This helps prevent issues debugging global destructors, - // where mapMultiArgs might be deleted before another - // global destructor calls LogPrint() - static boost::thread_specific_ptr<set<string> > ptrCategory; - if (ptrCategory.get() == NULL) - { - if (mapMultiArgs.count("-debug")) { - const vector<string>& categories = mapMultiArgs.at("-debug"); - ptrCategory.reset(new set<string>(categories.begin(), categories.end())); - // thread_specific_ptr automatically deletes the set when the thread ends. - } else - ptrCategory.reset(new set<string>()); + std::string ret; + int outcount = 0; + for (unsigned int i = 0; i < ARRAYLEN(LogCategories); i++) { + // Omit the special cases. + if (LogCategories[i].flag != BCLog::NONE && LogCategories[i].flag != BCLog::ALL) { + if (outcount != 0) ret += ", "; + ret += LogCategories[i].category; + outcount++; } - const set<string>& setCategories = *ptrCategory.get(); + } + return ret; +} - // if not debugging everything and not debugging specific category, LogPrint does nothing. - if (setCategories.count(string("")) == 0 && - setCategories.count(string("1")) == 0 && - setCategories.count(string(category)) == 0) - return false; +std::vector<CLogCategoryActive> ListActiveLogCategories() +{ + std::vector<CLogCategoryActive> ret; + for (unsigned int i = 0; i < ARRAYLEN(LogCategories); i++) { + // Omit the special cases. + if (LogCategories[i].flag != BCLog::NONE && LogCategories[i].flag != BCLog::ALL) { + CLogCategoryActive catActive; + catActive.category = LogCategories[i].category; + catActive.active = LogAcceptCategory(LogCategories[i].flag); + ret.push_back(catActive); + } } - return true; + return ret; } /** @@ -266,16 +300,20 @@ bool LogAcceptCategory(const char* category) */ static std::string LogTimestampStr(const std::string &str, std::atomic_bool *fStartedNewLine) { - string strStamped; + std::string strStamped; if (!fLogTimestamps) return str; if (*fStartedNewLine) { - int64_t nTimeMicros = GetLogTimeMicros(); + int64_t nTimeMicros = GetTimeMicros(); strStamped = DateTimeStrFormat("%Y-%m-%d %H:%M:%S", nTimeMicros/1000000); if (fLogTimeMicros) strStamped += strprintf(".%06d", nTimeMicros%1000000); + int64_t mocktime = GetMockTime(); + if (mocktime) { + strStamped += " (mocktime: " + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", mocktime) + ")"; + } strStamped += ' ' + str; } else strStamped = str; @@ -293,7 +331,7 @@ int LogPrintStr(const std::string &str) int ret = 0; // Returns total number of characters written static std::atomic_bool fStartedNewLine(true); - string strTimestamped = LogTimestampStr(str, &fStartedNewLine); + std::string strTimestamped = LogTimestampStr(str, &fStartedNewLine); if (fPrintToConsole) { @@ -317,8 +355,8 @@ int LogPrintStr(const std::string &str) // reopen the log file, if requested if (fReopenDebugLog) { fReopenDebugLog = false; - boost::filesystem::path pathDebug = GetDataDir() / "debug.log"; - if (freopen(pathDebug.string().c_str(),"a",fileout) != NULL) + fs::path pathDebug = GetDataDir() / "debug.log"; + if (fsbridge::freopen(pathDebug,"a",fileout) != NULL) setbuf(fileout, NULL); // unbuffered } @@ -346,11 +384,11 @@ static void InterpretNegativeSetting(std::string& strKey, std::string& strValue) } } -void ParseParameters(int argc, const char* const argv[]) +void ArgsManager::ParseParameters(int argc, const char* const argv[]) { LOCK(cs_args); mapArgs.clear(); - _mapMultiArgs.clear(); + mapMultiArgs.clear(); for (int i = 1; i < argc; i++) { @@ -378,17 +416,23 @@ void ParseParameters(int argc, const char* const argv[]) InterpretNegativeSetting(str, strValue); mapArgs[str] = strValue; - _mapMultiArgs[str].push_back(strValue); + mapMultiArgs[str].push_back(strValue); } } -bool IsArgSet(const std::string& strArg) +std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) +{ + LOCK(cs_args); + return mapMultiArgs.at(strArg); +} + +bool ArgsManager::IsArgSet(const std::string& strArg) { LOCK(cs_args); return mapArgs.count(strArg); } -std::string GetArg(const std::string& strArg, const std::string& strDefault) +std::string ArgsManager::GetArg(const std::string& strArg, const std::string& strDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -396,7 +440,7 @@ std::string GetArg(const std::string& strArg, const std::string& strDefault) return strDefault; } -int64_t GetArg(const std::string& strArg, int64_t nDefault) +int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -404,7 +448,7 @@ int64_t GetArg(const std::string& strArg, int64_t nDefault) return nDefault; } -bool GetBoolArg(const std::string& strArg, bool fDefault) +bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -412,16 +456,16 @@ bool GetBoolArg(const std::string& strArg, bool fDefault) return fDefault; } -bool SoftSetArg(const std::string& strArg, const std::string& strValue) +bool ArgsManager::SoftSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); if (mapArgs.count(strArg)) return false; - mapArgs[strArg] = strValue; + ForceSetArg(strArg, strValue); return true; } -bool SoftSetBoolArg(const std::string& strArg, bool fValue) +bool ArgsManager::SoftSetBoolArg(const std::string& strArg, bool fValue) { if (fValue) return SoftSetArg(strArg, std::string("1")); @@ -429,10 +473,11 @@ bool SoftSetBoolArg(const std::string& strArg, bool fValue) return SoftSetArg(strArg, std::string("0")); } -void ForceSetArg(const std::string& strArg, const std::string& strValue) +void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); mapArgs[strArg] = strValue; + mapMultiArgs[strArg].push_back(strValue); } @@ -475,9 +520,8 @@ void PrintExceptionContinue(const std::exception* pex, const char* pszThread) fprintf(stderr, "\n\n************************\n%s\n", message.c_str()); } -boost::filesystem::path GetDefaultDataDir() +fs::path GetDefaultDataDir() { - namespace fs = boost::filesystem; // Windows < Vista: C:\Documents and Settings\Username\Application Data\Bitcoin // Windows >= Vista: C:\Users\Username\AppData\Roaming\Bitcoin // Mac: ~/Library/Application Support/Bitcoin @@ -502,13 +546,12 @@ boost::filesystem::path GetDefaultDataDir() #endif } -static boost::filesystem::path pathCached; -static boost::filesystem::path pathCachedNetSpecific; +static fs::path pathCached; +static fs::path pathCachedNetSpecific; static CCriticalSection csPathCached; -const boost::filesystem::path &GetDataDir(bool fNetSpecific) +const fs::path &GetDataDir(bool fNetSpecific) { - namespace fs = boost::filesystem; LOCK(csPathCached); @@ -540,39 +583,39 @@ void ClearDatadirCache() { LOCK(csPathCached); - pathCached = boost::filesystem::path(); - pathCachedNetSpecific = boost::filesystem::path(); + pathCached = fs::path(); + pathCachedNetSpecific = fs::path(); } -boost::filesystem::path GetConfigFile(const std::string& confPath) +fs::path GetConfigFile(const std::string& confPath) { - boost::filesystem::path pathConfigFile(confPath); + fs::path pathConfigFile(confPath); if (!pathConfigFile.is_complete()) pathConfigFile = GetDataDir(false) / pathConfigFile; return pathConfigFile; } -void ReadConfigFile(const std::string& confPath) +void ArgsManager::ReadConfigFile(const std::string& confPath) { - boost::filesystem::ifstream streamConfig(GetConfigFile(confPath)); + fs::ifstream streamConfig(GetConfigFile(confPath)); if (!streamConfig.good()) return; // No bitcoin.conf file is OK { LOCK(cs_args); - set<string> setOptions; + std::set<std::string> setOptions; setOptions.insert("*"); for (boost::program_options::detail::config_file_iterator it(streamConfig, setOptions), end; it != end; ++it) { // Don't overwrite existing settings so command line settings override bitcoin.conf - string strKey = string("-") + it->string_key; - string strValue = it->value[0]; + std::string strKey = std::string("-") + it->string_key; + std::string strValue = it->value[0]; InterpretNegativeSetting(strKey, strValue); if (mapArgs.count(strKey) == 0) mapArgs[strKey] = strValue; - _mapMultiArgs[strKey].push_back(strValue); + mapMultiArgs[strKey].push_back(strValue); } } // If datadir is changed in .conf file: @@ -580,16 +623,16 @@ void ReadConfigFile(const std::string& confPath) } #ifndef WIN32 -boost::filesystem::path GetPidFile() +fs::path GetPidFile() { - boost::filesystem::path pathPidFile(GetArg("-pid", BITCOIN_PID_FILENAME)); + fs::path pathPidFile(GetArg("-pid", BITCOIN_PID_FILENAME)); if (!pathPidFile.is_complete()) pathPidFile = GetDataDir() / pathPidFile; return pathPidFile; } -void CreatePidFile(const boost::filesystem::path &path, pid_t pid) +void CreatePidFile(const fs::path &path, pid_t pid) { - FILE* file = fopen(path.string().c_str(), "w"); + FILE* file = fsbridge::fopen(path, "w"); if (file) { fprintf(file, "%d\n", pid); @@ -598,7 +641,7 @@ void CreatePidFile(const boost::filesystem::path &path, pid_t pid) } #endif -bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest) +bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 return MoveFileExA(src.string().c_str(), dest.string().c_str(), @@ -614,13 +657,13 @@ bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest) * Specifically handles case where path p exists, but it wasn't possible for the user to * write to the parent directory. */ -bool TryCreateDirectory(const boost::filesystem::path& p) +bool TryCreateDirectory(const fs::path& p) { try { - return boost::filesystem::create_directory(p); - } catch (const boost::filesystem::filesystem_error&) { - if (!boost::filesystem::exists(p) || !boost::filesystem::is_directory(p)) + return fs::create_directory(p); + } catch (const fs::filesystem_error&) { + if (!fs::exists(p) || !fs::is_directory(p)) throw; } @@ -727,11 +770,11 @@ void ShrinkDebugFile() // Amount of debug.log to save at end when shrinking (must fit in memory) constexpr size_t RECENT_DEBUG_HISTORY_SIZE = 10 * 1000000; // Scroll debug.log if it's getting too big - boost::filesystem::path pathLog = GetDataDir() / "debug.log"; - FILE* file = fopen(pathLog.string().c_str(), "r"); + fs::path pathLog = GetDataDir() / "debug.log"; + FILE* file = fsbridge::fopen(pathLog, "r"); // If debug.log file is more than 10% bigger the RECENT_DEBUG_HISTORY_SIZE // trim it down by saving only the last RECENT_DEBUG_HISTORY_SIZE bytes - if (file && boost::filesystem::file_size(pathLog) > 11 * (RECENT_DEBUG_HISTORY_SIZE / 10)) + if (file && fs::file_size(pathLog) > 11 * (RECENT_DEBUG_HISTORY_SIZE / 10)) { // Restart the file with some of the end std::vector<char> vch(RECENT_DEBUG_HISTORY_SIZE, 0); @@ -739,7 +782,7 @@ void ShrinkDebugFile() int nBytes = fread(vch.data(), 1, vch.size(), file); fclose(file); - file = fopen(pathLog.string().c_str(), "w"); + file = fsbridge::fopen(pathLog, "w"); if (file) { fwrite(vch.data(), 1, nBytes, file); @@ -751,10 +794,8 @@ void ShrinkDebugFile() } #ifdef WIN32 -boost::filesystem::path GetSpecialFolderPath(int nFolder, bool fCreate) +fs::path GetSpecialFolderPath(int nFolder, bool fCreate) { - namespace fs = boost::filesystem; - char pszPath[MAX_PATH] = ""; if(SHGetSpecialFolderPathA(NULL, pszPath, nFolder, fCreate)) @@ -792,6 +833,16 @@ void RenameThread(const char* name) void SetupEnvironment() { +#ifdef HAVE_MALLOPT_ARENA_MAX + // glibc-specific: On 32-bit systems set the number of arenas to 1. + // By default, since glibc 2.10, the C library will create up to two heap + // arenas per core. This is known to cause excessive virtual address space + // usage in our usage. Work around it by setting the maximum number of + // arenas to 1. + if (sizeof(void*) == 4) { + mallopt(M_ARENA_MAX, 1); + } +#endif // On most POSIX systems (e.g. Linux, but not BSD) the environment's locale // may be invalid, in which case the "C" locale is used as fallback. #if !defined(WIN32) && !defined(MAC_OSX) && !defined(__FreeBSD__) && !defined(__OpenBSD__) @@ -804,9 +855,9 @@ void SetupEnvironment() // The path locale is lazy initialized and to avoid deinitialization errors // in multithreading environments, it is set explicitly by the main thread. // A dummy locale is used to extract the internal default locale, used by - // boost::filesystem::path, which is then used to explicitly imbue the path. - std::locale loc = boost::filesystem::path::imbue(std::locale::classic()); - boost::filesystem::path::imbue(loc); + // fs::path, which is then used to explicitly imbue the path. + std::locale loc = fs::path::imbue(std::locale::classic()); + fs::path::imbue(loc); } bool SetupNetworking() @@ -839,4 +890,4 @@ std::string CopyrightHolders(const std::string& strPrefix) strCopyrightHolders += "\n" + strPrefix + "The Bitcoin Core developers"; } return strCopyrightHolders; -}
\ No newline at end of file +} diff --git a/src/util.h b/src/util.h index e27ce121c8..4386ddd550 100644 --- a/src/util.h +++ b/src/util.h @@ -15,6 +15,8 @@ #endif #include "compat.h" +#include "fs.h" +#include "sync.h" #include "tinyformat.h" #include "utiltime.h" @@ -25,7 +27,6 @@ #include <string> #include <vector> -#include <boost/filesystem/path.hpp> #include <boost/signals2/signal.hpp> #include <boost/thread/exceptions.hpp> @@ -41,8 +42,6 @@ public: boost::signals2::signal<std::string (const char* psz)> Translate; }; -extern const std::map<std::string, std::vector<std::string> >& mapMultiArgs; -extern bool fDebug; extern bool fPrintToConsole; extern bool fPrintToDebugLog; @@ -55,6 +54,8 @@ extern CTranslationInterface translationInterface; extern const char * const BITCOIN_CONF_FILENAME; extern const char * const BITCOIN_PID_FILENAME; +extern std::atomic<uint32_t> logCategories; + /** * Translation function: Call Translate signal on UI interface, which returns a boost::optional result. * If no translation slot is registered, nothing is returned, and simply return the input. @@ -68,19 +69,75 @@ inline std::string _(const char* psz) void SetupEnvironment(); bool SetupNetworking(); +struct CLogCategoryActive +{ + std::string category; + bool active; +}; + +namespace BCLog { + enum LogFlags : uint32_t { + NONE = 0, + NET = (1 << 0), + TOR = (1 << 1), + MEMPOOL = (1 << 2), + HTTP = (1 << 3), + BENCH = (1 << 4), + ZMQ = (1 << 5), + DB = (1 << 6), + RPC = (1 << 7), + ESTIMATEFEE = (1 << 8), + ADDRMAN = (1 << 9), + SELECTCOINS = (1 << 10), + REINDEX = (1 << 11), + CMPCTBLOCK = (1 << 12), + RAND = (1 << 13), + PRUNE = (1 << 14), + PROXY = (1 << 15), + MEMPOOLREJ = (1 << 16), + LIBEVENT = (1 << 17), + COINDB = (1 << 18), + QT = (1 << 19), + LEVELDB = (1 << 20), + ALL = ~(uint32_t)0, + }; +} /** Return true if log accepts specified category */ -bool LogAcceptCategory(const char* category); +static inline bool LogAcceptCategory(uint32_t category) +{ + return (logCategories.load(std::memory_order_relaxed) & category) != 0; +} + +/** Returns a string with the log categories. */ +std::string ListLogCategories(); + +/** Returns a vector of the active log categories. */ +std::vector<CLogCategoryActive> ListActiveLogCategories(); + +/** Return true if str parses as a log category and set the flags in f */ +bool GetLogCategory(uint32_t *f, const std::string *str); + /** Send a string to the log output */ int LogPrintStr(const std::string &str); -#define LogPrint(category, ...) do { \ - if (LogAcceptCategory((category))) { \ - LogPrintStr(tfm::format(__VA_ARGS__)); \ +/** Get format string from VA_ARGS for error reporting */ +template<typename... Args> std::string FormatStringFromLogArgs(const char *fmt, const Args&... args) { return fmt; } + +#define LogPrintf(...) do { \ + std::string _log_msg_; /* Unlikely name to avoid shadowing variables */ \ + try { \ + _log_msg_ = tfm::format(__VA_ARGS__); \ + } catch (tinyformat::format_error &fmterr) { \ + /* Original format string will have newline so don't add one here */ \ + _log_msg_ = "Error \"" + std::string(fmterr.what()) + "\" while formatting log message: " + FormatStringFromLogArgs(__VA_ARGS__); \ } \ + LogPrintStr(_log_msg_); \ } while(0) -#define LogPrintf(...) do { \ - LogPrintStr(tfm::format(__VA_ARGS__)); \ +#define LogPrint(category, ...) do { \ + if (LogAcceptCategory((category))) { \ + LogPrintf(__VA_ARGS__); \ + } \ } while(0) template<typename... Args> @@ -91,24 +148,22 @@ bool error(const char* fmt, const Args&... args) } void PrintExceptionContinue(const std::exception *pex, const char* pszThread); -void ParseParameters(int argc, const char*const argv[]); void FileCommit(FILE *file); bool TruncateFile(FILE *file, unsigned int length); int RaiseFileDescriptorLimit(int nMinFD); void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length); -bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest); -bool TryCreateDirectory(const boost::filesystem::path& p); -boost::filesystem::path GetDefaultDataDir(); -const boost::filesystem::path &GetDataDir(bool fNetSpecific = true); +bool RenameOver(fs::path src, fs::path dest); +bool TryCreateDirectory(const fs::path& p); +fs::path GetDefaultDataDir(); +const fs::path &GetDataDir(bool fNetSpecific = true); void ClearDatadirCache(); -boost::filesystem::path GetConfigFile(const std::string& confPath); +fs::path GetConfigFile(const std::string& confPath); #ifndef WIN32 -boost::filesystem::path GetPidFile(); -void CreatePidFile(const boost::filesystem::path &path, pid_t pid); +fs::path GetPidFile(); +void CreatePidFile(const fs::path &path, pid_t pid); #endif -void ReadConfigFile(const std::string& confPath); #ifdef WIN32 -boost::filesystem::path GetSpecialFolderPath(int nFolder, bool fCreate = true); +fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif void OpenDebugLog(); void ShrinkDebugFile(); @@ -123,6 +178,16 @@ inline bool IsSwitchChar(char c) #endif } +class ArgsManager +{ +protected: + CCriticalSection cs_args; + std::map<std::string, std::string> mapArgs; + std::map<std::string, std::vector<std::string> > mapMultiArgs; +public: + void ParseParameters(int argc, const char*const argv[]); + void ReadConfigFile(const std::string& confPath); + std::vector<std::string> GetArgs(const std::string& strArg); /** * Return true if the given argument has been manually set * @@ -176,8 +241,58 @@ bool SoftSetArg(const std::string& strArg, const std::string& strValue); */ bool SoftSetBoolArg(const std::string& strArg, bool fValue); -// Forces a arg setting, used only in testing +// Forces an arg setting. Called by SoftSetArg() if the arg hasn't already +// been set. Also called directly in testing. void ForceSetArg(const std::string& strArg, const std::string& strValue); +}; + +extern ArgsManager gArgs; + +// wrappers using the global ArgsManager: +static inline void ParseParameters(int argc, const char*const argv[]) +{ + gArgs.ParseParameters(argc, argv); +} + +static inline void ReadConfigFile(const std::string& confPath) +{ + gArgs.ReadConfigFile(confPath); +} + +static inline bool SoftSetArg(const std::string& strArg, const std::string& strValue) +{ + return gArgs.SoftSetArg(strArg, strValue); +} + +static inline void ForceSetArg(const std::string& strArg, const std::string& strValue) +{ + gArgs.ForceSetArg(strArg, strValue); +} + +static inline bool IsArgSet(const std::string& strArg) +{ + return gArgs.IsArgSet(strArg); +} + +static inline std::string GetArg(const std::string& strArg, const std::string& strDefault) +{ + return gArgs.GetArg(strArg, strDefault); +} + +static inline int64_t GetArg(const std::string& strArg, int64_t nDefault) +{ + return gArgs.GetArg(strArg, nDefault); +} + +static inline bool GetBoolArg(const std::string& strArg, bool fDefault) +{ + return gArgs.GetBoolArg(strArg, fDefault); +} + +static inline bool SoftSetBoolArg(const std::string& strArg, bool fValue) +{ + return gArgs.SoftSetBoolArg(strArg, fValue); +} /** * Format a string to be used as group of options in help messages diff --git a/src/utilmoneystr.cpp b/src/utilmoneystr.cpp index bebe56130d..6e6e33184e 100644 --- a/src/utilmoneystr.cpp +++ b/src/utilmoneystr.cpp @@ -9,8 +9,6 @@ #include "tinyformat.h" #include "utilstrencodings.h" -using namespace std; - std::string FormatMoney(const CAmount& n) { // Note: not using straight sprintf here because we do NOT want @@ -18,7 +16,7 @@ std::string FormatMoney(const CAmount& n) int64_t n_abs = (n > 0 ? n : -n); int64_t quotient = n_abs/COIN; int64_t remainder = n_abs%COIN; - string str = strprintf("%d.%08d", quotient, remainder); + std::string str = strprintf("%d.%08d", quotient, remainder); // Right-trim excess zeros before the decimal point: int nTrim = 0; @@ -33,14 +31,14 @@ std::string FormatMoney(const CAmount& n) } -bool ParseMoney(const string& str, CAmount& nRet) +bool ParseMoney(const std::string& str, CAmount& nRet) { return ParseMoney(str.c_str(), nRet); } bool ParseMoney(const char* pszIn, CAmount& nRet) { - string strWhole; + std::string strWhole; int64_t nUnits = 0; const char* p = pszIn; while (isspace(*p)) diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 025040c43a..74bf66fbf6 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -12,19 +12,18 @@ #include <errno.h> #include <limits> -using namespace std; +static const std::string CHARS_ALPHA_NUM = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; -static const string CHARS_ALPHA_NUM = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - -static const string SAFE_CHARS[] = +static const std::string SAFE_CHARS[] = { CHARS_ALPHA_NUM + " .,;-_/:?@()", // SAFE_CHARS_DEFAULT - CHARS_ALPHA_NUM + " .,;-_?@" // SAFE_CHARS_UA_COMMENT + CHARS_ALPHA_NUM + " .,;-_?@", // SAFE_CHARS_UA_COMMENT + CHARS_ALPHA_NUM + ".-_", // SAFE_CHARS_FILENAME }; -string SanitizeString(const string& str, int rule) +std::string SanitizeString(const std::string& str, int rule) { - string strResult; + std::string strResult; for (std::string::size_type i = 0; i < str.size(); i++) { if (SAFE_CHARS[rule].find(str[i]) != std::string::npos) @@ -56,7 +55,7 @@ signed char HexDigit(char c) return p_util_hexdigit[(unsigned char)c]; } -bool IsHex(const string& str) +bool IsHex(const std::string& str) { for(std::string::const_iterator it(str.begin()); it != str.end(); ++it) { @@ -66,10 +65,10 @@ bool IsHex(const string& str) return (str.size() > 0) && (str.size()%2 == 0); } -vector<unsigned char> ParseHex(const char* psz) +std::vector<unsigned char> ParseHex(const char* psz) { // convert hex dump to vector - vector<unsigned char> vch; + std::vector<unsigned char> vch; while (true) { while (isspace(*psz)) @@ -87,16 +86,16 @@ vector<unsigned char> ParseHex(const char* psz) return vch; } -vector<unsigned char> ParseHex(const string& str) +std::vector<unsigned char> ParseHex(const std::string& str) { return ParseHex(str.c_str()); } -string EncodeBase64(const unsigned char* pch, size_t len) +std::string EncodeBase64(const unsigned char* pch, size_t len) { static const char *pbase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - string strRet=""; + std::string strRet = ""; strRet.reserve((len+2)/3*4); int mode=0, left=0; @@ -138,12 +137,12 @@ string EncodeBase64(const unsigned char* pch, size_t len) return strRet; } -string EncodeBase64(const string& str) +std::string EncodeBase64(const std::string& str) { return EncodeBase64((const unsigned char*)str.c_str(), str.size()); } -vector<unsigned char> DecodeBase64(const char* p, bool* pfInvalid) +std::vector<unsigned char> DecodeBase64(const char* p, bool* pfInvalid) { static const int decode64_table[256] = { @@ -165,7 +164,7 @@ vector<unsigned char> DecodeBase64(const char* p, bool* pfInvalid) if (pfInvalid) *pfInvalid = false; - vector<unsigned char> vchRet; + std::vector<unsigned char> vchRet; vchRet.reserve(strlen(p)*3/4); int mode = 0; @@ -226,17 +225,17 @@ vector<unsigned char> DecodeBase64(const char* p, bool* pfInvalid) return vchRet; } -string DecodeBase64(const string& str) +std::string DecodeBase64(const std::string& str) { - vector<unsigned char> vchRet = DecodeBase64(str.c_str()); - return (vchRet.size() == 0) ? string() : string((const char*)&vchRet[0], vchRet.size()); + std::vector<unsigned char> vchRet = DecodeBase64(str.c_str()); + return (vchRet.size() == 0) ? std::string() : std::string((const char*)&vchRet[0], vchRet.size()); } -string EncodeBase32(const unsigned char* pch, size_t len) +std::string EncodeBase32(const unsigned char* pch, size_t len) { static const char *pbase32 = "abcdefghijklmnopqrstuvwxyz234567"; - string strRet=""; + std::string strRet=""; strRet.reserve((len+4)/5*8); int mode=0, left=0; @@ -291,12 +290,12 @@ string EncodeBase32(const unsigned char* pch, size_t len) return strRet; } -string EncodeBase32(const string& str) +std::string EncodeBase32(const std::string& str) { return EncodeBase32((const unsigned char*)str.c_str(), str.size()); } -vector<unsigned char> DecodeBase32(const char* p, bool* pfInvalid) +std::vector<unsigned char> DecodeBase32(const char* p, bool* pfInvalid) { static const int decode32_table[256] = { @@ -318,7 +317,7 @@ vector<unsigned char> DecodeBase32(const char* p, bool* pfInvalid) if (pfInvalid) *pfInvalid = false; - vector<unsigned char> vchRet; + std::vector<unsigned char> vchRet; vchRet.reserve((strlen(p))*5/8); int mode = 0; @@ -413,10 +412,10 @@ vector<unsigned char> DecodeBase32(const char* p, bool* pfInvalid) return vchRet; } -string DecodeBase32(const string& str) +std::string DecodeBase32(const std::string& str) { - vector<unsigned char> vchRet = DecodeBase32(str.c_str()); - return (vchRet.size() == 0) ? string() : string((const char*)&vchRet[0], vchRet.size()); + std::vector<unsigned char> vchRet = DecodeBase32(str.c_str()); + return (vchRet.size() == 0) ? std::string() : std::string((const char*)&vchRet[0], vchRet.size()); } static bool ParsePrechecks(const std::string& str) diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index cb6f014fc2..e2a1b9bef9 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -26,7 +26,8 @@ enum SafeChars { SAFE_CHARS_DEFAULT, //!< The full set of allowed chars - SAFE_CHARS_UA_COMMENT //!< BIP-0014 subset + SAFE_CHARS_UA_COMMENT, //!< BIP-0014 subset + SAFE_CHARS_FILENAME, //!< Chars allowed in filenames }; /** diff --git a/src/utiltime.cpp b/src/utiltime.cpp index c7b3e4f168..e07069125d 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -9,16 +9,17 @@ #include "utiltime.h" +#include <atomic> + #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/thread.hpp> -using namespace std; - -static int64_t nMockTime = 0; //!< For unit testing +static std::atomic<int64_t> nMockTime(0); //!< For unit testing int64_t GetTime() { - if (nMockTime) return nMockTime; + int64_t mocktime = nMockTime.load(std::memory_order_relaxed); + if (mocktime) return mocktime; time_t now = time(NULL); assert(now > 0); @@ -27,7 +28,12 @@ int64_t GetTime() void SetMockTime(int64_t nMockTimeIn) { - nMockTime = nMockTimeIn; + nMockTime.store(nMockTimeIn, std::memory_order_relaxed); +} + +int64_t GetMockTime() +{ + return nMockTime.load(std::memory_order_relaxed); } int64_t GetTimeMillis() @@ -51,14 +57,6 @@ int64_t GetSystemTimeInSeconds() return GetTimeMicros()/1000000; } -/** Return a time useful for the debug log */ -int64_t GetLogTimeMicros() -{ - if (nMockTime) return nMockTime*1000000; - - return GetTimeMicros(); -} - void MilliSleep(int64_t n) { diff --git a/src/utiltime.h b/src/utiltime.h index cc3290c631..8ae8540b89 100644 --- a/src/utiltime.h +++ b/src/utiltime.h @@ -23,8 +23,8 @@ int64_t GetTime(); int64_t GetTimeMillis(); int64_t GetTimeMicros(); int64_t GetSystemTimeInSeconds(); // Like GetTime(), but not mockable -int64_t GetLogTimeMicros(); void SetMockTime(int64_t nMockTimeIn); +int64_t GetMockTime(); void MilliSleep(int64_t n); std::string DateTimeStrFormat(const char* pszFormat, int64_t nTime); diff --git a/src/validation.cpp b/src/validation.cpp index e84b1a7281..de65839eef 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -6,12 +6,15 @@ #include "validation.h" #include "arith_uint256.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" #include "consensus/consensus.h" #include "consensus/merkle.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" +#include "fs.h" #include "hash.h" #include "init.h" #include "policy/fees.h" @@ -41,8 +44,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/join.hpp> -#include <boost/filesystem.hpp> -#include <boost/filesystem/fstream.hpp> #include <boost/math/distributions/poisson.hpp> #include <boost/thread.hpp> @@ -81,7 +82,8 @@ uint256 hashAssumeValid; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool(::minRelayTxFee); +CBlockPolicyEstimator feeEstimator; +CTxMemPool mempool(&feeEstimator); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -155,39 +157,6 @@ namespace { std::set<int> setDirtyFileInfo; } // anon namespace -/* Use this class to start tracking transactions that are removed from the - * mempool and pass all those transactions through SyncTransaction when the - * object goes out of scope. This is currently only used to call SyncTransaction - * on conflicts removed from the mempool during block connection. Applied in - * ActivateBestChain around ActivateBestStep which in turn calls: - * ConnectTip->removeForBlock->removeConflicts - */ -class MemPoolConflictRemovalTracker -{ -private: - std::vector<CTransactionRef> conflictedTxs; - CTxMemPool &pool; - -public: - MemPoolConflictRemovalTracker(CTxMemPool &_pool) : pool(_pool) { - pool.NotifyEntryRemoved.connect(boost::bind(&MemPoolConflictRemovalTracker::NotifyEntryRemoved, this, _1, _2)); - } - - void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { - if (reason == MemPoolRemovalReason::CONFLICT) { - conflictedTxs.push_back(txRemoved); - } - } - - ~MemPoolConflictRemovalTracker() { - pool.NotifyEntryRemoved.disconnect(boost::bind(&MemPoolConflictRemovalTracker::NotifyEntryRemoved, this, _1, _2)); - for (const auto& tx : conflictedTxs) { - GetMainSignals().SyncTransaction(*tx, NULL, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } - conflictedTxs.clear(); - } -}; - CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) { // Find the first block the caller has in the main chain @@ -220,19 +189,6 @@ enum FlushStateMode { bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0); void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); -bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) -{ - if (tx.nLockTime == 0) - return true; - if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) - return true; - for (const auto& txin : tx.vin) { - if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) - return false; - } - return true; -} - bool CheckFinalTx(const CTransaction &tx, int flags) { AssertLockHeld(cs_main); @@ -265,89 +221,6 @@ bool CheckFinalTx(const CTransaction &tx, int flags) return IsFinalTx(tx, nBlockHeight, nBlockTime); } -/** - * Calculates the block height and previous block's median time past at - * which the transaction will be considered final in the context of BIP 68. - * Also removes from the vector of input heights any entries which did not - * correspond to sequence locked inputs as they do not affect the calculation. - */ -static std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) -{ - assert(prevHeights->size() == tx.vin.size()); - - // Will be set to the equivalent height- and time-based nLockTime - // values that would be necessary to satisfy all relative lock- - // time constraints given our view of block chain history. - // The semantics of nLockTime are the last invalid height/time, so - // use -1 to have the effect of any height or time being valid. - int nMinHeight = -1; - int64_t nMinTime = -1; - - // tx.nVersion is signed integer so requires cast to unsigned otherwise - // we would be doing a signed comparison and half the range of nVersion - // wouldn't support BIP 68. - bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 - && flags & LOCKTIME_VERIFY_SEQUENCE; - - // Do not enforce sequence numbers as a relative lock time - // unless we have been instructed to - if (!fEnforceBIP68) { - return std::make_pair(nMinHeight, nMinTime); - } - - for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { - const CTxIn& txin = tx.vin[txinIndex]; - - // Sequence numbers with the most significant bit set are not - // treated as relative lock-times, nor are they given any - // consensus-enforced meaning at this point. - if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { - // The height of this input is not relevant for sequence locks - (*prevHeights)[txinIndex] = 0; - continue; - } - - int nCoinHeight = (*prevHeights)[txinIndex]; - - if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { - int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); - // NOTE: Subtract 1 to maintain nLockTime semantics - // BIP 68 relative lock times have the semantics of calculating - // the first block or time at which the transaction would be - // valid. When calculating the effective block time or height - // for the entire transaction, we switch to using the - // semantics of nLockTime which is the last invalid block - // time or height. Thus we subtract 1 from the calculated - // time or height. - - // Time-based relative lock-times are measured from the - // smallest allowed timestamp of the block containing the - // txout being spent, which is the median time past of the - // block prior. - nMinTime = std::max(nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); - } else { - nMinHeight = std::max(nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); - } - } - - return std::make_pair(nMinHeight, nMinTime); -} - -static bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair) -{ - assert(block.pprev); - int64_t nBlockTime = block.pprev->GetMedianTimePast(); - if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) - return false; - - return true; -} - -bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) -{ - return EvaluateSequenceLocks(block, CalculateSequenceLocks(tx, flags, prevHeights, block)); -} - bool TestLockPointValidity(const LockPoints* lp) { AssertLockHeld(cs_main); @@ -395,15 +268,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn& txin = tx.vin[txinIndex]; - CCoins coins; - if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) { + Coin coin; + if (!viewMemPool.GetCoin(txin.prevout, coin)) { return error("%s: Missing input", __func__); } - if (coins.nHeight == MEMPOOL_HEIGHT) { + if (coin.nHeight == MEMPOOL_HEIGHT) { // Assume all mempool transaction confirm in the next block prevheights[txinIndex] = tip->nHeight + 1; } else { - prevheights[txinIndex] = coins.nHeight; + prevheights[txinIndex] = coin.nHeight; } } lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); @@ -436,116 +309,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool return EvaluateSequenceLocks(index, lockPair); } - -unsigned int GetLegacySigOpCount(const CTransaction& tx) -{ - unsigned int nSigOps = 0; - for (const auto& txin : tx.vin) - { - nSigOps += txin.scriptSig.GetSigOpCount(false); - } - for (const auto& txout : tx.vout) - { - nSigOps += txout.scriptPubKey.GetSigOpCount(false); - } - return nSigOps; -} - -unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& inputs) -{ - if (tx.IsCoinBase()) - return 0; - - unsigned int nSigOps = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); - if (prevout.scriptPubKey.IsPayToScriptHash()) - nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); - } - return nSigOps; -} - -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) -{ - int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; - - if (tx.IsCoinBase()) - return nSigOps; - - if (flags & SCRIPT_VERIFY_P2SH) { - nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; - } - - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); - nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); - } - return nSigOps; -} - - - - - -bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) -{ - // Basic checks that don't depend on any context - if (tx.vin.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); - if (tx.vout.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); - // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) - if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); - - // Check for negative or overflow output values - CAmount nValueOut = 0; - for (const auto& txout : tx.vout) - { - if (txout.nValue < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); - if (txout.nValue > MAX_MONEY) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); - nValueOut += txout.nValue; - if (!MoneyRange(nValueOut)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); - } - - // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock - if (fCheckDuplicateInputs) { - std::set<COutPoint> vInOutPoints; - for (const auto& txin : tx.vin) - { - if (!vInOutPoints.insert(txin.prevout).second) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); - } - } - - if (tx.IsCoinBase()) - { - if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); - } - else - { - for (const auto& txin : tx.vin) - if (txin.prevout.IsNull()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); - } - - return true; -} - void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { int expired = pool.Expire(GetTime() - age); - if (expired != 0) - LogPrint("mempool", "Expired %i transactions from the memory pool\n", expired); + if (expired != 0) { + LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); + } - std::vector<uint256> vNoSpendsRemaining; + std::vector<COutPoint> vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); - BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining) + BOOST_FOREACH(const COutPoint& removed, vNoSpendsRemaining) pcoinsTip->Uncache(removed); } @@ -570,9 +342,59 @@ static bool IsCurrentForFeeEstimation() return true; } +/* Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + +void UpdateMempoolForReorg(DisconnectedBlockTransactions &disconnectpool, bool fAddToMempool) +{ + AssertLockHeld(cs_main); + std::vector<uint256> vHashUpdate; + // disconnectpool's insertion_order index sorts the entries from + // oldest to newest, but the oldest entry will be the last tx from the + // latest mined block that was disconnected. + // Iterate disconnectpool in reverse, so that we add transactions + // back to the mempool starting with the earliest transaction that had + // been previously seen in a block. + auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); + while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { + // ignore validation errors in resurrected transactions + CValidationState stateDummy; + if (!fAddToMempool || (*it)->IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, *it, false, NULL, NULL, true)) { + // If the transaction doesn't make it in to the mempool, remove any + // transactions that depend on it (which would now be orphans). + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (mempool.exists((*it)->GetHash())) { + vHashUpdate.push_back((*it)->GetHash()); + } + ++it; + } + disconnectpool.queuedTx.clear(); + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in + // the disconnectpool that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); + + // We also need to remove any now-immature transactions + mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + // Re-limit mempool size, in case we added any transactions + LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); +} + bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<COutPoint>& coins_to_uncache) { const CTransaction& tx = *ptx; const uint256 hash = tx.GetHash(); @@ -665,30 +487,30 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C view.SetBackend(viewMemPool); // do we already have it? - bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash); - if (view.HaveCoins(hash)) { - if (!fHadTxInCache) - vHashTxnToUncache.push_back(hash); - return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + for (size_t out = 0; out < tx.vout.size(); out++) { + COutPoint outpoint(hash, out); + bool had_coin_in_cache = pcoinsTip->HaveCoinInCache(outpoint); + if (view.HaveCoin(outpoint)) { + if (!had_coin_in_cache) { + coins_to_uncache.push_back(outpoint); + } + return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + } } // do all inputs exist? - // Note that this does not check for the presence of actual outputs (see the next check for that), - // and only helps with filling in pfMissingInputs (to determine missing vs spent). BOOST_FOREACH(const CTxIn txin, tx.vin) { - if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash)) - vHashTxnToUncache.push_back(txin.prevout.hash); - if (!view.HaveCoins(txin.prevout.hash)) { - if (pfMissingInputs) + if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { + coins_to_uncache.push_back(txin.prevout); + } + if (!view.HaveCoin(txin.prevout)) { + if (pfMissingInputs) { *pfMissingInputs = true; + } return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() } } - // are the actual inputs available? - if (!view.HaveInputs(tx)) - return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); - // Bring the best block into scope view.GetBestBlock(); @@ -720,25 +542,21 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C CAmount nFees = nValueIn-nValueOut; // nModifiedFees includes any fee deltas from PrioritiseTransaction CAmount nModifiedFees = nFees; - double nPriorityDummy = 0; - pool.ApplyDeltas(hash, nPriorityDummy, nModifiedFees); - - CAmount inChainInputValue; - double dPriority = view.GetPriority(tx, chainActive.Height(), inChainInputValue); + pool.ApplyDelta(hash, nModifiedFees); // Keep track of transactions that spend a coinbase, which we re-scan // during reorgs to ensure COINBASE_MATURITY is still met. bool fSpendsCoinbase = false; BOOST_FOREACH(const CTxIn &txin, tx.vin) { - const CCoins *coins = view.AccessCoins(txin.prevout.hash); - if (coins->IsCoinBase()) { + const Coin &coin = view.AccessCoin(txin.prevout); + if (coin.IsCoinBase()) { fSpendsCoinbase = true; break; } } - CTxMemPoolEntry entry(ptx, nFees, nAcceptTime, dPriority, chainActive.Height(), - inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); + CTxMemPoolEntry entry(ptx, nFees, nAcceptTime, chainActive.Height(), + fSpendsCoinbase, nSigOpsCost, lp); unsigned int nSize = entry.GetTxSize(); // Check that the transaction doesn't have an excessive number of @@ -753,32 +571,11 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C CAmount mempoolRejectFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(nSize); if (mempoolRejectFee > 0 && nModifiedFees < mempoolRejectFee) { return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nFees, mempoolRejectFee)); - } else if (GetBoolArg("-relaypriority", DEFAULT_RELAYPRIORITY) && nModifiedFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(entry.GetPriority(chainActive.Height() + 1))) { - // Require that free transactions have sufficient priority to be mined in the next block. - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority"); } - // Continuously rate-limit free (really, very-low-fee) transactions - // This mitigates 'penny-flooding' -- sending thousands of free transactions just to - // be annoying or make others' transactions take longer to confirm. - if (fLimitFree && nModifiedFees < ::minRelayTxFee.GetFee(nSize)) - { - static CCriticalSection csFreeLimiter; - static double dFreeCount; - static int64_t nLastTime; - int64_t nNow = GetTime(); - - LOCK(csFreeLimiter); - - // Use an exponentially decaying ~10-minute window: - dFreeCount *= pow(1.0 - 1.0/600.0, (double)(nNow - nLastTime)); - nLastTime = nNow; - // -limitfreerelay unit is thousand-bytes-per-minute - // At default rate it would take over a month to fill 1GB - if (dFreeCount + nSize >= GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) * 10 * 1000) - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "rate limited free transaction"); - LogPrint("mempool", "Rate limit dFreeCount: %g => %g\n", dFreeCount, dFreeCount+nSize); - dFreeCount += nSize; + // No transactions are allowed below minRelayTxFee except from disconnected blocks + if (fLimitFree && nModifiedFees < ::minRelayTxFee.GetFee(nSize)) { + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "min relay fee not met"); } if (nAbsurdFee && nFees > nAbsurdFee) @@ -980,7 +777,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // Remove conflicting transactions from the mempool BOOST_FOREACH(const CTxMemPool::txiter it, allConflicting) { - LogPrint("mempool", "replacing tx %s with %s for %s BTC additional fees, %d delta bytes\n", + LogPrint(BCLog::MEMPOOL, "replacing tx %s with %s for %s BTC additional fees, %d delta bytes\n", it->GetTx().GetHash().ToString(), hash.ToString(), FormatMoney(nModifiedFees - nConflictingFees), @@ -1007,7 +804,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - GetMainSignals().SyncTransaction(tx, NULL, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + GetMainSignals().TransactionAddedToMempool(ptx); return true; } @@ -1016,10 +813,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { - std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + std::vector<COutPoint> coins_to_uncache; + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache); if (!res) { - BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) + BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) pcoinsTip->Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits @@ -1071,15 +868,8 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus } if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - const CCoinsViewCache& view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; - } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, hash); + if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; } if (pindexSlow) { @@ -1197,6 +987,7 @@ bool IsInitialBlockDownload() return true; if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; + LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); latchToFalse.store(true, std::memory_order_relaxed); return false; } @@ -1326,24 +1117,12 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); - unsigned nPos = txin.prevout.n; - - if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) - assert(false); - // mark an outpoint spent, and construct undo information - txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); - coins->Spend(nPos); - if (coins->vout.size() == 0) { - CTxInUndo& undo = txundo.vprevout.back(); - undo.nHeight = coins->nHeight; - undo.fCoinBase = coins->fCoinBase; - undo.nVersion = coins->nVersion; - } + txundo.vprevout.emplace_back(); + inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); } } // add outputs - inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); + AddCoins(inputs, tx, nHeight); } void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) @@ -1355,10 +1134,7 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; const CScriptWitness *witness = &ptxTo->vin[nIn].scriptWitness; - if (!VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error)) { - return false; - } - return true; + return VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error); } int GetSpendHeight(const CCoinsViewCache& inputs) @@ -1368,52 +1144,6 @@ int GetSpendHeight(const CCoinsViewCache& inputs) return pindexPrev->nHeight + 1; } -namespace Consensus { -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) -{ - // This doesn't trigger the DoS code on purpose; if it did, it would make it easier - // for an attacker to attempt to split the network. - if (!inputs.HaveInputs(tx)) - return state.Invalid(false, 0, "", "Inputs unavailable"); - - CAmount nValueIn = 0; - CAmount nFees = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); - - // If prev is coinbase, check that it's matured - if (coins->IsCoinBase()) { - if (nSpendHeight - coins->nHeight < COINBASE_MATURITY) - return state.Invalid(false, - REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", - strprintf("tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight)); - } - - // Check for negative or overflow input values - nValueIn += coins->vout[prevout.n].nValue; - if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); - - } - - if (nValueIn < tx.GetValueOut()) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, - strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); - - // Tally transaction fees - CAmount nTxFee = nValueIn - tx.GetValueOut(); - if (nTxFee < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); - nFees += nTxFee; - if (!MoneyRange(nFees)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); - return true; -} -}// namespace Consensus - bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) { if (!tx.IsCoinBase()) @@ -1436,11 +1166,19 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi if (fScriptChecks) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // We very carefully only pass in things to CScriptCheck which + // are clearly committed to by tx' witness hash. This provides + // a sanity check that our caching is not introducing consensus + // failures through additional data in, eg, the coins being + // spent being checked as a part of CScriptCheck. + const CScript& scriptPubKey = coin.out.scriptPubKey; + const CAmount amount = coin.out.nValue; // Verify signature - CScriptCheck check(*coins, tx, i, flags, cacheStore, &txdata); + CScriptCheck check(scriptPubKey, amount, tx, i, flags, cacheStore, &txdata); if (pvChecks) { pvChecks->push_back(CScriptCheck()); check.swap(pvChecks->back()); @@ -1452,7 +1190,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // arguments; if so, don't trigger DoS protection to // avoid splitting the network between upgraded and // non-upgraded nodes. - CScriptCheck check2(*coins, tx, i, + CScriptCheck check2(scriptPubKey, amount, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore, &txdata); if (check2()) return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); @@ -1511,8 +1249,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin // Read block uint256 hashChecksum; + CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - filein >> blockundo; + verifier << hashBlock; + verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -1520,10 +1260,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin } // Verify checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - if (hashChecksum != hasher.GetHash()) + if (hashChecksum != verifier.GetHash()) return error("%s: Checksum mismatch", __func__); return true; @@ -1549,57 +1286,66 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std } // anon namespace +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + /** - * Apply the undo operation of a CTxInUndo to the given chain state. - * @param undo The undo object. + * Restore the UTXO in a Coin at a given COutPoint + * @param undo The Coin to be restored. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. - * @return True on success. + * @return A DisconnectResult as an int */ -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; - CCoinsModifier coins = view.ModifyCoins(out.hash); - if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) - fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); - coins->Clear(); - coins->fCoinBase = undo.fCoinBase; - coins->nHeight = undo.nHeight; - coins->nVersion = undo.nVersion; - } else { - if (coins->IsPruned()) - fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + if (view.HaveCoin(out)) fClean = false; // overwriting transaction output + + if (undo.nHeight == 0) { + // Missing undo metadata (height and coinbase). Older versions included this + // information only in undo records for the last spend of a transactions' + // outputs. This implies that it must be present for some other output of the same tx. + const Coin& alternate = AccessByTxid(view, out.hash); + if (!alternate.IsSpent()) { + undo.nHeight = alternate.nHeight; + undo.fCoinBase = alternate.fCoinBase; + } else { + return DISCONNECT_FAILED; // adding output for transaction without known metadata + } } - if (coins->IsAvailable(out.n)) - fClean = fClean && error("%s: undo data overwriting existing output", __func__); - if (coins->vout.size() < out.n+1) - coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.txout; + view.AddCoin(out, std::move(undo), undo.fCoinBase); - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) +/** Undo the effects of this block (with given index) on the UTXO set represented by coins. + * When UNCLEAN or FAILED is returned, view is left in an indeterminate state. */ +static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) { assert(pindex->GetBlockHash() == view.GetBestBlock()); - if (pfClean) - *pfClean = false; - bool fClean = true; CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); - if (pos.IsNull()) - return error("DisconnectBlock(): no undo data available"); - if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) - return error("DisconnectBlock(): failure reading undo data"); + if (pos.IsNull()) { + error("DisconnectBlock(): no undo data available"); + return DISCONNECT_FAILED; + } + if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) { + error("DisconnectBlock(): failure reading undo data"); + return DISCONNECT_FAILED; + } - if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) - return error("DisconnectBlock(): block and undo data inconsistent"); + if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { + error("DisconnectBlock(): block and undo data inconsistent"); + return DISCONNECT_FAILED; + } // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -1608,46 +1354,38 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // Check that all outputs are available and match the outputs in the block itself // exactly. - { - CCoinsModifier outs = view.ModifyCoins(hash); - outs->ClearUnspendable(); - - CCoins outsBlock(tx, pindex->nHeight); - // The CCoins serialization does not serialize negative numbers. - // No network rules currently depend on the version here, so an inconsistency is harmless - // but it must be corrected before txout nversion ever influences a network rule. - if (outsBlock.nVersion < 0) - outs->nVersion = outsBlock.nVersion; - if (*outs != outsBlock) - fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); - - // remove outputs - outs->Clear(); + for (size_t o = 0; o < tx.vout.size(); o++) { + if (!tx.vout[o].scriptPubKey.IsUnspendable()) { + COutPoint out(hash, o); + Coin coin; + view.SpendCoin(out, &coin); + if (tx.vout[o] != coin.out) { + fClean = false; // transaction output mismatch + } + } } // restore inputs if (i > 0) { // not coinbases - const CTxUndo &txundo = blockUndo.vtxundo[i-1]; - if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock(): transaction and undo data inconsistent"); + CTxUndo &txundo = blockUndo.vtxundo[i-1]; + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; - const CTxInUndo &undo = txundo.vprevout[j]; - if (!ApplyTxInUndo(undo, view, out)) - fClean = false; + int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out); + if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; + fClean = fClean && res != DISCONNECT_UNCLEAN; } + // At this point, all of txundo.vprevout should have been moved out. } } // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); - if (pfClean) { - *pfClean = fClean; - return true; - } - - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } void static FlushBlockFile(bool fFinalize = false) @@ -1735,11 +1473,17 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) +/** Apply the effects of this block (with given index) on the UTXO set represented by coins. + * Validity checks that depend on the UTXO set are also done; ConnectBlock() + * can fail if those validity checks fail (among other reasons). */ +static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) { AssertLockHeld(cs_main); - + assert(pindex); + // pindex->phashBlock can be null if called by CreateNewBlock/TestBlockValidity + assert((pindex->phashBlock == NULL) || + (*pindex->phashBlock == block.GetHash())); int64_t nTimeStart = GetTimeMicros(); // Check it again in case a previous version let a bad block in @@ -1785,7 +1529,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } int64_t nTime1 = GetTimeMicros(); nTimeCheck += nTime1 - nTimeStart; - LogPrint("bench", " - Sanity checks: %.2fms [%.2fs]\n", 0.001 * (nTime1 - nTimeStart), nTimeCheck * 0.000001); + LogPrint(BCLog::BENCH, " - Sanity checks: %.2fms [%.2fs]\n", 0.001 * (nTime1 - nTimeStart), nTimeCheck * 0.000001); // Do not allow blocks that contain transactions which 'overwrite' older transactions, // unless those are already completely spent. @@ -1815,10 +1559,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fEnforceBIP30) { for (const auto& tx : block.vtx) { - const CCoins* coins = view.AccessCoins(tx->GetHash()); - if (coins && !coins->IsPruned()) - return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), - REJECT_INVALID, "bad-txns-BIP30"); + for (size_t o = 0; o < tx->vout.size(); o++) { + if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { + return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), + REJECT_INVALID, "bad-txns-BIP30"); + } + } } } @@ -1852,7 +1598,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; - LogPrint("bench", " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); + LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); CBlockUndo blockundo; @@ -1885,7 +1631,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // be in ConnectBlock because they require the UTXO set prevheights.resize(tx.vin.size()); for (size_t j = 0; j < tx.vin.size(); j++) { - prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; + prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight; } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { @@ -1926,7 +1672,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; - LogPrint("bench", " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001); + LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001); CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); if (block.vtx[0]->GetValueOut() > blockReward) @@ -1936,9 +1682,9 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin REJECT_INVALID, "bad-cb-amount"); if (!control.Wait()) - return state.DoS(100, false); + return state.DoS(100, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; - LogPrint("bench", " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime4 - nTime2), nInputs <= 1 ? 0 : 0.001 * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * 0.000001); + LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime4 - nTime2), nInputs <= 1 ? 0 : 0.001 * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * 0.000001); if (fJustCheck) return true; @@ -1970,16 +1716,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin view.SetBestBlock(pindex->GetBlockHash()); int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; - LogPrint("bench", " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); - - // Watch for changes to the previous coinbase transaction. - static uint256 hashPrevBestCoinBase; - GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); - hashPrevBestCoinBase = block.vtx[0]->GetHash(); - + LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; - LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); + LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); return true; } @@ -2027,10 +1767,10 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n nLastSetChain = nNow; } int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 100 MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - 100 * 1024 * 1024); + // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. @@ -2071,12 +1811,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush) { - // Typical CCoins structures on disk are around 128 bytes in size. + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) return state.Error("out of disk space"); // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) @@ -2105,6 +1845,16 @@ void PruneAndFlush() { FlushStateToDisk(state, FLUSH_STATE_NONE); } +static void DoWarning(const std::string& strWarning) +{ + static bool fWarned = false; + SetMiscWarning(strWarning); + if (!fWarned) { + AlertNotify(strWarning); + fWarned = true; + } +} + /** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); @@ -2114,7 +1864,6 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { cvBlockChange.notify_all(); - static bool fWarned = false; std::vector<std::string> warningMessages; if (!IsInitialBlockDownload()) { @@ -2124,15 +1873,11 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { WarningBitsConditionChecker checker(bit); ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); if (state == THRESHOLD_ACTIVE || state == THRESHOLD_LOCKED_IN) { + const std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); if (state == THRESHOLD_ACTIVE) { - std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + DoWarning(strWarning); } else { - warningMessages.push_back(strprintf("unknown new rules are about to activate (versionbit %i)", bit)); + warningMessages.push_back(strWarning); } } } @@ -2145,19 +1890,15 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { pindex = pindex->pprev; } if (nUpgraded > 0) - warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); + warningMessages.push_back(strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); if (nUpgraded > 100/2) { std::string strWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); // notify GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + DoWarning(strWarning); } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), @@ -2168,57 +1909,57 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } -/** Disconnect chainActive's tip. You probably want to call mempool.removeForReorg and manually re-limit mempool size after this, with cs_main held. */ -bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, bool fBare = false) +/** Disconnect chainActive's tip. + * After calling, the mempool will be in an inconsistent state, with + * transactions from disconnected blocks being added to disconnectpool. You + * should make the mempool consistent again by calling UpdateMempoolForReorg. + * with cs_main held. + * + * If disconnectpool is NULL, then no disconnected transactions are added to + * disconnectpool (note that the caller is responsible for mempool consistency + * in any case). + */ +bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); // Read block from disk. - CBlock block; + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); + CBlock& block = *pblock; if (!ReadBlockFromDisk(block, pindexDelete, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(pcoinsTip); - if (!DisconnectBlock(block, state, pindexDelete, view)) + if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); bool flushed = view.Flush(); assert(flushed); } - LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); + LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; - if (!fBare) { - // Resurrect mempool transactions from the disconnected block. - std::vector<uint256> vHashUpdate; - for (const auto& it : block.vtx) { - const CTransaction& tx = *it; - // ignore validation errors in resurrected transactions - CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, NULL, true)) { - mempool.removeRecursive(tx, MemPoolRemovalReason::REORG); - } else if (mempool.exists(tx.GetHash())) { - vHashUpdate.push_back(tx.GetHash()); - } + if (disconnectpool) { + // Save transactions to re-add to mempool at end of reorg + for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { + disconnectpool->addTransaction(*it); + } + while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { + // Drop the earliest entry, and remove its children from the mempool. + auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + disconnectpool->removeEntry(it); } - // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have - // no in-mempool children, which is generally not true when adding - // previously-confirmed transactions back to the mempool. - // UpdateTransactionsFromBlock finds descendants of any transactions in this - // block that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); } // Update chainActive and related variables. UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - for (const auto& tx : block.vtx) { - GetMainSignals().SyncTransaction(*tx, pindexDelete->pprev, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } + GetMainSignals().BlockDisconnected(pblock); return true; } @@ -2228,40 +1969,96 @@ static int64_t nTimeFlush = 0; static int64_t nTimeChainState = 0; static int64_t nTimePostConnect = 0; +struct PerBlockConnectTrace { + CBlockIndex* pindex = NULL; + std::shared_ptr<const CBlock> pblock; + std::shared_ptr<std::vector<CTransactionRef>> conflictedTxs; + PerBlockConnectTrace() : conflictedTxs(std::make_shared<std::vector<CTransactionRef>>()) {} +}; /** * Used to track blocks whose transactions were applied to the UTXO state as a * part of a single ActivateBestChainStep call. + * + * This class also tracks transactions that are removed from the mempool as + * conflicts (per block) and can be used to pass all those transactions + * through SyncTransaction. + * + * This class assumes (and asserts) that the conflicted transactions for a given + * block are added via mempool callbacks prior to the BlockConnected() associated + * with those transactions. If any transactions are marked conflicted, it is + * assumed that an associated block will always be added. + * + * This class is single-use, once you call GetBlocksConnected() you have to throw + * it away and make a new one. */ -struct ConnectTrace { - std::vector<std::pair<CBlockIndex*, std::shared_ptr<const CBlock> > > blocksConnected; +class ConnectTrace { +private: + std::vector<PerBlockConnectTrace> blocksConnected; + CTxMemPool &pool; + +public: + ConnectTrace(CTxMemPool &_pool) : blocksConnected(1), pool(_pool) { + pool.NotifyEntryRemoved.connect(boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); + } + + ~ConnectTrace() { + pool.NotifyEntryRemoved.disconnect(boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); + } + + void BlockConnected(CBlockIndex* pindex, std::shared_ptr<const CBlock> pblock) { + assert(!blocksConnected.back().pindex); + assert(pindex); + assert(pblock); + blocksConnected.back().pindex = pindex; + blocksConnected.back().pblock = std::move(pblock); + blocksConnected.emplace_back(); + } + + std::vector<PerBlockConnectTrace>& GetBlocksConnected() { + // We always keep one extra block at the end of our list because + // blocks are added after all the conflicted transactions have + // been filled in. Thus, the last entry should always be an empty + // one waiting for the transactions from the next block. We pop + // the last entry here to make sure the list we return is sane. + assert(!blocksConnected.back().pindex); + assert(blocksConnected.back().conflictedTxs->empty()); + blocksConnected.pop_back(); + return blocksConnected; + } + + void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { + assert(!blocksConnected.back().pindex); + if (reason == MemPoolRemovalReason::CONFLICT) { + blocksConnected.back().conflictedTxs->emplace_back(std::move(txRemoved)); + } + } }; /** * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. * - * The block is always added to connectTrace (either after loading from disk or by copying - * pblock) - if that is not intended, care must be taken to remove the last entry in - * blocksConnected in case of failure. + * The block is added to connectTrace if connection succeeds. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. int64_t nTime1 = GetTimeMicros(); + std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - connectTrace.blocksConnected.emplace_back(pindexNew, pblockNew); if (!ReadBlockFromDisk(*pblockNew, pindexNew, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); + pthisBlock = pblockNew; } else { - connectTrace.blocksConnected.emplace_back(pindexNew, pblock); + pthisBlock = pblock; } - const CBlock& blockConnecting = *connectTrace.blocksConnected.back().second; + const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; int64_t nTime3; - LogPrint("bench", " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { CCoinsViewCache view(pcoinsTip); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); @@ -2272,25 +2069,28 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, return error("ConnectTip(): ConnectBlock %s failed", pindexNew->GetBlockHash().ToString()); } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; - LogPrint("bench", " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); + LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); bool flushed = view.Flush(); assert(flushed); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; - LogPrint("bench", " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); + LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; - LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); + LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + disconnectpool.removeForBlock(blockConnecting.vtx); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; - LogPrint("bench", " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); - LogPrint("bench", "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); + LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); + LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); + + connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true; } @@ -2376,9 +2176,14 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; + DisconnectedBlockTransactions disconnectpool; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { - if (!DisconnectTip(state, chainparams)) + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // This is likely a fatal error, but keep the mempool consistent, + // just in case. Only remove from the mempool in this case. + UpdateMempoolForReorg(disconnectpool, false); return false; + } fBlocksDisconnected = true; } @@ -2401,7 +2206,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Connect new blocks. BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace)) { + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. if (!state.CorruptionPossible()) @@ -2409,11 +2214,12 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c state = CValidationState(); fInvalidFound = true; fContinue = false; - // If we didn't actually connect the block, don't notify listeners about it - connectTrace.blocksConnected.pop_back(); break; } else { // A system error occurred (disk space, database error, ...). + // Make the mempool consistent with the current tip, just in case + // any observers try to use it before shutdown. + UpdateMempoolForReorg(disconnectpool, false); return false; } } else { @@ -2428,8 +2234,9 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c } if (fBlocksDisconnected) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // If any blocks were disconnected, disconnectpool may be non empty. Add + // any disconnected transactions back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); } mempool.check(pcoinsTip); @@ -2482,18 +2289,11 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, break; const CBlockIndex *pindexFork; - ConnectTrace connectTrace; bool fInitialDownload; { LOCK(cs_main); - { // TODO: Temporarily ensure that mempool removals are notified before - // connected transactions. This shouldn't matter, but the abandoned - // state of transactions in our wallet is currently cleared when we - // receive another notification and there is a race condition where - // notification of a connected conflict might cause an outside process - // to abandon a transaction and then have it inadvertently cleared by - // the notification that the conflicted transaction was evicted. - MemPoolConflictRemovalTracker mrt(mempool); + ConnectTrace connectTrace(mempool); // Destructed before cs_main is unlocked + CBlockIndex *pindexOldTip = chainActive.Tip(); if (pindexMostWork == NULL) { pindexMostWork = FindMostWorkChain(); @@ -2516,16 +2316,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, pindexFork = chainActive.FindFork(pindexOldTip); fInitialDownload = IsInitialBlockDownload(); - // throw all transactions though the signal-interface - - } // MemPoolConflictRemovalTracker destroyed and conflict evictions are notified - - // Transactions in the connected block are notified - for (const auto& pair : connectTrace.blocksConnected) { - assert(pair.second); - const CBlock& block = *(pair.second); - for (unsigned int i = 0; i < block.vtx.size(); i++) - GetMainSignals().SyncTransaction(*block.vtx[i], pair.first, i); + for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { + assert(trace.pblock && trace.pindex); + GetMainSignals().BlockConnected(trace.pblock, trace.pindex, *trace.conflictedTxs); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -2547,6 +2340,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, return false; } + int nStopAtHeight = GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); + if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + return true; } @@ -2589,6 +2385,7 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); + DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { CBlockIndex *pindexWalk = chainActive.Tip(); pindexWalk->nStatus |= BLOCK_FAILED_CHILD; @@ -2596,13 +2393,17 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C setBlockIndexCandidates.erase(pindexWalk); // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, chainparams)) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // It's probably hopeless to try to make the mempool consistent + // here if DisconnectTip failed, but we can try. + UpdateMempoolForReorg(disconnectpool, false); return false; } } - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // DisconnectTip will add transactions to disconnectpool; try to add these + // back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); // The resulting new best tip may not be in setBlockIndexCandidates anymore, so // add it again. @@ -2615,7 +2416,6 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C } InvalidChainFound(pindex); - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); return true; } @@ -2689,7 +2489,7 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) +static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2697,7 +2497,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (IsWitnessEnabled(pindexNew->pprev, Params().GetConsensus())) { + if (IsWitnessEnabled(pindexNew->pprev, consensusParams)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); @@ -2903,10 +2703,12 @@ static bool CheckIndexAgainstCheckpoint(const CBlockIndex* pindexPrev, CValidati return true; int nHeight = pindexPrev->nHeight+1; - // Don't accept any forks from the main chain prior to last checkpoint + // Don't accept any forks from the main chain prior to last checkpoint. + // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our + // MapBlockIndex. CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(chainparams.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight)); + return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), REJECT_CHECKPOINT, "bad-fork-prior-to-checkpoint"); return true; } @@ -2922,9 +2724,11 @@ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& pa static int GetWitnessCommitmentIndex(const CBlock& block) { int commitpos = -1; - for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { - if (block.vtx[0]->vout[o].scriptPubKey.size() >= 38 && block.vtx[0]->vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0]->vout[o].scriptPubKey[1] == 0x24 && block.vtx[0]->vout[o].scriptPubKey[2] == 0xaa && block.vtx[0]->vout[o].scriptPubKey[3] == 0x21 && block.vtx[0]->vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0]->vout[o].scriptPubKey[5] == 0xed) { - commitpos = o; + if (!block.vtx.empty()) { + for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { + if (block.vtx[0]->vout[o].scriptPubKey.size() >= 38 && block.vtx[0]->vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0]->vout[o].scriptPubKey[1] == 0x24 && block.vtx[0]->vout[o].scriptPubKey[2] == 0xaa && block.vtx[0]->vout[o].scriptPubKey[3] == 0x21 && block.vtx[0]->vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0]->vout[o].scriptPubKey[5] == 0xed) { + commitpos = o; + } } } return commitpos; @@ -2973,7 +2777,8 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) { - const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; + assert(pindexPrev != NULL); + const int nHeight = pindexPrev->nHeight + 1; // Check proof of work if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) return state.DoS(100, false, REJECT_INVALID, "bad-diffbits", false, "incorrect proof of work"); @@ -2983,7 +2788,7 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta return state.Invalid(false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); // Check timestamp - if (block.GetBlockTime() > nAdjustedTime + 2 * 60 * 60) + if (block.GetBlockTime() > nAdjustedTime + MAX_FUTURE_BLOCK_TIME) return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: @@ -3058,8 +2863,8 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // No witness data is allowed in blocks that don't commit to witness data, as this would otherwise leave room for spam if (!fHaveWitness) { - for (size_t i = 0; i < block.vtx.size(); i++) { - if (block.vtx[i]->HasWitness()) { + for (const auto& tx : block.vtx) { + if (tx->HasWitness()) { return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); } } @@ -3104,7 +2909,7 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state CBlockIndex* pindexPrev = NULL; BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); if (mi == mapBlockIndex.end()) - return state.DoS(10, error("%s: prev block not found", __func__), 0, "bad-prevblk"); + return state.DoS(10, error("%s: prev block not found", __func__), 0, "prev-blk-not-found"); pindexPrev = (*mi).second; if (pindexPrev->nStatus & BLOCK_FAILED_MASK) return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); @@ -3173,7 +2978,7 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation bool fTooFarAhead = (pindex->nHeight > int(chainActive.Height() + MIN_BLOCKS_TO_KEEP)); // TODO: Decouple this function from the block download logic by removing fRequested - // This requires some new chain datastructure to efficiently look up if a + // This requires some new chain data structure to efficiently look up if a // block is in a chain leading to a candidate for best tip, despite not // being such a candidate itself. @@ -3214,7 +3019,7 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation if (dbp == NULL) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) AbortNode(state, "Failed to write block"); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("AcceptBlock(): ReceivedBlockTransactions failed"); } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error: ") + e.what()); @@ -3335,8 +3140,8 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) { for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { CDiskBlockPos pos(*it, 0); - boost::filesystem::remove(GetBlockPosFilename(pos, "blk")); - boost::filesystem::remove(GetBlockPosFilename(pos, "rev")); + fs::remove(GetBlockPosFilename(pos, "blk")); + fs::remove(GetBlockPosFilename(pos, "rev")); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -3412,7 +3217,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight } } - LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", + LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", nPruneTarget/1024/1024, nCurrentUsage/1024/1024, ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, nLastBlockWeCanPrune, count); @@ -3420,7 +3225,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight bool CheckDiskSpace(uint64_t nAdditionalBytes) { - uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available; + uint64_t nFreeBytesAvailable = fs::space(GetDataDir()).available; // Check for nMinDiskSpace bytes (currently 50MB) if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) @@ -3433,11 +3238,11 @@ FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { if (pos.IsNull()) return NULL; - boost::filesystem::path path = GetBlockPosFilename(pos, prefix); - boost::filesystem::create_directories(path.parent_path()); - FILE* file = fopen(path.string().c_str(), "rb+"); + fs::path path = GetBlockPosFilename(pos, prefix); + fs::create_directories(path.parent_path()); + FILE* file = fsbridge::fopen(path, "rb+"); if (!file && !fReadOnly) - file = fopen(path.string().c_str(), "wb+"); + file = fsbridge::fopen(path, "wb+"); if (!file) { LogPrintf("Unable to open file %s\n", path.string()); return NULL; @@ -3460,7 +3265,7 @@ FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { return OpenDiskFile(pos, "rev", fReadOnly); } -boost::filesystem::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) { return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); } @@ -3661,15 +3466,17 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { - bool fClean = true; - if (!DisconnectBlock(block, state, pindex, coins, &fClean)) + DisconnectResult res = DisconnectBlock(block, pindex, coins); + if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + } pindexState = pindex->pprev; - if (!fClean) { + if (res == DISCONNECT_UNCLEAN) { nGoodTransactions = 0; pindexFailure = pindex; - } else + } else { nGoodTransactions += block.vtx.size(); + } } if (ShutdownRequested()) return true; @@ -3722,7 +3529,7 @@ bool RewindBlockIndex(const CChainParams& params) // of the blockchain). break; } - if (!DisconnectTip(state, params, true)) { + if (!DisconnectTip(state, params, NULL)) { return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); } // Occasionally flush state to disk. @@ -3845,7 +3652,7 @@ bool InitBlockIndex(const CChainParams& chainparams) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) return error("LoadBlockIndex(): writing genesis block to disk failed"); CBlockIndex *pindex = AddToBlockIndex(block); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("LoadBlockIndex(): genesis block not accepted"); // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); @@ -3906,7 +3713,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB // detect out of order blocks, and store them for later uint256 hash = block.GetHash(); if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex.find(block.hashPrevBlock) == mapBlockIndex.end()) { - LogPrint("reindex", "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), + LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp) mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp)); @@ -3922,7 +3729,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB if (state.IsError()) break; } else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) { - LogPrint("reindex", "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); + LogPrint(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); } // Activate the genesis block so normal node progress can continue @@ -3947,7 +3754,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) { - LogPrint("reindex", "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), + LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); CValidationState dummy; @@ -4174,6 +3981,12 @@ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::D return VersionBitsState(chainActive.Tip(), params, pos, versionbitscache); } +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + LOCK(cs_main); + return VersionBitsStatistics(chainActive.Tip(), params, pos); +} + int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos) { LOCK(cs_main); @@ -4185,7 +3998,7 @@ static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(void) { int64_t nExpiryTimeout = GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; - FILE* filestr = fopen((GetDataDir() / "mempool.dat").string().c_str(), "rb"); + FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat", "rb"); CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); @@ -4205,7 +4018,6 @@ bool LoadMempool(void) } uint64_t num; file >> num; - double prioritydummy = 0; while (num--) { CTransactionRef tx; int64_t nTime; @@ -4216,7 +4028,7 @@ bool LoadMempool(void) CAmount amountdelta = nFeeDelta; if (amountdelta) { - mempool.PrioritiseTransaction(tx->GetHash(), prioritydummy, amountdelta); + mempool.PrioritiseTransaction(tx->GetHash(), amountdelta); } CValidationState state; if (nTime + nExpiryTimeout > nNow) { @@ -4237,7 +4049,7 @@ bool LoadMempool(void) file >> mapDeltas; for (const auto& i : mapDeltas) { - mempool.PrioritiseTransaction(i.first, prioritydummy, i.second); + mempool.PrioritiseTransaction(i.first, i.second); } } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); @@ -4258,7 +4070,7 @@ void DumpMempool(void) { LOCK(mempool.cs); for (const auto &i : mempool.mapDeltas) { - mapDeltas[i.first] = i.second.second; + mapDeltas[i.first] = i.second; } vinfo = mempool.infoAll(); } @@ -4266,7 +4078,7 @@ void DumpMempool(void) int64_t mid = GetTimeMicros(); try { - FILE* filestr = fopen((GetDataDir() / "mempool.dat.new").string().c_str(), "wb"); + FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat.new", "wb"); if (!filestr) { return; } diff --git a/src/validation.h b/src/validation.h index 9c606f2419..096fd0a9ee 100644 --- a/src/validation.h +++ b/src/validation.h @@ -11,9 +11,10 @@ #endif #include "amount.h" -#include "chain.h" #include "coins.h" +#include "fs.h" #include "protocol.h" // For CMessageHeader::MessageStartChars +#include "policy/feerate.h" #include "script/script_error.h" #include "sync.h" #include "versionbits.h" @@ -30,7 +31,6 @@ #include <atomic> #include <boost/unordered_map.hpp> -#include <boost/filesystem/path.hpp> class CBlockIndex; class CBlockTreeDB; @@ -39,6 +39,7 @@ class CChainParams; class CInv; class CConnman; class CScriptCheck; +class CBlockPolicyEstimator; class CTxMemPool; class CValidationInterface; class CValidationState; @@ -69,6 +70,8 @@ static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ @@ -104,7 +107,7 @@ static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; /** Maximum length of reject messages. */ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; /** Average delay between local address broadcasts in seconds. */ -static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 24 * 60; +static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 60 * 60; /** Average delay between peer address broadcasts in seconds. */ static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; /** Average delay between trickled inventory transmissions in seconds. @@ -122,8 +125,6 @@ static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; /** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; -static const unsigned int DEFAULT_LIMITFREERELAY = 0; -static const bool DEFAULT_RELAYPRIORITY = true; static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; /** Maximum age of our tip in seconds for us to be considered current for fee estimation */ static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; @@ -133,7 +134,8 @@ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; - +/** Default for -persistmempool */ +static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -mempoolreplacement */ static const bool DEFAULT_ENABLE_REPLACEMENT = true; /** Default for using fee filter */ @@ -147,6 +149,9 @@ static const int MAX_UNCONNECTING_HEADERS = 10; static const bool DEFAULT_PEERBLOOMFILTERS = true; +/** Default for -stopatheight */ +static const int DEFAULT_STOPATHEIGHT = 0; + struct BlockHasher { size_t operator()(const uint256& hash) const { return hash.GetCheapHash(); } @@ -154,6 +159,7 @@ struct BlockHasher extern CScript COINBASE_FLAGS; extern CCriticalSection cs_main; +extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef boost::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap mapBlockIndex; @@ -252,7 +258,7 @@ FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); /** Open an undo file (rev?????.dat) */ FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -boost::filesystem::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); +fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); /** Import blocks from an external file */ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = NULL); /** Initialize a new block tree database + block data on disk */ @@ -316,7 +322,7 @@ void FlushStateToDisk(); /** Prune block files and flush state to disk. */ void PruneAndFlush(); /** Prune block files up to a given height */ -void PruneBlockFilesManual(int nPruneUpToHeight); +void PruneBlockFilesManual(int nManualPruneHeight); /** (try to) add transaction to memory pool * plTxnReplaced will be appended to with all transactions replaced from mempool **/ @@ -335,33 +341,12 @@ std::string FormatStateMessage(const CValidationState &state); /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); +/** Get the numerical statistics for the BIP9 state for a given deployment at the current tip. */ +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos); + /** Get the block height at which the BIP9 deployment switched into the state for the block building on the current tip. */ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos); -/** - * Count ECDSA signature operations the old-fashioned (pre-0.6) way - * @return number of sigops this transaction's outputs will produce when spent - * @see CTransaction::FetchInputs - */ -unsigned int GetLegacySigOpCount(const CTransaction& tx); - -/** - * Count ECDSA signature operations in pay-to-script-hash inputs. - * - * @param[in] mapInputs Map of previous transactions that have outputs we're spending - * @return maximum number of sigops required to validate this transaction's inputs - * @see CTransaction::FetchInputs - */ -unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& mapInputs); - -/** - * Compute total signature operation cost of a transaction. - * @param[in] tx Transaction for which we are computing the cost - * @param[in] inputs Map of previous transactions that have outputs we're spending - * @param[out] flags Script verification flags - * @return Total signature operation cost of tx - */ -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags); /** * Check whether all inputs of this transaction are valid (no double spends, scripts & sigs, amounts) @@ -376,26 +361,6 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); /** Transaction validation functions */ -/** Context-independent validity checks */ -bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); - -namespace Consensus { - -/** - * Check whether all inputs of this transaction are valid (no double spends and amounts) - * This does not modify the UTXO set. This does not check scripts and sigs. - * Preconditions: tx.IsCoinBase() is false. - */ -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight); - -} // namespace Consensus - -/** - * Check if transaction is final and can be included in a block with the - * specified height and time. Consensus critical. - */ -bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime); - /** * Check if transaction will be final in the next block to be created. * @@ -411,12 +376,6 @@ bool CheckFinalTx(const CTransaction &tx, int flags = -1); bool TestLockPointValidity(const LockPoints* lp); /** - * Check if transaction is final per BIP 68 sequence numbers and can be included in a block. - * Consensus critical. Takes as input a list of heights at which tx's inputs (in order) confirmed. - */ -bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); - -/** * Check if transaction will be BIP 68 final in the next block to be created. * * Simulates calling SequenceLocks() with data from the tip of the current active chain. @@ -447,8 +406,8 @@ private: public: CScriptCheck(): amount(0), ptxTo(0), nIn(0), nFlags(0), cacheStore(false), error(SCRIPT_ERR_UNKNOWN_ERROR) {} - CScriptCheck(const CCoins& txFromIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : - scriptPubKey(txFromIn.vout[txToIn.vin[nInIn].prevout.n].scriptPubKey), amount(txFromIn.vout[txToIn.vin[nInIn].prevout.n].nValue), + CScriptCheck(const CScript& scriptPubKeyIn, const CAmount amountIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : + scriptPubKey(scriptPubKeyIn), amount(amountIn), ptxTo(&txToIn), nIn(nInIn), nFlags(nFlagsIn), cacheStore(cacheIn), error(SCRIPT_ERR_UNKNOWN_ERROR), txdata(txdataIn) { } bool operator()(); @@ -485,18 +444,6 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime); bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); -/** Apply the effects of this block (with given index) on the UTXO set represented by coins. - * Validity checks that depend on the UTXO set are also done; ConnectBlock() - * can fail if those validity checks fail (among other reasons). */ -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, - const CChainParams& chainparams, bool fJustCheck = false); - -/** Undo the effects of this block (with given index) on the UTXO set represented by coins. - * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean - * will be true if no problems were found. Otherwise, the return value will be false in case - * of problems. Note that in any case, coins may be modified. */ -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& coins, bool* pfClean = NULL); - /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index d4121a28bc..46d7c9b329 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -14,39 +14,39 @@ CMainSignals& GetMainSignals() void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); - g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); - g_signals.UpdatedTransaction.connect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); + g_signals.TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); + g_signals.BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); + g_signals.BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.BlockChecked.connect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); g_signals.ScriptForMining.connect(boost::bind(&CValidationInterface::GetScriptForMining, pwalletIn, _1)); - g_signals.BlockFound.connect(boost::bind(&CValidationInterface::ResetRequestCount, pwalletIn, _1)); g_signals.NewPoWValidBlock.connect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterValidationInterface(CValidationInterface* pwalletIn) { - g_signals.BlockFound.disconnect(boost::bind(&CValidationInterface::ResetRequestCount, pwalletIn, _1)); g_signals.ScriptForMining.disconnect(boost::bind(&CValidationInterface::GetScriptForMining, pwalletIn, _1)); g_signals.BlockChecked.disconnect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.UpdatedTransaction.disconnect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); - g_signals.SyncTransaction.disconnect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); + g_signals.TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); + g_signals.BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); + g_signals.BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); g_signals.UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); g_signals.NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterAllValidationInterfaces() { - g_signals.BlockFound.disconnect_all_slots(); g_signals.ScriptForMining.disconnect_all_slots(); g_signals.BlockChecked.disconnect_all_slots(); g_signals.Broadcast.disconnect_all_slots(); g_signals.Inventory.disconnect_all_slots(); g_signals.SetBestChain.disconnect_all_slots(); - g_signals.UpdatedTransaction.disconnect_all_slots(); - g_signals.SyncTransaction.disconnect_all_slots(); + g_signals.TransactionAddedToMempool.disconnect_all_slots(); + g_signals.BlockConnected.disconnect_all_slots(); + g_signals.BlockDisconnected.disconnect_all_slots(); g_signals.UpdatedBlockTip.disconnect_all_slots(); g_signals.NewPoWValidBlock.disconnect_all_slots(); } diff --git a/src/validationinterface.h b/src/validationinterface.h index a494eb6990..460aecf243 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -7,16 +7,16 @@ #define BITCOIN_VALIDATIONINTERFACE_H #include <boost/signals2/signal.hpp> -#include <boost/shared_ptr.hpp> #include <memory> +#include "primitives/transaction.h" // CTransaction(Ref) + class CBlock; class CBlockIndex; struct CBlockLocator; class CBlockIndex; class CConnman; class CReserveScript; -class CTransaction; class CValidationInterface; class CValidationState; class uint256; @@ -33,14 +33,14 @@ void UnregisterAllValidationInterfaces(); class CValidationInterface { protected: virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} - virtual void SyncTransaction(const CTransaction &tx, const CBlockIndex *pindex, int posInBlock) {} + virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} + virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} + virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} virtual void SetBestChain(const CBlockLocator &locator) {} - virtual void UpdatedTransaction(const uint256 &hash) {} virtual void Inventory(const uint256 &hash) {} virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} virtual void BlockChecked(const CBlock&, const CValidationState&) {} - virtual void GetScriptForMining(boost::shared_ptr<CReserveScript>&) {}; - virtual void ResetRequestCount(const uint256 &hash) {}; + virtual void GetScriptForMining(std::shared_ptr<CReserveScript>&) {}; virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {}; friend void ::RegisterValidationInterface(CValidationInterface*); friend void ::UnregisterValidationInterface(CValidationInterface*); @@ -50,31 +50,30 @@ protected: struct CMainSignals { /** Notifies listeners of updated block chain tip */ boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; - /** A posInBlock value for SyncTransaction calls for transactions not - * included in connected blocks such as transactions removed from mempool, - * accepted to mempool or appearing in disconnected blocks.*/ - static const int SYNC_TRANSACTION_NOT_IN_BLOCK = -1; - /** Notifies listeners of updated transaction data (transaction, and - * optionally the block it is found in). Called with block data when - * transaction is included in a connected block, and without block data when - * transaction was accepted to mempool, removed from mempool (only when - * removal was due to conflict from connected block), or appeared in a - * disconnected block.*/ - boost::signals2::signal<void (const CTransaction &, const CBlockIndex *pindex, int posInBlock)> SyncTransaction; - /** Notifies listeners of an updated transaction without new data (for now: a coinbase potentially becoming visible). */ - boost::signals2::signal<void (const uint256 &)> UpdatedTransaction; + /** Notifies listeners of a transaction having been added to mempool. */ + boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; + /** + * Notifies listeners of a block being connected. + * Provides a vector of transactions evicted from the mempool as a result. + */ + boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &)> BlockConnected; + /** Notifies listeners of a block being disconnected */ + boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; /** Notifies listeners of a new active block chain. */ boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; /** Notifies listeners about an inventory item being seen on the network. */ boost::signals2::signal<void (const uint256 &)> Inventory; /** Tells listeners to broadcast their data. */ boost::signals2::signal<void (int64_t nBestBlockTime, CConnman* connman)> Broadcast; - /** Notifies listeners of a block validation result */ + /** + * Notifies listeners of a block validation result. + * If the provided CValidationState IsValid, the provided block + * is guaranteed to be the current best block at the time the + * callback was generated (not necessarily now) + */ boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; /** Notifies listeners that a key for mining is required (coinbase) */ - boost::signals2::signal<void (boost::shared_ptr<CReserveScript>&)> ScriptForMining; - /** Notifies listeners that a block has been successfully mined */ - boost::signals2::signal<void (const uint256 &)> BlockFound; + boost::signals2::signal<void (std::shared_ptr<CReserveScript>&)> ScriptForMining; /** * Notifies listeners that a block which builds directly on our current tip * has been received and connected to the headers tree, though not validated yet */ diff --git a/src/versionbits.cpp b/src/versionbits.cpp index d73f340510..4bb352f23c 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -3,10 +3,9 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "versionbits.h" - #include "consensus/params.h" -const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { +const struct VBDeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { { /*.name =*/ "testdummy", /*.gbt_force =*/ true, @@ -17,7 +16,7 @@ const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION }, { /*.name =*/ "segwit", - /*.gbt_force =*/ false, + /*.gbt_force =*/ true, } }; @@ -105,6 +104,36 @@ ThresholdState AbstractThresholdConditionChecker::GetStateFor(const CBlockIndex* return state; } +// return the numerical statistics of blocks signalling the specified BIP9 condition in this current period +BIP9Stats AbstractThresholdConditionChecker::GetStateStatisticsFor(const CBlockIndex* pindex, const Consensus::Params& params) const +{ + BIP9Stats stats; + + stats.period = Period(params); + stats.threshold = Threshold(params); + + if (pindex == NULL) + return stats; + + // Find beginning of period + const CBlockIndex* pindexEndOfPrevPeriod = pindex->GetAncestor(pindex->nHeight - ((pindex->nHeight + 1) % stats.period)); + stats.elapsed = pindex->nHeight - pindexEndOfPrevPeriod->nHeight; + + // Count from current block to beginning of period + int count = 0; + const CBlockIndex* currentIndex = pindex; + while (pindexEndOfPrevPeriod->nHeight != currentIndex->nHeight){ + if (Condition(currentIndex, params)) + count++; + currentIndex = currentIndex->pprev; + } + + stats.count = count; + stats.possible = (stats.period - stats.threshold ) >= (stats.elapsed - count); + + return stats; +} + int AbstractThresholdConditionChecker::GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const { const ThresholdState initialState = GetStateFor(pindexPrev, params, cache); @@ -167,6 +196,11 @@ ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus:: return VersionBitsConditionChecker(pos).GetStateFor(pindexPrev, params, cache.caches[pos]); } +BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + return VersionBitsConditionChecker(pos).GetStateStatisticsFor(pindexPrev, params); +} + int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache) { return VersionBitsConditionChecker(pos).GetStateSinceHeightFor(pindexPrev, params, cache.caches[pos]); diff --git a/src/versionbits.h b/src/versionbits.h index 7a929266aa..f4dfb71515 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -30,14 +30,22 @@ enum ThresholdState { // will either be NULL or a block with (height + 1) % Period() == 0. typedef std::map<const CBlockIndex*, ThresholdState> ThresholdConditionCache; -struct BIP9DeploymentInfo { +struct VBDeploymentInfo { /** Deployment name */ const char *name; /** Whether GBT clients can safely ignore this rule in simplified usage */ bool gbt_force; }; -extern const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[]; +struct BIP9Stats { + int period; + int threshold; + int elapsed; + int count; + bool possible; +}; + +extern const struct VBDeploymentInfo VersionBitsDeploymentInfo[]; /** * Abstract class that implements BIP9-style threshold logic, and caches results. @@ -51,6 +59,7 @@ protected: virtual int Threshold(const Consensus::Params& params) const =0; public: + BIP9Stats GetStateStatisticsFor(const CBlockIndex* pindex, const Consensus::Params& params) const; // Note that the functions below take a pindexPrev as input: they compute information for block B based on its parent. ThresholdState GetStateFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; int GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; @@ -64,6 +73,7 @@ struct VersionBitsCache }; ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); +BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos); int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); uint32_t VersionBitsMask(const Consensus::Params& params, Consensus::DeploymentPos pos); diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index eaf4ff8062..cb4719ae90 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -5,7 +5,9 @@ #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H +#include "policy/feerate.h" #include "primitives/transaction.h" +#include "wallet/wallet.h" /** Coin Control Features. */ class CCoinControl @@ -16,14 +18,14 @@ public: bool fAllowOtherInputs; //! Includes watch only addresses which match the ISMINE_WATCH_SOLVABLE criteria bool fAllowWatchOnly; - //! Minimum absolute fee (not per kilobyte) - CAmount nMinimumTotalFee; //! Override estimated feerate bool fOverrideFeeRate; //! Feerate to use if overrideFeeRate is true CFeeRate nFeeRate; //! Override the default confirmation target, 0 = use default int nConfirmTarget; + //! Signal BIP-125 replace by fee. + bool signalRbf; CCoinControl() { @@ -36,10 +38,10 @@ public: fAllowOtherInputs = false; fAllowWatchOnly = false; setSelected.clear(); - nMinimumTotalFee = 0; nFeeRate = CFeeRate(0); fOverrideFeeRate = false; nConfirmTarget = 0; + signalRbf = fWalletRbf; } bool HasSelected() const diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 7d1b429b30..25f6bdd9d9 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -6,6 +6,7 @@ #include "db.h" #include "addrman.h" +#include "fs.h" #include "hash.h" #include "protocol.h" #include "util.h" @@ -17,13 +18,10 @@ #include <sys/stat.h> #endif -#include <boost/filesystem.hpp> +#include <boost/foreach.hpp> #include <boost/thread.hpp> #include <boost/version.hpp> -using namespace std; - - // // CDB // @@ -68,7 +66,7 @@ void CDBEnv::Close() EnvShutdown(); } -bool CDBEnv::Open(const boost::filesystem::path& pathIn) +bool CDBEnv::Open(const fs::path& pathIn) { if (fDbEnvInit) return true; @@ -76,9 +74,9 @@ bool CDBEnv::Open(const boost::filesystem::path& pathIn) boost::this_thread::interruption_point(); strPath = pathIn.string(); - boost::filesystem::path pathLogDir = pathIn / "database"; + fs::path pathLogDir = pathIn / "database"; TryCreateDirectory(pathLogDir); - boost::filesystem::path pathErrorFile = pathIn / "db.log"; + fs::path pathErrorFile = pathIn / "db.log"; LogPrintf("CDBEnv::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); unsigned int nEnvFlags = 0; @@ -91,7 +89,7 @@ bool CDBEnv::Open(const boost::filesystem::path& pathIn) dbenv->set_lg_max(1048576); dbenv->set_lk_max_locks(40000); dbenv->set_lk_max_objects(40000); - dbenv->set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug + dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); @@ -116,11 +114,11 @@ bool CDBEnv::Open(const boost::filesystem::path& pathIn) void CDBEnv::MakeMock() { if (fDbEnvInit) - throw runtime_error("CDBEnv::MakeMock: Already initialized"); + throw std::runtime_error("CDBEnv::MakeMock: Already initialized"); boost::this_thread::interruption_point(); - LogPrint("db", "CDBEnv::MakeMock\n"); + LogPrint(BCLog::DB, "CDBEnv::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); @@ -139,13 +137,13 @@ void CDBEnv::MakeMock() DB_PRIVATE, S_IRUSR | S_IWUSR); if (ret > 0) - throw runtime_error(strprintf("CDBEnv::MakeMock: Error %d opening database environment.", ret)); + throw std::runtime_error(strprintf("CDBEnv::MakeMock: Error %d opening database environment.", ret)); fDbEnvInit = true; fMockDb = true; } -CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile)) +CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(const std::string& strFile)) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); @@ -158,10 +156,134 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFu return RECOVER_FAIL; // Try to recover: - bool fRecovered = (*recoverFunc)(*this, strFile); + bool fRecovered = (*recoverFunc)(strFile); return (fRecovered ? RECOVER_OK : RECOVER_FAIL); } +bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)) +{ + // Recovery procedure: + // move wallet file to wallet.timestamp.bak + // Call Salvage with fAggressive=true to + // get as much data as possible. + // Rewrite salvaged data to fresh wallet file + // Set -rescan so any missing transactions will be + // found. + int64_t now = GetTime(); + std::string newFilename = strprintf("wallet.%d.bak", now); + + int result = bitdb.dbenv->dbrename(NULL, filename.c_str(), NULL, + newFilename.c_str(), DB_AUTO_COMMIT); + if (result == 0) + LogPrintf("Renamed %s to %s\n", filename, newFilename); + else + { + LogPrintf("Failed to rename %s to %s\n", filename, newFilename); + return false; + } + + std::vector<CDBEnv::KeyValPair> salvagedData; + bool fSuccess = bitdb.Salvage(newFilename, true, salvagedData); + if (salvagedData.empty()) + { + LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + return false; + } + LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); + + std::unique_ptr<Db> pdbCopy(new Db(bitdb.dbenv, 0)); + int ret = pdbCopy->open(NULL, // Txn pointer + filename.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) + { + LogPrintf("Cannot create database file %s\n", filename); + return false; + } + + DbTxn* ptxn = bitdb.TxnBegin(); + BOOST_FOREACH(CDBEnv::KeyValPair& row, salvagedData) + { + if (recoverKVcallback) + { + CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); + CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); + std::string strType, strErr; + if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) + continue; + } + Dbt datKey(&row.first[0], row.first.size()); + Dbt datValue(&row.second[0], row.second.size()); + int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) + fSuccess = false; + } + ptxn->commit(0); + pdbCopy->close(0); + + return fSuccess; +} + +bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +{ + LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); + LogPrintf("Using wallet %s\n", walletFile); + + // Wallet file must be a plain filename without a directory + if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) + { + errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string()); + return false; + } + + if (!bitdb.Open(dataDir)) + { + // try moving the database env out of the way + fs::path pathDatabase = dataDir / "database"; + fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime()); + try { + fs::rename(pathDatabase, pathDatabaseBak); + LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); + } catch (const fs::filesystem_error&) { + // failure is ok (well, not really, but it's not worse than what we started with) + } + + // try again + if (!bitdb.Open(dataDir)) { + // if it still fails, it probably means we can't even create the database env + errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir()); + return false; + } + } + return true; +} + +bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile)) +{ + if (fs::exists(dataDir / walletFile)) + { + CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc); + if (r == CDBEnv::RECOVER_OK) + { + warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!" + " Original %s saved as %s in %s; if" + " your balance or transactions are incorrect you should" + " restore from a backup."), + walletFile, "wallet.{timestamp}.bak", dataDir); + } + if (r == CDBEnv::RECOVER_FAIL) + { + errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); + return false; + } + } + // also return true if files does not exists + return true; +} + /* End of headers, beginning of key/value data */ static const char *HEADER_END = "HEADER=END"; /* End of key/value data */ @@ -176,7 +298,7 @@ bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<C if (fAggressive) flags |= DB_AGGRESSIVE; - stringstream strDump; + std::stringstream strDump; Db db(dbenv, 0); int result = db.verify(strFile.c_str(), NULL, &strDump, flags); @@ -200,7 +322,7 @@ bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<C // ... repeated // DATA=END - string strLine; + std::string strLine; while (!strDump.eof() && strLine != HEADER_END) getline(strDump, strLine); // Skip past header @@ -237,13 +359,16 @@ void CDBEnv::CheckpointLSN(const std::string& strFile) } -CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL) +CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL) { int ret; fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; - if (strFilename.empty()) + env = dbw.env; + if (dbw.IsDummy()) { return; + } + const std::string &strFilename = dbw.strFile; bool fCreate = strchr(pszMode, 'c') != NULL; unsigned int nFlags = DB_THREAD; @@ -251,22 +376,22 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose nFlags |= DB_CREATE; { - LOCK(bitdb.cs_db); - if (!bitdb.Open(GetDataDir())) - throw runtime_error("CDB: Failed to open database environment."); + LOCK(env->cs_db); + if (!env->Open(GetDataDir())) + throw std::runtime_error("CDB: Failed to open database environment."); strFile = strFilename; - ++bitdb.mapFileUseCount[strFile]; - pdb = bitdb.mapDb[strFile]; + ++env->mapFileUseCount[strFile]; + pdb = env->mapDb[strFile]; if (pdb == NULL) { - pdb = new Db(bitdb.dbenv, 0); + pdb = new Db(env->dbenv, 0); - bool fMockDb = bitdb.IsMock(); + bool fMockDb = env->IsMock(); if (fMockDb) { DbMpoolFile* mpf = pdb->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); if (ret != 0) - throw runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile)); + throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile)); } ret = pdb->open(NULL, // Txn pointer @@ -279,19 +404,19 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose if (ret != 0) { delete pdb; pdb = NULL; - --bitdb.mapFileUseCount[strFile]; + --env->mapFileUseCount[strFile]; strFile = ""; - throw runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); + throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } - if (fCreate && !Exists(string("version"))) { + if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; fReadOnly = false; WriteVersion(CLIENT_VERSION); fReadOnly = fTmp; } - bitdb.mapDb[strFile] = pdb; + env->mapDb[strFile] = pdb; } } } @@ -306,7 +431,7 @@ void CDB::Flush() if (fReadOnly) nMinutes = 1; - bitdb.dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + env->dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } void CDB::Close() @@ -322,12 +447,12 @@ void CDB::Close() Flush(); { - LOCK(bitdb.cs_db); - --bitdb.mapFileUseCount[strFile]; + LOCK(env->cs_db); + --env->mapFileUseCount[strFile]; } } -void CDBEnv::CloseDb(const string& strFile) +void CDBEnv::CloseDb(const std::string& strFile) { { LOCK(cs_db); @@ -341,32 +466,28 @@ void CDBEnv::CloseDb(const string& strFile) } } -bool CDBEnv::RemoveDb(const string& strFile) -{ - this->CloseDb(strFile); - - LOCK(cs_db); - int rc = dbenv->dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT); - return (rc == 0); -} - -bool CDB::Rewrite(const string& strFile, const char* pszSkip) +bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip) { + if (dbw.IsDummy()) { + return true; + } + CDBEnv *env = dbw.env; + const std::string& strFile = dbw.strFile; while (true) { { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) { + LOCK(env->cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); - bitdb.mapFileUseCount.erase(strFile); + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); bool fSuccess = true; LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile); - string strFileRes = strFile + ".rewrite"; + std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} - CDB db(strFile.c_str(), "r"); - Db* pdbCopy = new Db(bitdb.dbenv, 0); + CDB db(dbw, "r"); + Db* pdbCopy = new Db(env->dbenv, 0); int ret = pdbCopy->open(NULL, // Txn pointer strFileRes.c_str(), // Filename @@ -409,17 +530,17 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip) } if (fSuccess) { db.Close(); - bitdb.CloseDb(strFile); + env->CloseDb(strFile); if (pdbCopy->close(0)) fSuccess = false; delete pdbCopy; } } if (fSuccess) { - Db dbA(bitdb.dbenv, 0); + Db dbA(env->dbenv, 0); if (dbA.remove(strFile.c_str(), NULL, 0)) fSuccess = false; - Db dbB(bitdb.dbenv, 0); + Db dbB(env->dbenv, 0); if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) fSuccess = false; } @@ -438,38 +559,130 @@ void CDBEnv::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use - LogPrint("db", "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); + LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) return; { LOCK(cs_db); - map<string, int>::iterator mi = mapFileUseCount.begin(); + std::map<std::string, int>::iterator mi = mapFileUseCount.begin(); while (mi != mapFileUseCount.end()) { - string strFile = (*mi).first; + std::string strFile = (*mi).first; int nRefCount = (*mi).second; - LogPrint("db", "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); + LogPrint(BCLog::DB, "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); - LogPrint("db", "CDBEnv::Flush: %s checkpoint\n", strFile); + LogPrint(BCLog::DB, "CDBEnv::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); - LogPrint("db", "CDBEnv::Flush: %s detach\n", strFile); + LogPrint(BCLog::DB, "CDBEnv::Flush: %s detach\n", strFile); if (!fMockDb) dbenv->lsn_reset(strFile.c_str(), 0); - LogPrint("db", "CDBEnv::Flush: %s closed\n", strFile); + LogPrint(BCLog::DB, "CDBEnv::Flush: %s closed\n", strFile); mapFileUseCount.erase(mi++); } else mi++; } - LogPrint("db", "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); + LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char** listp; if (mapFileUseCount.empty()) { dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); if (!fMockDb) - boost::filesystem::remove_all(boost::filesystem::path(strPath) / "database"); + fs::remove_all(fs::path(strPath) / "database"); + } + } + } +} + +bool CDB::PeriodicFlush(CWalletDBWrapper& dbw) +{ + if (dbw.IsDummy()) { + return true; + } + bool ret = false; + CDBEnv *env = dbw.env; + const std::string& strFile = dbw.strFile; + TRY_LOCK(bitdb.cs_db,lockDb); + if (lockDb) + { + // Don't do this if any databases are in use + int nRefCount = 0; + std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin(); + while (mit != env->mapFileUseCount.end()) + { + nRefCount += (*mit).second; + mit++; + } + + if (nRefCount == 0) + { + boost::this_thread::interruption_point(); + std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); + if (mi != env->mapFileUseCount.end()) + { + LogPrint(BCLog::DB, "Flushing %s\n", strFile); + int64_t nStart = GetTimeMillis(); + + // Flush wallet file so it's self contained + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + + env->mapFileUseCount.erase(mi++); + LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + ret = true; } } } + + return ret; +} + +bool CWalletDBWrapper::Rewrite(const char* pszSkip) +{ + return CDB::Rewrite(*this, pszSkip); +} + +bool CWalletDBWrapper::Backup(const std::string& strDest) +{ + if (IsDummy()) { + return false; + } + while (true) + { + { + LOCK(env->cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) + { + // Flush log data to the dat file + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); + + // Copy wallet file + fs::path pathSrc = GetDataDir() / strFile; + fs::path pathDest(strDest); + if (fs::is_directory(pathDest)) + pathDest /= strFile; + + try { + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); + LogPrintf("copied %s to %s\n", strFile, pathDest.string()); + return true; + } catch (const fs::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what()); + return false; + } + } + } + MilliSleep(100); + } + return false; +} + +void CWalletDBWrapper::Flush(bool shutdown) +{ + if (!IsDummy()) { + env->Flush(shutdown); + } } diff --git a/src/wallet/db.h b/src/wallet/db.h index b4ce044e7f..3c6870d169 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -7,6 +7,7 @@ #define BITCOIN_WALLET_DB_H #include "clientversion.h" +#include "fs.h" #include "serialize.h" #include "streams.h" #include "sync.h" @@ -16,8 +17,6 @@ #include <string> #include <vector> -#include <boost/filesystem/path.hpp> - #include <db_cxx.h> static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; @@ -28,7 +27,7 @@ class CDBEnv private: bool fDbEnvInit; bool fMockDb; - // Don't change into boost::filesystem::path, as that can result in + // Don't change into fs::path, as that can result in // shutdown problems/crashes caused by a static initialized internal pointer. std::string strPath; @@ -56,7 +55,7 @@ public: enum VerifyResult { VERIFY_OK, RECOVER_OK, RECOVER_FAIL }; - VerifyResult Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile)); + VerifyResult Verify(const std::string& strFile, bool (*recoverFunc)(const std::string& strFile)); /** * Salvage data from a file that Verify says is bad. * fAggressive sets the DB_AGGRESSIVE flag (see berkeley DB->verify() method documentation). @@ -67,13 +66,12 @@ public: typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; bool Salvage(const std::string& strFile, bool fAggressive, std::vector<KeyValPair>& vResult); - bool Open(const boost::filesystem::path& path); + bool Open(const fs::path& path); void Close(); void Flush(bool fShutdown); void CheckpointLSN(const std::string& strFile); void CloseDb(const std::string& strFile); - bool RemoveDb(const std::string& strFile); DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) { @@ -87,6 +85,52 @@ public: extern CDBEnv bitdb; +/** An instance of this class represents one database. + * For BerkeleyDB this is just a (env, strFile) tuple. + **/ +class CWalletDBWrapper +{ + friend class CDB; +public: + /** Create dummy DB handle */ + CWalletDBWrapper(): env(nullptr) + { + } + + /** Create DB handle to real database */ + CWalletDBWrapper(CDBEnv *env_in, const std::string &strFile_in): + env(env_in), strFile(strFile_in) + { + } + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip=nullptr); + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest); + + /** Get a name for this database, for debugging etc. + */ + std::string GetName() const { return strFile; } + + /** Make sure all changes are flushed to disk. + */ + void Flush(bool shutdown); + +private: + /** BerkeleyDB specific */ + CDBEnv *env; + std::string strFile; + + /** Return whether this database handle is a dummy for testing. + * Only to be used at a low level, application should ideally not care + * about this. + */ + bool IsDummy() { return env == nullptr; } +}; + /** RAII class that provides access to a Berkeley database */ class CDB @@ -97,19 +141,29 @@ protected: DbTxn* activeTxn; bool fReadOnly; bool fFlushOnClose; + CDBEnv *env; - explicit CDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnCloseIn=true); +public: + explicit CDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool fFlushOnCloseIn=true); ~CDB() { Close(); } -public: void Flush(); void Close(); + static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)); + + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + static bool PeriodicFlush(CWalletDBWrapper& dbw); + /* verifies the database environment */ + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + /* verifies the database file */ + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile)); private: CDB(const CDB&); void operator=(const CDB&); -protected: +public: template <typename K, typename T> bool Read(const K& key, T& value) { @@ -126,29 +180,30 @@ protected: Dbt datValue; datValue.set_flags(DB_DBT_MALLOC); int ret = pdb->get(activeTxn, &datKey, &datValue, 0); - memset(datKey.get_data(), 0, datKey.get_size()); - if (datValue.get_data() == NULL) - return false; - - // Unserialize value - try { - CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); - ssValue >> value; - } catch (const std::exception&) { - return false; + memory_cleanse(datKey.get_data(), datKey.get_size()); + bool success = false; + if (datValue.get_data() != NULL) { + // Unserialize value + try { + CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); + ssValue >> value; + success = true; + } catch (const std::exception&) { + // In this case success remains 'false' + } + + // Clear and free memory + memory_cleanse(datValue.get_data(), datValue.get_size()); + free(datValue.get_data()); } - - // Clear and free memory - memset(datValue.get_data(), 0, datValue.get_size()); - free(datValue.get_data()); - return (ret == 0); + return ret == 0 && success; } template <typename K, typename T> bool Write(const K& key, const T& value, bool fOverwrite = true) { if (!pdb) - return false; + return true; if (fReadOnly) assert(!"Write called on database in read-only mode"); @@ -168,8 +223,8 @@ protected: int ret = pdb->put(activeTxn, &datKey, &datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); // Clear memory in case it was a private key - memset(datKey.get_data(), 0, datKey.get_size()); - memset(datValue.get_data(), 0, datValue.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); + memory_cleanse(datValue.get_data(), datValue.get_size()); return (ret == 0); } @@ -191,7 +246,7 @@ protected: int ret = pdb->del(activeTxn, &datKey, 0); // Clear memory - memset(datKey.get_data(), 0, datKey.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); return (ret == 0 || ret == DB_NOTFOUND); } @@ -211,7 +266,7 @@ protected: int ret = pdb->exists(activeTxn, &datKey, 0); // Clear memory - memset(datKey.get_data(), 0, datKey.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); return (ret == 0); } @@ -254,8 +309,8 @@ protected: ssValue.write((char*)datValue.get_data(), datValue.get_size()); // Clear and free memory - memset(datKey.get_data(), 0, datKey.get_size()); - memset(datValue.get_data(), 0, datValue.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); + memory_cleanse(datValue.get_data(), datValue.get_size()); free(datKey.get_data()); free(datValue.get_data()); return 0; @@ -302,7 +357,7 @@ public: return Write(std::string("version"), nVersion); } - bool static Rewrite(const std::string& strFile, const char* pszSkip = NULL); + bool static Rewrite(CWalletDBWrapper& dbw, const char* pszSkip = NULL); }; #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp new file mode 100644 index 0000000000..46ef87b7b1 --- /dev/null +++ b/src/wallet/feebumper.cpp @@ -0,0 +1,288 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "consensus/validation.h" +#include "wallet/feebumper.h" +#include "wallet/wallet.h" +#include "policy/fees.h" +#include "policy/policy.h" +#include "policy/rbf.h" +#include "validation.h" //for mempool access +#include "txmempool.h" +#include "utilmoneystr.h" +#include "util.h" +#include "net.h" + +// Calculate the size of the transaction assuming all signatures are max size +// Use DummySignatureCreator, which inserts 72 byte signatures everywhere. +// TODO: re-use this in CWallet::CreateTransaction (right now +// CreateTransaction uses the constructed dummy-signed tx to do a priority +// calculation, but we should be able to refactor after priority is removed). +// NOTE: this requires that all inputs must be in mapWallet (eg the tx should +// be IsAllFromMe). +int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWallet) +{ + CMutableTransaction txNew(tx); + std::vector<CInputCoin> vCoins; + // Look up the inputs. We should have already checked that this transaction + // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our + // wallet, with a valid index into the vout array. + for (auto& input : tx.vin) { + const auto mi = pWallet->mapWallet.find(input.prevout.hash); + assert(mi != pWallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); + vCoins.emplace_back(CInputCoin(&(mi->second), input.prevout.n)); + } + if (!pWallet->DummySignTx(txNew, vCoins)) { + // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) + // implies that we can sign for every input. + return -1; + } + return GetVirtualTransactionSize(txNew); +} + +bool CFeeBumper::preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx) { + if (pWallet->HasWalletSpend(wtx.GetHash())) { + vErrors.push_back("Transaction has descendants in the wallet"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return false; + } + + { + LOCK(mempool.cs); + auto it_mp = mempool.mapTx.find(wtx.GetHash()); + if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { + vErrors.push_back("Transaction has descendants in the mempool"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return false; + } + } + + if (wtx.GetDepthInMainChain() != 0) { + vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + currentResult = BumpFeeResult::WALLET_ERROR; + return false; + } + return true; +} + +CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable) + : + txid(std::move(txidIn)), + nOldFee(0), + nNewFee(0) +{ + vErrors.clear(); + bumpedTxid.SetNull(); + AssertLockHeld(pWallet->cs_wallet); + if (!pWallet->mapWallet.count(txid)) { + vErrors.push_back("Invalid or non-wallet transaction id"); + currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; + return; + } + auto it = pWallet->mapWallet.find(txid); + const CWalletTx& wtx = it->second; + + if (!preconditionChecks(pWallet, wtx)) { + return; + } + + if (!SignalsOptInRBF(wtx)) { + vErrors.push_back("Transaction is not BIP 125 replaceable"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + if (wtx.mapValue.count("replaced_by_txid")) { + vErrors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", txid.ToString(), wtx.mapValue.at("replaced_by_txid"))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // check that original tx consists entirely of our inputs + // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) + if (!pWallet->IsAllFromMe(wtx, ISMINE_SPENDABLE)) { + vErrors.push_back("Transaction contains inputs that don't belong to this wallet"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // figure out which output was change + // if there was no change output or multiple change outputs, fail + int nOutput = -1; + for (size_t i = 0; i < wtx.tx->vout.size(); ++i) { + if (pWallet->IsChange(wtx.tx->vout[i])) { + if (nOutput != -1) { + vErrors.push_back("Transaction has multiple change outputs"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + nOutput = i; + } + } + if (nOutput == -1) { + vErrors.push_back("Transaction does not have a change output"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // Calculate the expected size of the new transaction. + int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); + const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, pWallet); + if (maxNewTxSize < 0) { + vErrors.push_back("Transaction contains inputs that cannot be signed"); + currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; + return; + } + + // calculate the old fee and fee-rate + nOldFee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + CFeeRate nOldFeeRate(nOldFee, txSize); + CFeeRate nNewFeeRate; + // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to + // future proof against changes to network wide policy for incremental relay + // fee that our node may not be aware of. + CFeeRate walletIncrementalRelayFee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); + if (::incrementalRelayFee > walletIncrementalRelayFee) { + walletIncrementalRelayFee = ::incrementalRelayFee; + } + + if (totalFee > 0) { + CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); + if (totalFee < minTotalFee) { + vErrors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", + FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return; + } + CAmount requiredFee = CWallet::GetRequiredFee(maxNewTxSize); + if (totalFee < requiredFee) { + vErrors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", + FormatMoney(requiredFee))); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return; + } + nNewFee = totalFee; + nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); + } else { + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, ignoreGlobalPayTxFee); + nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); + + // New fee rate must be at least old rate + minimum incremental relay rate + // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized + // in that unit (fee per kb). + // However, nOldFeeRate is a calculated value from the tx fee/size, so + // add 1 satoshi to the result, because it may have been rounded down. + if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { + nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); + nNewFee = nNewFeeRate.GetFee(maxNewTxSize); + } + } + + // Check that in all cases the new fee doesn't violate maxTxFee + if (nNewFee > maxTxFee) { + vErrors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", + FormatMoney(nNewFee), FormatMoney(maxTxFee))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // check that fee rate is higher than mempool's minimum fee + // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) + // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, + // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a + // moment earlier. In this case, we report an error to the user, who may use totalFee to make an adjustment. + CFeeRate minMempoolFeeRate = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { + vErrors.push_back(strprintf("New fee rate (%s) is less than the minimum fee rate (%s) to get into the mempool. totalFee value should to be at least %s or settxfee value should be at least %s to add transaction.", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), FormatMoney(minMempoolFeeRate.GetFeePerK()))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // Now modify the output to increase the fee. + // If the output is not large enough to pay the fee, fail. + CAmount nDelta = nNewFee - nOldFee; + assert(nDelta > 0); + mtx = *wtx.tx; + CTxOut* poutput = &(mtx.vout[nOutput]); + if (poutput->nValue < nDelta) { + vErrors.push_back("Change output is too small to bump the fee"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // If the output would become dust, discard it (converting the dust to fee) + poutput->nValue -= nDelta; + if (poutput->nValue <= GetDustThreshold(*poutput, ::dustRelayFee)) { + LogPrint(BCLog::RPC, "Bumping fee and discarding dust output\n"); + nNewFee += poutput->nValue; + mtx.vout.erase(mtx.vout.begin() + nOutput); + } + + // Mark new tx not replaceable, if requested. + if (!newTxReplaceable) { + for (auto& input : mtx.vin) { + if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; + } + } + + currentResult = BumpFeeResult::OK; +} + +bool CFeeBumper::signTransaction(CWallet *pWallet) +{ + return pWallet->SignTransaction(mtx); +} + +bool CFeeBumper::commit(CWallet *pWallet) +{ + AssertLockHeld(pWallet->cs_wallet); + if (!vErrors.empty() || currentResult != BumpFeeResult::OK) { + return false; + } + if (txid.IsNull() || !pWallet->mapWallet.count(txid)) { + vErrors.push_back("Invalid or non-wallet transaction id"); + currentResult = BumpFeeResult::MISC_ERROR; + return false; + } + CWalletTx& oldWtx = pWallet->mapWallet[txid]; + + // make sure the transaction still has no descendants and hasn't been mined in the meantime + if (!preconditionChecks(pWallet, oldWtx)) { + return false; + } + + CWalletTx wtxBumped(pWallet, MakeTransactionRef(std::move(mtx))); + // commit/broadcast the tx + CReserveKey reservekey(pWallet); + wtxBumped.mapValue = oldWtx.mapValue; + wtxBumped.mapValue["replaces_txid"] = oldWtx.GetHash().ToString(); + wtxBumped.vOrderForm = oldWtx.vOrderForm; + wtxBumped.strFromAccount = oldWtx.strFromAccount; + wtxBumped.fTimeReceivedIsTxTime = true; + wtxBumped.fFromMe = true; + CValidationState state; + if (!pWallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { + // NOTE: CommitTransaction never returns false, so this should never happen. + vErrors.push_back(strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); + return false; + } + + bumpedTxid = wtxBumped.GetHash(); + if (state.IsInvalid()) { + // This can happen if the mempool rejected the transaction. Report + // what happened in the "errors" response. + vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); + } + + // mark the original tx as bumped + if (!pWallet->MarkReplaced(oldWtx.GetHash(), wtxBumped.GetHash())) { + // TODO: see if JSON-RPC has a standard way of returning a response + // along with an exception. It would be good to return information about + // wtxBumped to the caller even if marking the original transaction + // replaced does not succeed for some reason. + vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); + } + return true; +} + diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h new file mode 100644 index 0000000000..fc32316704 --- /dev/null +++ b/src/wallet/feebumper.h @@ -0,0 +1,59 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_FEEBUMPER_H +#define BITCOIN_WALLET_FEEBUMPER_H + +#include <primitives/transaction.h> + +class CWallet; +class CWalletTx; +class uint256; + +enum class BumpFeeResult +{ + OK, + INVALID_ADDRESS_OR_KEY, + INVALID_REQUEST, + INVALID_PARAMETER, + WALLET_ERROR, + MISC_ERROR, +}; + +class CFeeBumper +{ +public: + CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable); + BumpFeeResult getResult() const { return currentResult; } + const std::vector<std::string>& getErrors() const { return vErrors; } + CAmount getOldFee() const { return nOldFee; } + CAmount getNewFee() const { return nNewFee; } + uint256 getBumpedTxId() const { return bumpedTxid; } + + /* signs the new transaction, + * returns false if the tx couldn't be found or if it was + * impossible to create the signature(s) + */ + bool signTransaction(CWallet *pWallet); + + /* commits the fee bump, + * returns true, in case of CWallet::CommitTransaction was successful + * but, eventually sets vErrors if the tx could not be added to the mempool (will try later) + * or if the old transaction could not be marked as replaced + */ + bool commit(CWallet *pWalletNonConst); + +private: + bool preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx); + + const uint256 txid; + uint256 bumpedTxid; + CMutableTransaction mtx; + std::vector<std::string> vErrors; + BumpFeeResult currentResult; + CAmount nOldFee; + CAmount nNewFee; +}; + +#endif // BITCOIN_WALLET_FEEBUMPER_H diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 20a3cbda1e..457e937bb8 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -16,6 +16,8 @@ #include "merkleblock.h" #include "core_io.h" +#include "rpcwallet.h" + #include <fstream> #include <stdint.h> @@ -27,11 +29,6 @@ #include <boost/assign/list_of.hpp> #include <boost/foreach.hpp> -using namespace std; - -void EnsureWalletIsUnlocked(); -bool EnsureWalletIsAvailable(bool avoidException); - std::string static EncodeDumpTime(int64_t nTime) { return DateTimeStrFormat("%Y-%m-%dT%H:%M:%SZ", nTime); } @@ -77,11 +74,13 @@ std::string DecodeDumpString(const std::string &str) { UniValue importprivkey(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; - + } + if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) - throw runtime_error( + throw std::runtime_error( "importprivkey \"bitcoinprivkey\" ( \"label\" ) ( rescan )\n" "\nAdds a private key (as returned by dumpprivkey) to your wallet.\n" "\nArguments:\n" @@ -96,17 +95,19 @@ UniValue importprivkey(const JSONRPCRequest& request) + HelpExampleCli("importprivkey", "\"mykey\"") + "\nImport using a label and without rescan\n" + HelpExampleCli("importprivkey", "\"mykey\" \"testing\" false") + + "\nImport using default blank label and without rescan\n" + + HelpExampleCli("importprivkey", "\"mykey\" \"\" false") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("importprivkey", "\"mykey\", \"testing\", false") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - EnsureWalletIsUnlocked(); + EnsureWalletIsUnlocked(pwallet); - string strSecret = request.params[0].get_str(); - string strLabel = ""; + std::string strSecret = request.params[0].get_str(); + std::string strLabel = ""; if (request.params.size() > 1) strLabel = request.params[1].get_str(); @@ -130,68 +131,100 @@ UniValue importprivkey(const JSONRPCRequest& request) assert(key.VerifyPubKey(pubkey)); CKeyID vchAddress = pubkey.GetID(); { - pwalletMain->MarkDirty(); - pwalletMain->SetAddressBook(vchAddress, strLabel, "receive"); + pwallet->MarkDirty(); + pwallet->SetAddressBook(vchAddress, strLabel, "receive"); // Don't throw error in case a key is already there - if (pwalletMain->HaveKey(vchAddress)) + if (pwallet->HaveKey(vchAddress)) { return NullUniValue; + } - pwalletMain->mapKeyMetadata[vchAddress].nCreateTime = 1; + pwallet->mapKeyMetadata[vchAddress].nCreateTime = 1; - if (!pwalletMain->AddKeyPubKey(key, pubkey)) + if (!pwallet->AddKeyPubKey(key, pubkey)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); + } // whenever a key is imported, we need to scan the whole chain - pwalletMain->UpdateTimeFirstKey(1); + pwallet->UpdateTimeFirstKey(1); if (fRescan) { - pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true); + pwallet->ScanForWalletTransactions(chainActive.Genesis(), true); } } return NullUniValue; } -void ImportAddress(const CBitcoinAddress& address, const string& strLabel); -void ImportScript(const CScript& script, const string& strLabel, bool isRedeemScript) +UniValue abortrescan(const JSONRPCRequest& request) +{ + CWallet* const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + if (request.fHelp || request.params.size() > 0) + throw std::runtime_error( + "abortrescan\n" + "\nStops current wallet rescan triggered e.g. by an importprivkey call.\n" + "\nExamples:\n" + "\nImport a private key\n" + + HelpExampleCli("importprivkey", "\"mykey\"") + + "\nAbort the running wallet rescan\n" + + HelpExampleCli("abortrescan", "") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("abortrescan", "") + ); + + if (!pwallet->IsScanning() || pwallet->IsAbortingRescan()) return false; + pwallet->AbortRescan(); + return true; +} + +void ImportAddress(CWallet*, const CBitcoinAddress& address, const std::string& strLabel); +void ImportScript(CWallet* const pwallet, const CScript& script, const std::string& strLabel, bool isRedeemScript) { - if (!isRedeemScript && ::IsMine(*pwalletMain, script) == ISMINE_SPENDABLE) + if (!isRedeemScript && ::IsMine(*pwallet, script) == ISMINE_SPENDABLE) { throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); + } - pwalletMain->MarkDirty(); + pwallet->MarkDirty(); - if (!pwalletMain->HaveWatchOnly(script) && !pwalletMain->AddWatchOnly(script, 0 /* nCreateTime */)) + if (!pwallet->HaveWatchOnly(script) && !pwallet->AddWatchOnly(script, 0 /* nCreateTime */)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } if (isRedeemScript) { - if (!pwalletMain->HaveCScript(script) && !pwalletMain->AddCScript(script)) + if (!pwallet->HaveCScript(script) && !pwallet->AddCScript(script)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding p2sh redeemScript to wallet"); - ImportAddress(CBitcoinAddress(CScriptID(script)), strLabel); + } + ImportAddress(pwallet, CBitcoinAddress(CScriptID(script)), strLabel); } else { CTxDestination destination; if (ExtractDestination(script, destination)) { - pwalletMain->SetAddressBook(destination, strLabel, "receive"); + pwallet->SetAddressBook(destination, strLabel, "receive"); } } } -void ImportAddress(const CBitcoinAddress& address, const string& strLabel) +void ImportAddress(CWallet* const pwallet, const CBitcoinAddress& address, const std::string& strLabel) { CScript script = GetScriptForDestination(address.Get()); - ImportScript(script, strLabel, false); + ImportScript(pwallet, script, strLabel, false); // add to address book or update label if (address.IsValid()) - pwalletMain->SetAddressBook(address.Get(), strLabel, "receive"); + pwallet->SetAddressBook(address.Get(), strLabel, "receive"); } UniValue importaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; - + } + if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) - throw runtime_error( + throw std::runtime_error( "importaddress \"address\" ( \"label\" rescan p2sh )\n" "\nAdds a script (in hex) or address that can be watched as if it were in your wallet but cannot be used to spend.\n" "\nArguments:\n" @@ -213,7 +246,7 @@ UniValue importaddress(const JSONRPCRequest& request) ); - string strLabel = ""; + std::string strLabel = ""; if (request.params.size() > 1) strLabel = request.params[1].get_str(); @@ -230,24 +263,24 @@ UniValue importaddress(const JSONRPCRequest& request) if (request.params.size() > 3) fP2SH = request.params[3].get_bool(); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); CBitcoinAddress address(request.params[0].get_str()); if (address.IsValid()) { if (fP2SH) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Cannot use the p2sh flag with an address - use a script instead"); - ImportAddress(address, strLabel); + ImportAddress(pwallet, address, strLabel); } else if (IsHex(request.params[0].get_str())) { std::vector<unsigned char> data(ParseHex(request.params[0].get_str())); - ImportScript(CScript(data.begin(), data.end()), strLabel, fP2SH); + ImportScript(pwallet, CScript(data.begin(), data.end()), strLabel, fP2SH); } else { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address or script"); } if (fRescan) { - pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true); - pwalletMain->ReacceptWalletTransactions(); + pwallet->ScanForWalletTransactions(chainActive.Genesis(), true); + pwallet->ReacceptWalletTransactions(); } return NullUniValue; @@ -255,11 +288,13 @@ UniValue importaddress(const JSONRPCRequest& request) UniValue importprunedfunds(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 2) - throw runtime_error( + throw std::runtime_error( "importprunedfunds\n" "\nImports funds without rescan. Corresponding address or script must previously be included in wallet. Aimed towards pruned wallets. The end-user is responsible to import additional transactions that subsequently spend the imported outputs or rescan after the point in the blockchain the transaction is included.\n" "\nArguments:\n" @@ -271,15 +306,15 @@ UniValue importprunedfunds(const JSONRPCRequest& request) if (!DecodeHexTx(tx, request.params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); uint256 hashTx = tx.GetHash(); - CWalletTx wtx(pwalletMain, MakeTransactionRef(std::move(tx))); + CWalletTx wtx(pwallet, MakeTransactionRef(std::move(tx))); CDataStream ssMB(ParseHexV(request.params[1], "proof"), SER_NETWORK, PROTOCOL_VERSION); CMerkleBlock merkleBlock; ssMB >> merkleBlock; //Search partial merkle tree in proof for our transaction and index in valid block - vector<uint256> vMatch; - vector<unsigned int> vIndex; + std::vector<uint256> vMatch; + std::vector<unsigned int> vIndex; unsigned int txnIndex = 0; if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) == merkleBlock.header.hashMerkleRoot) { @@ -288,7 +323,7 @@ UniValue importprunedfunds(const JSONRPCRequest& request) if (!mapBlockIndex.count(merkleBlock.header.GetHash()) || !chainActive.Contains(mapBlockIndex[merkleBlock.header.GetHash()])) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); - vector<uint256>::const_iterator it; + std::vector<uint256>::const_iterator it; if ((it = std::find(vMatch.begin(), vMatch.end(), hashTx))==vMatch.end()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction given doesn't exist in proof"); } @@ -302,10 +337,10 @@ UniValue importprunedfunds(const JSONRPCRequest& request) wtx.nIndex = txnIndex; wtx.hashBlock = merkleBlock.header.GetHash(); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - if (pwalletMain->IsMine(wtx)) { - pwalletMain->AddToWallet(wtx, false); + if (pwallet->IsMine(wtx)) { + pwallet->AddToWallet(wtx, false); return NullUniValue; } @@ -314,13 +349,15 @@ UniValue importprunedfunds(const JSONRPCRequest& request) UniValue removeprunedfunds(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "removeprunedfunds \"txid\"\n" - "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will effect wallet balances.\n" + "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will affect wallet balances.\n" "\nArguments:\n" "1. \"txid\" (string, required) The hex-encoded id of the transaction you are deleting\n" "\nExamples:\n" @@ -329,20 +366,20 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) + HelpExampleRpc("removprunedfunds", "\"a8d0c0184dde994a09ec054286f1ce581bebf46446a512166eae7628734ea0a5\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; hash.SetHex(request.params[0].get_str()); - vector<uint256> vHash; + std::vector<uint256> vHash; vHash.push_back(hash); - vector<uint256> vHashOut; + std::vector<uint256> vHashOut; - if(pwalletMain->ZapSelectTx(vHash, vHashOut) != DB_LOAD_OK) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Could not properly delete the transaction."); + if (pwallet->ZapSelectTx(vHash, vHashOut) != DB_LOAD_OK) { + throw JSONRPCError(RPC_WALLET_ERROR, "Could not properly delete the transaction."); } if(vHashOut.empty()) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction does not exist in wallet."); + throw JSONRPCError(RPC_INVALID_PARAMETER, "Transaction does not exist in wallet."); } return NullUniValue; @@ -350,11 +387,13 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) UniValue importpubkey(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) - throw runtime_error( + throw std::runtime_error( "importpubkey \"pubkey\" ( \"label\" rescan )\n" "\nAdds a public key (in hex) that can be watched as if it were in your wallet but cannot be used to spend.\n" "\nArguments:\n" @@ -372,7 +411,7 @@ UniValue importpubkey(const JSONRPCRequest& request) ); - string strLabel = ""; + std::string strLabel = ""; if (request.params.size() > 1) strLabel = request.params[1].get_str(); @@ -391,15 +430,15 @@ UniValue importpubkey(const JSONRPCRequest& request) if (!pubKey.IsFullyValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey is not a valid public key"); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - ImportAddress(CBitcoinAddress(pubKey.GetID()), strLabel); - ImportScript(GetScriptForRawPubKey(pubKey), strLabel, false); + ImportAddress(pwallet, CBitcoinAddress(pubKey.GetID()), strLabel); + ImportScript(pwallet, GetScriptForRawPubKey(pubKey), strLabel, false); if (fRescan) { - pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true); - pwalletMain->ReacceptWalletTransactions(); + pwallet->ScanForWalletTransactions(chainActive.Genesis(), true); + pwallet->ReacceptWalletTransactions(); } return NullUniValue; @@ -408,11 +447,13 @@ UniValue importpubkey(const JSONRPCRequest& request) UniValue importwallet(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; - + } + if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "importwallet \"filename\"\n" "\nImports keys from a wallet dump file (see dumpwallet).\n" "\nArguments:\n" @@ -429,11 +470,11 @@ UniValue importwallet(const JSONRPCRequest& request) if (fPruneMode) throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled in pruned mode"); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - EnsureWalletIsUnlocked(); + EnsureWalletIsUnlocked(pwallet); - ifstream file; + std::ifstream file; file.open(request.params[0].get_str().c_str(), std::ios::in | std::ios::ate); if (!file.is_open()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); @@ -445,9 +486,9 @@ UniValue importwallet(const JSONRPCRequest& request) int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); file.seekg(0, file.beg); - pwalletMain->ShowProgress(_("Importing..."), 0); // show progress dialog in GUI + pwallet->ShowProgress(_("Importing..."), 0); // show progress dialog in GUI while (file.good()) { - pwalletMain->ShowProgress("", std::max(1, std::min(99, (int)(((double)file.tellg() / (double)nFilesize) * 100)))); + pwallet->ShowProgress("", std::max(1, std::min(99, (int)(((double)file.tellg() / (double)nFilesize) * 100)))); std::string line; std::getline(file, line); if (line.empty() || line[0] == '#') @@ -464,7 +505,7 @@ UniValue importwallet(const JSONRPCRequest& request) CPubKey pubkey = key.GetPubKey(); assert(key.VerifyPubKey(pubkey)); CKeyID keyid = pubkey.GetID(); - if (pwalletMain->HaveKey(keyid)) { + if (pwallet->HaveKey(keyid)) { LogPrintf("Skipping import of %s (key already present)\n", CBitcoinAddress(keyid).ToString()); continue; } @@ -484,27 +525,24 @@ UniValue importwallet(const JSONRPCRequest& request) } } LogPrintf("Importing %s...\n", CBitcoinAddress(keyid).ToString()); - if (!pwalletMain->AddKeyPubKey(key, pubkey)) { + if (!pwallet->AddKeyPubKey(key, pubkey)) { fGood = false; continue; } - pwalletMain->mapKeyMetadata[keyid].nCreateTime = nTime; + pwallet->mapKeyMetadata[keyid].nCreateTime = nTime; if (fLabel) - pwalletMain->SetAddressBook(keyid, strLabel, "receive"); + pwallet->SetAddressBook(keyid, strLabel, "receive"); nTimeBegin = std::min(nTimeBegin, nTime); } file.close(); - pwalletMain->ShowProgress("", 100); // hide progress dialog in GUI + pwallet->ShowProgress("", 100); // hide progress dialog in GUI + pwallet->UpdateTimeFirstKey(nTimeBegin); - CBlockIndex *pindex = chainActive.Tip(); - while (pindex && pindex->pprev && pindex->GetBlockTime() > nTimeBegin - 7200) - pindex = pindex->pprev; + CBlockIndex *pindex = chainActive.FindEarliestAtLeast(nTimeBegin - TIMESTAMP_WINDOW); - pwalletMain->UpdateTimeFirstKey(nTimeBegin); - - LogPrintf("Rescanning last %i blocks\n", chainActive.Height() - pindex->nHeight + 1); - pwalletMain->ScanForWalletTransactions(pindex); - pwalletMain->MarkDirty(); + LogPrintf("Rescanning last %i blocks\n", pindex ? chainActive.Height() - pindex->nHeight + 1 : 0); + pwallet->ScanForWalletTransactions(pindex); + pwallet->MarkDirty(); if (!fGood) throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys to wallet"); @@ -514,11 +552,13 @@ UniValue importwallet(const JSONRPCRequest& request) UniValue dumpprivkey(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; - + } + if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "dumpprivkey \"address\"\n" "\nReveals the private key corresponding to 'address'.\n" "Then the importprivkey can be used with this output\n" @@ -532,11 +572,11 @@ UniValue dumpprivkey(const JSONRPCRequest& request) + HelpExampleRpc("dumpprivkey", "\"myaddress\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - EnsureWalletIsUnlocked(); + EnsureWalletIsUnlocked(pwallet); - string strAddress = request.params[0].get_str(); + std::string strAddress = request.params[0].get_str(); CBitcoinAddress address; if (!address.SetString(strAddress)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); @@ -544,41 +584,50 @@ UniValue dumpprivkey(const JSONRPCRequest& request) if (!address.GetKeyID(keyID)) throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to a key"); CKey vchSecret; - if (!pwalletMain->GetKey(keyID, vchSecret)) + if (!pwallet->GetKey(keyID, vchSecret)) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key for address " + strAddress + " is not known"); + } return CBitcoinSecret(vchSecret).ToString(); } UniValue dumpwallet(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; - + } + if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "dumpwallet \"filename\"\n" "\nDumps all wallet keys in a human-readable format.\n" "\nArguments:\n" - "1. \"filename\" (string, required) The filename\n" + "1. \"filename\" (string, required) The filename with path (either absolute or relative to bitcoind)\n" + "\nResult:\n" + "{ (json object)\n" + " \"filename\" : { (string) The filename with full absolute path\n" + "}\n" "\nExamples:\n" + HelpExampleCli("dumpwallet", "\"test\"") + HelpExampleRpc("dumpwallet", "\"test\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - EnsureWalletIsUnlocked(); + EnsureWalletIsUnlocked(pwallet); - ofstream file; - file.open(request.params[0].get_str().c_str()); + std::ofstream file; + boost::filesystem::path filepath = request.params[0].get_str(); + filepath = boost::filesystem::absolute(filepath); + file.open(filepath.string().c_str()); if (!file.is_open()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); std::map<CTxDestination, int64_t> mapKeyBirth; std::set<CKeyID> setKeyPool; - pwalletMain->GetKeyBirthTimes(mapKeyBirth); - pwalletMain->GetAllReserveKeys(setKeyPool); + pwallet->GetKeyBirthTimes(mapKeyBirth); + pwallet->GetAllReserveKeys(setKeyPool); // sort time/key pairs std::vector<std::pair<int64_t, CKeyID> > vKeyBirth; @@ -598,12 +647,11 @@ UniValue dumpwallet(const JSONRPCRequest& request) file << "\n"; // add the base58check encoded extended master if the wallet uses HD - CKeyID masterKeyID = pwalletMain->GetHDChain().masterKeyID; + CKeyID masterKeyID = pwallet->GetHDChain().masterKeyID; if (!masterKeyID.IsNull()) { CKey key; - if (pwalletMain->GetKey(masterKeyID, key)) - { + if (pwallet->GetKey(masterKeyID, key)) { CExtKey masterKey; masterKey.SetMaster(key.begin(), key.size()); @@ -618,30 +666,34 @@ UniValue dumpwallet(const JSONRPCRequest& request) std::string strTime = EncodeDumpTime(it->first); std::string strAddr = CBitcoinAddress(keyid).ToString(); CKey key; - if (pwalletMain->GetKey(keyid, key)) { + if (pwallet->GetKey(keyid, key)) { file << strprintf("%s %s ", CBitcoinSecret(key).ToString(), strTime); - if (pwalletMain->mapAddressBook.count(keyid)) { - file << strprintf("label=%s", EncodeDumpString(pwalletMain->mapAddressBook[keyid].name)); + if (pwallet->mapAddressBook.count(keyid)) { + file << strprintf("label=%s", EncodeDumpString(pwallet->mapAddressBook[keyid].name)); } else if (keyid == masterKeyID) { file << "hdmaster=1"; } else if (setKeyPool.count(keyid)) { file << "reserve=1"; - } else if (pwalletMain->mapKeyMetadata[keyid].hdKeypath == "m") { + } else if (pwallet->mapKeyMetadata[keyid].hdKeypath == "m") { file << "inactivehdmaster=1"; } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (pwalletMain->mapKeyMetadata[keyid].hdKeypath.size() > 0 ? " hdkeypath="+pwalletMain->mapKeyMetadata[keyid].hdKeypath : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].hdKeypath.size() > 0 ? " hdkeypath="+pwallet->mapKeyMetadata[keyid].hdKeypath : "")); } } file << "\n"; file << "# End of dump\n"; file.close(); - return NullUniValue; + + UniValue reply(UniValue::VOBJ); + reply.push_back(Pair("filename", filepath.string())); + + return reply; } -UniValue ProcessImport(const UniValue& data, const int64_t timestamp) +UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) { try { bool success = false; @@ -655,16 +707,16 @@ UniValue ProcessImport(const UniValue& data, const int64_t timestamp) } // Optional fields. - const string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; + const std::string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; const UniValue& pubKeys = data.exists("pubkeys") ? data["pubkeys"].get_array() : UniValue(); const UniValue& keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); const bool& internal = data.exists("internal") ? data["internal"].get_bool() : false; const bool& watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; - const string& label = data.exists("label") && !internal ? data["label"].get_str() : ""; + const std::string& label = data.exists("label") && !internal ? data["label"].get_str() : ""; bool isScript = scriptPubKey.getType() == UniValue::VSTR; bool isP2SH = strRedeemScript.length() > 0; - const string& output = isScript ? scriptPubKey.get_str() : scriptPubKey["address"].get_str(); + const std::string& output = isScript ? scriptPubKey.get_str() : scriptPubKey["address"].get_str(); // Parse the output. CScript script; @@ -723,38 +775,38 @@ UniValue ProcessImport(const UniValue& data, const int64_t timestamp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid P2SH address / script"); } - pwalletMain->MarkDirty(); + pwallet->MarkDirty(); - if (!pwalletMain->HaveWatchOnly(redeemScript) && !pwalletMain->AddWatchOnly(redeemScript, timestamp)) { + if (!pwallet->AddWatchOnly(redeemScript, timestamp)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } - if (!pwalletMain->HaveCScript(redeemScript) && !pwalletMain->AddCScript(redeemScript)) { + if (!pwallet->HaveCScript(redeemScript) && !pwallet->AddCScript(redeemScript)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding p2sh redeemScript to wallet"); } CBitcoinAddress redeemAddress = CBitcoinAddress(CScriptID(redeemScript)); CScript redeemDestination = GetScriptForDestination(redeemAddress.Get()); - if (::IsMine(*pwalletMain, redeemDestination) == ISMINE_SPENDABLE) { + if (::IsMine(*pwallet, redeemDestination) == ISMINE_SPENDABLE) { throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); } - pwalletMain->MarkDirty(); + pwallet->MarkDirty(); - if (!pwalletMain->HaveWatchOnly(redeemDestination) && !pwalletMain->AddWatchOnly(redeemDestination, timestamp)) { + if (!pwallet->AddWatchOnly(redeemDestination, timestamp)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } // add to address book or update label if (address.IsValid()) { - pwalletMain->SetAddressBook(address.Get(), label, "receive"); + pwallet->SetAddressBook(address.Get(), label, "receive"); } // Import private keys. if (keys.size()) { for (size_t i = 0; i < keys.size(); i++) { - const string& privkey = keys[i].get_str(); + const std::string& privkey = keys[i].get_str(); CBitcoinSecret vchSecret; bool fGood = vchSecret.SetString(privkey); @@ -773,20 +825,20 @@ UniValue ProcessImport(const UniValue& data, const int64_t timestamp) assert(key.VerifyPubKey(pubkey)); CKeyID vchAddress = pubkey.GetID(); - pwalletMain->MarkDirty(); - pwalletMain->SetAddressBook(vchAddress, label, "receive"); + pwallet->MarkDirty(); + pwallet->SetAddressBook(vchAddress, label, "receive"); - if (pwalletMain->HaveKey(vchAddress)) { + if (pwallet->HaveKey(vchAddress)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Already have this key"); } - pwalletMain->mapKeyMetadata[vchAddress].nCreateTime = timestamp; + pwallet->mapKeyMetadata[vchAddress].nCreateTime = timestamp; - if (!pwalletMain->AddKeyPubKey(key, pubkey)) { + if (!pwallet->AddKeyPubKey(key, pubkey)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); } - pwalletMain->UpdateTimeFirstKey(timestamp); + pwallet->UpdateTimeFirstKey(timestamp); } } @@ -794,7 +846,7 @@ UniValue ProcessImport(const UniValue& data, const int64_t timestamp) } else { // Import public keys. if (pubKeys.size() && keys.size() == 0) { - const string& strPubKey = pubKeys[0].get_str(); + const std::string& strPubKey = pubKeys[0].get_str(); if (!IsHex(strPubKey)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey must be a hex string"); @@ -829,31 +881,31 @@ UniValue ProcessImport(const UniValue& data, const int64_t timestamp) CScript pubKeyScript = GetScriptForDestination(pubKeyAddress.Get()); - if (::IsMine(*pwalletMain, pubKeyScript) == ISMINE_SPENDABLE) { + if (::IsMine(*pwallet, pubKeyScript) == ISMINE_SPENDABLE) { throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); } - pwalletMain->MarkDirty(); + pwallet->MarkDirty(); - if (!pwalletMain->HaveWatchOnly(pubKeyScript) && !pwalletMain->AddWatchOnly(pubKeyScript, timestamp)) { + if (!pwallet->AddWatchOnly(pubKeyScript, timestamp)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } // add to address book or update label if (pubKeyAddress.IsValid()) { - pwalletMain->SetAddressBook(pubKeyAddress.Get(), label, "receive"); + pwallet->SetAddressBook(pubKeyAddress.Get(), label, "receive"); } // TODO Is this necessary? CScript scriptRawPubKey = GetScriptForRawPubKey(pubKey); - if (::IsMine(*pwalletMain, scriptRawPubKey) == ISMINE_SPENDABLE) { + if (::IsMine(*pwallet, scriptRawPubKey) == ISMINE_SPENDABLE) { throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); } - pwalletMain->MarkDirty(); + pwallet->MarkDirty(); - if (!pwalletMain->HaveWatchOnly(scriptRawPubKey) && !pwalletMain->AddWatchOnly(scriptRawPubKey, timestamp)) { + if (!pwallet->AddWatchOnly(scriptRawPubKey, timestamp)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } @@ -862,7 +914,7 @@ UniValue ProcessImport(const UniValue& data, const int64_t timestamp) // Import private keys. if (keys.size()) { - const string& strPrivkey = keys[0].get_str(); + const std::string& strPrivkey = keys[0].get_str(); // Checks. CBitcoinSecret vchSecret; @@ -901,40 +953,40 @@ UniValue ProcessImport(const UniValue& data, const int64_t timestamp) } CKeyID vchAddress = pubKey.GetID(); - pwalletMain->MarkDirty(); - pwalletMain->SetAddressBook(vchAddress, label, "receive"); + pwallet->MarkDirty(); + pwallet->SetAddressBook(vchAddress, label, "receive"); - if (pwalletMain->HaveKey(vchAddress)) { + if (pwallet->HaveKey(vchAddress)) { return false; } - pwalletMain->mapKeyMetadata[vchAddress].nCreateTime = timestamp; + pwallet->mapKeyMetadata[vchAddress].nCreateTime = timestamp; - if (!pwalletMain->AddKeyPubKey(key, pubKey)) { + if (!pwallet->AddKeyPubKey(key, pubKey)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); } - pwalletMain->UpdateTimeFirstKey(timestamp); + pwallet->UpdateTimeFirstKey(timestamp); success = true; } // Import scriptPubKey only. if (pubKeys.size() == 0 && keys.size() == 0) { - if (::IsMine(*pwalletMain, script) == ISMINE_SPENDABLE) { + if (::IsMine(*pwallet, script) == ISMINE_SPENDABLE) { throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); } - pwalletMain->MarkDirty(); + pwallet->MarkDirty(); - if (!pwalletMain->HaveWatchOnly(script) && !pwalletMain->AddWatchOnly(script, timestamp)) { + if (!pwallet->AddWatchOnly(script, timestamp)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } if (scriptPubKey.getType() == UniValue::VOBJ) { // add to address book or update label if (address.IsValid()) { - pwalletMain->SetAddressBook(address.Get(), label, "receive"); + pwallet->SetAddressBook(address.Get(), label, "receive"); } } @@ -974,9 +1026,14 @@ int64_t GetImportTimestamp(const UniValue& data, int64_t now) UniValue importmulti(const JSONRPCRequest& mainRequest) { + CWallet * const pwallet = GetWalletForJSONRPCRequest(mainRequest); + if (!EnsureWalletIsAvailable(pwallet, mainRequest.fHelp)) { + return NullUniValue; + } + // clang-format off if (mainRequest.fHelp || mainRequest.params.size() < 1 || mainRequest.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "importmulti \"requests\" \"options\"\n\n" "Import addresses/scripts (with private or public keys, redeem script (P2SH)), rescanning all addresses in one-shot-only (rescan can be disabled via options).\n\n" "Arguments:\n" @@ -1012,9 +1069,6 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) " [{ \"success\": true } , { \"success\": false, \"error\": { \"code\": -1, \"message\": \"Internal Server Error\"} }, ... ]\n"); // clang-format on - if (!EnsureWalletIsAvailable(mainRequest.fHelp)) { - return NullUniValue; - } RPCTypeCheck(mainRequest.params, boost::assign::list_of(UniValue::VARR)(UniValue::VOBJ)); @@ -1031,8 +1085,8 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } } - LOCK2(cs_main, pwalletMain->cs_wallet); - EnsureWalletIsUnlocked(); + LOCK2(cs_main, pwallet->cs_wallet); + EnsureWalletIsUnlocked(pwallet); // Verify all timestamps are present before importing any keys. const int64_t now = chainActive.Tip() ? chainActive.Tip()->GetMedianTimePast() : 0; @@ -1054,7 +1108,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) BOOST_FOREACH (const UniValue& data, requests.getValues()) { const int64_t timestamp = std::max(GetImportTimestamp(data, now), minimumTimestamp); - const UniValue result = ProcessImport(data, timestamp); + const UniValue result = ProcessImport(pwallet, data, timestamp); response.push_back(result); if (!fRescan) { @@ -1073,14 +1127,14 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } if (fRescan && fRunScan && requests.size()) { - CBlockIndex* pindex = nLowestTimestamp > minimumTimestamp ? chainActive.FindEarliestAtLeast(std::max<int64_t>(nLowestTimestamp - 7200, 0)) : chainActive.Genesis(); - CBlockIndex* scannedRange = nullptr; + CBlockIndex* pindex = nLowestTimestamp > minimumTimestamp ? chainActive.FindEarliestAtLeast(std::max<int64_t>(nLowestTimestamp - TIMESTAMP_WINDOW, 0)) : chainActive.Genesis(); + CBlockIndex* scanFailed = nullptr; if (pindex) { - scannedRange = pwalletMain->ScanForWalletTransactions(pindex, true); - pwalletMain->ReacceptWalletTransactions(); + scanFailed = pwallet->ScanForWalletTransactions(pindex, true); + pwallet->ReacceptWalletTransactions(); } - if (!scannedRange || scannedRange->nHeight > pindex->nHeight) { + if (scanFailed) { std::vector<UniValue> results = response.getValues(); response.clear(); response.setArray(); @@ -1090,12 +1144,23 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) // range, or if the import result already has an error set, let // the result stand unmodified. Otherwise replace the result // with an error message. - if (GetImportTimestamp(request, now) - 7200 >= scannedRange->GetBlockTimeMax() || results.at(i).exists("error")) { + if (GetImportTimestamp(request, now) - TIMESTAMP_WINDOW > scanFailed->GetBlockTimeMax() || results.at(i).exists("error")) { response.push_back(results.at(i)); } else { UniValue result = UniValue(UniValue::VOBJ); result.pushKV("success", UniValue(false)); - result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, strprintf("Failed to rescan before time %d, transactions may be missing.", scannedRange->GetBlockTimeMax()))); + result.pushKV( + "error", + JSONRPCError( + RPC_MISC_ERROR, + strprintf("Rescan failed for key with creation timestamp %d. There was an error reading a " + "block from time %d, which is after or within %d seconds of key creation, and " + "could contain transactions pertaining to the key. As a result, transactions " + "and coins using this key may not appear in the wallet. This error could be " + "caused by pruning or data corruption (see bitcoind log for details) and could " + "be dealt with by downloading and rescanning the relevant blocks (see -reindex " + "and -rescan options).", + GetImportTimestamp(request, now), scanFailed->GetBlockTimeMax(), TIMESTAMP_WINDOW))); response.push_back(std::move(result)); } ++i; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index d785c95ae0..3b6dfd42ca 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -9,8 +9,11 @@ #include "consensus/validation.h" #include "core_io.h" #include "init.h" +#include "wallet/coincontrol.h" #include "validation.h" #include "net.h" +#include "policy/feerate.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "rpc/server.h" @@ -18,8 +21,9 @@ #include "timedata.h" #include "util.h" #include "utilmoneystr.h" -#include "wallet.h" -#include "walletdb.h" +#include "wallet/feebumper.h" +#include "wallet/wallet.h" +#include "wallet/walletdb.h" #include <stdint.h> @@ -27,22 +31,21 @@ #include <univalue.h> -using namespace std; - -int64_t nWalletUnlockTime; -static CCriticalSection cs_nWalletUnlockTime; +CWallet *GetWalletForJSONRPCRequest(const JSONRPCRequest& request) +{ + return pwalletMain; +} -std::string HelpRequiringPassphrase() +std::string HelpRequiringPassphrase(CWallet * const pwallet) { - return pwalletMain && pwalletMain->IsCrypted() + return pwallet && pwallet->IsCrypted() ? "\nRequires wallet passphrase to be set with walletpassphrase call." : ""; } -bool EnsureWalletIsAvailable(bool avoidException) +bool EnsureWalletIsAvailable(CWallet * const pwallet, bool avoidException) { - if (!pwalletMain) - { + if (!pwallet) { if (!avoidException) throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found (disabled)"); else @@ -51,10 +54,11 @@ bool EnsureWalletIsAvailable(bool avoidException) return true; } -void EnsureWalletIsUnlocked() +void EnsureWalletIsUnlocked(CWallet * const pwallet) { - if (pwalletMain->IsLocked()) + if (pwallet->IsLocked()) { throw JSONRPCError(RPC_WALLET_UNLOCK_NEEDED, "Error: Please enter the wallet passphrase with walletpassphrase first."); + } } void WalletTxToJSON(const CWalletTx& wtx, UniValue& entry) @@ -92,13 +96,13 @@ void WalletTxToJSON(const CWalletTx& wtx, UniValue& entry) } entry.push_back(Pair("bip125-replaceable", rbfStatus)); - BOOST_FOREACH(const PAIRTYPE(string,string)& item, wtx.mapValue) + BOOST_FOREACH(const PAIRTYPE(std::string, std::string)& item, wtx.mapValue) entry.push_back(Pair(item.first, item.second)); } -string AccountFromValue(const UniValue& value) +std::string AccountFromValue(const UniValue& value) { - string strAccount = value.get_str(); + std::string strAccount = value.get_str(); if (strAccount == "*") throw JSONRPCError(RPC_WALLET_INVALID_ACCOUNT_NAME, "Invalid account name"); return strAccount; @@ -106,11 +110,13 @@ string AccountFromValue(const UniValue& value) UniValue getnewaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "getnewaddress ( \"account\" )\n" "\nReturns a new Bitcoin address for receiving payments.\n" "If 'account' is specified (DEPRECATED), it is added to the address book \n" @@ -124,32 +130,34 @@ UniValue getnewaddress(const JSONRPCRequest& request) + HelpExampleRpc("getnewaddress", "") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); // Parse the account first so we don't generate a key if there's an error - string strAccount; + std::string strAccount; if (request.params.size() > 0) strAccount = AccountFromValue(request.params[0]); - if (!pwalletMain->IsLocked()) - pwalletMain->TopUpKeyPool(); + if (!pwallet->IsLocked()) { + pwallet->TopUpKeyPool(); + } // Generate a new key that is added to wallet CPubKey newKey; - if (!pwalletMain->GetKeyFromPool(newKey)) + if (!pwallet->GetKeyFromPool(newKey)) { throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); + } CKeyID keyID = newKey.GetID(); - pwalletMain->SetAddressBook(keyID, strAccount, "receive"); + pwallet->SetAddressBook(keyID, strAccount, "receive"); return CBitcoinAddress(keyID).ToString(); } -CBitcoinAddress GetAccountAddress(string strAccount, bool bForceNew=false) +CBitcoinAddress GetAccountAddress(CWallet* const pwallet, std::string strAccount, bool bForceNew=false) { CPubKey pubKey; - if (!pwalletMain->GetAccountPubkey(pubKey, strAccount, bForceNew)) { + if (!pwallet->GetAccountPubkey(pubKey, strAccount, bForceNew)) { throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); } @@ -158,11 +166,13 @@ CBitcoinAddress GetAccountAddress(string strAccount, bool bForceNew=false) UniValue getaccountaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "getaccountaddress \"account\"\n" "\nDEPRECATED. Returns the current Bitcoin address for receiving payments to this account.\n" "\nArguments:\n" @@ -176,25 +186,27 @@ UniValue getaccountaddress(const JSONRPCRequest& request) + HelpExampleRpc("getaccountaddress", "\"myaccount\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); // Parse the account first so we don't generate a key if there's an error - string strAccount = AccountFromValue(request.params[0]); + std::string strAccount = AccountFromValue(request.params[0]); UniValue ret(UniValue::VSTR); - ret = GetAccountAddress(strAccount).ToString(); + ret = GetAccountAddress(pwallet, strAccount).ToString(); return ret; } UniValue getrawchangeaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "getrawchangeaddress\n" "\nReturns a new Bitcoin address, for receiving change.\n" "This is for use with raw transactions, NOT normal use.\n" @@ -205,14 +217,15 @@ UniValue getrawchangeaddress(const JSONRPCRequest& request) + HelpExampleRpc("getrawchangeaddress", "") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - if (!pwalletMain->IsLocked()) - pwalletMain->TopUpKeyPool(); + if (!pwallet->IsLocked()) { + pwallet->TopUpKeyPool(); + } - CReserveKey reservekey(pwalletMain); + CReserveKey reservekey(pwallet); CPubKey vchPubKey; - if (!reservekey.GetReservedKey(vchPubKey)) + if (!reservekey.GetReservedKey(vchPubKey, true)) throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); reservekey.KeepKey(); @@ -225,11 +238,13 @@ UniValue getrawchangeaddress(const JSONRPCRequest& request) UniValue setaccount(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "setaccount \"address\" \"account\"\n" "\nDEPRECATED. Sets the account associated with the given address.\n" "\nArguments:\n" @@ -240,27 +255,26 @@ UniValue setaccount(const JSONRPCRequest& request) + HelpExampleRpc("setaccount", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\", \"tabby\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); CBitcoinAddress address(request.params[0].get_str()); if (!address.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); - string strAccount; + std::string strAccount; if (request.params.size() > 1) strAccount = AccountFromValue(request.params[1]); // Only add the account if the address is yours. - if (IsMine(*pwalletMain, address.Get())) - { + if (IsMine(*pwallet, address.Get())) { // Detect when changing the account of an address that is the 'unused current key' of another account: - if (pwalletMain->mapAddressBook.count(address.Get())) - { - string strOldAccount = pwalletMain->mapAddressBook[address.Get()].name; - if (address == GetAccountAddress(strOldAccount)) - GetAccountAddress(strOldAccount, true); + if (pwallet->mapAddressBook.count(address.Get())) { + std::string strOldAccount = pwallet->mapAddressBook[address.Get()].name; + if (address == GetAccountAddress(pwallet, strOldAccount)) { + GetAccountAddress(pwallet, strOldAccount, true); + } } - pwalletMain->SetAddressBook(address.Get(), strAccount, "receive"); + pwallet->SetAddressBook(address.Get(), strAccount, "receive"); } else throw JSONRPCError(RPC_MISC_ERROR, "setaccount can only be used with own address"); @@ -271,11 +285,13 @@ UniValue setaccount(const JSONRPCRequest& request) UniValue getaccount(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "getaccount \"address\"\n" "\nDEPRECATED. Returns the account associated with the given address.\n" "\nArguments:\n" @@ -287,27 +303,30 @@ UniValue getaccount(const JSONRPCRequest& request) + HelpExampleRpc("getaccount", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); CBitcoinAddress address(request.params[0].get_str()); if (!address.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); - string strAccount; - map<CTxDestination, CAddressBookData>::iterator mi = pwalletMain->mapAddressBook.find(address.Get()); - if (mi != pwalletMain->mapAddressBook.end() && !(*mi).second.name.empty()) + std::string strAccount; + std::map<CTxDestination, CAddressBookData>::iterator mi = pwallet->mapAddressBook.find(address.Get()); + if (mi != pwallet->mapAddressBook.end() && !(*mi).second.name.empty()) { strAccount = (*mi).second.name; + } return strAccount; } UniValue getaddressesbyaccount(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "getaddressesbyaccount \"account\"\n" "\nDEPRECATED. Returns the list of addresses for the given account.\n" "\nArguments:\n" @@ -322,25 +341,24 @@ UniValue getaddressesbyaccount(const JSONRPCRequest& request) + HelpExampleRpc("getaddressesbyaccount", "\"tabby\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - string strAccount = AccountFromValue(request.params[0]); + std::string strAccount = AccountFromValue(request.params[0]); // Find all addresses that have the given account UniValue ret(UniValue::VARR); - BOOST_FOREACH(const PAIRTYPE(CBitcoinAddress, CAddressBookData)& item, pwalletMain->mapAddressBook) - { + for (const std::pair<CBitcoinAddress, CAddressBookData>& item : pwallet->mapAddressBook) { const CBitcoinAddress& address = item.first; - const string& strName = item.second.name; + const std::string& strName = item.second.name; if (strName == strAccount) ret.push_back(address.ToString()); } return ret; } -static void SendMoney(const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, CWalletTx& wtxNew) +static void SendMoney(CWallet * const pwallet, const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, CWalletTx& wtxNew) { - CAmount curBalance = pwalletMain->GetBalance(); + CAmount curBalance = pwallet->GetBalance(); // Check amount if (nValue <= 0) @@ -349,27 +367,28 @@ static void SendMoney(const CTxDestination &address, CAmount nValue, bool fSubtr if (nValue > curBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds"); - if (pwalletMain->GetBroadcastTransactions() && !g_connman) + if (pwallet->GetBroadcastTransactions() && !g_connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); + } // Parse Bitcoin address CScript scriptPubKey = GetScriptForDestination(address); // Create and send the transaction - CReserveKey reservekey(pwalletMain); + CReserveKey reservekey(pwallet); CAmount nFeeRequired; std::string strError; - vector<CRecipient> vecSend; + std::vector<CRecipient> vecSend; int nChangePosRet = -1; CRecipient recipient = {scriptPubKey, nValue, fSubtractFeeFromAmount}; vecSend.push_back(recipient); - if (!pwalletMain->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError)) { + if (!pwallet->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError)) { if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); throw JSONRPCError(RPC_WALLET_ERROR, strError); } CValidationState state; - if (!pwalletMain->CommitTransaction(wtxNew, reservekey, g_connman.get(), state)) { + if (!pwallet->CommitTransaction(wtxNew, reservekey, g_connman.get(), state)) { strError = strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason()); throw JSONRPCError(RPC_WALLET_ERROR, strError); } @@ -377,14 +396,16 @@ static void SendMoney(const CTxDestination &address, CAmount nValue, bool fSubtr UniValue sendtoaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 2 || request.params.size() > 5) - throw runtime_error( + throw std::runtime_error( "sendtoaddress \"address\" amount ( \"comment\" \"comment_to\" subtractfeefromamount )\n" "\nSend an amount to a given address.\n" - + HelpRequiringPassphrase() + + + HelpRequiringPassphrase(pwallet) + "\nArguments:\n" "1. \"address\" (string, required) The bitcoin address to send to.\n" "2. \"amount\" (numeric or string, required) The amount in " + CURRENCY_UNIT + " to send. eg 0.1\n" @@ -404,7 +425,7 @@ UniValue sendtoaddress(const JSONRPCRequest& request) + HelpExampleRpc("sendtoaddress", "\"1M72Sfpbz1BPpXFHz9m3CdqATR44Jvaydd\", 0.1, \"donation\", \"seans outpost\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); CBitcoinAddress address(request.params[0].get_str()); if (!address.IsValid()) @@ -426,20 +447,22 @@ UniValue sendtoaddress(const JSONRPCRequest& request) if (request.params.size() > 4) fSubtractFeeFromAmount = request.params[4].get_bool(); - EnsureWalletIsUnlocked(); + EnsureWalletIsUnlocked(pwallet); - SendMoney(address.Get(), nAmount, fSubtractFeeFromAmount, wtx); + SendMoney(pwallet, address.Get(), nAmount, fSubtractFeeFromAmount, wtx); return wtx.GetHash().GetHex(); } UniValue listaddressgroupings(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp) - throw runtime_error( + throw std::runtime_error( "listaddressgroupings\n" "\nLists groups of addresses which have had their common ownership\n" "made public by common use as inputs or as the resulting change\n" @@ -461,12 +484,11 @@ UniValue listaddressgroupings(const JSONRPCRequest& request) + HelpExampleRpc("listaddressgroupings", "") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); UniValue jsonGroupings(UniValue::VARR); - map<CTxDestination, CAmount> balances = pwalletMain->GetAddressBalances(); - BOOST_FOREACH(set<CTxDestination> grouping, pwalletMain->GetAddressGroupings()) - { + std::map<CTxDestination, CAmount> balances = pwallet->GetAddressBalances(); + for (std::set<CTxDestination> grouping : pwallet->GetAddressGroupings()) { UniValue jsonGrouping(UniValue::VARR); BOOST_FOREACH(CTxDestination address, grouping) { @@ -474,8 +496,9 @@ UniValue listaddressgroupings(const JSONRPCRequest& request) addressInfo.push_back(CBitcoinAddress(address).ToString()); addressInfo.push_back(ValueFromAmount(balances[address])); { - if (pwalletMain->mapAddressBook.find(CBitcoinAddress(address).Get()) != pwalletMain->mapAddressBook.end()) - addressInfo.push_back(pwalletMain->mapAddressBook.find(CBitcoinAddress(address).Get())->second.name); + if (pwallet->mapAddressBook.find(CBitcoinAddress(address).Get()) != pwallet->mapAddressBook.end()) { + addressInfo.push_back(pwallet->mapAddressBook.find(CBitcoinAddress(address).Get())->second.name); + } } jsonGrouping.push_back(addressInfo); } @@ -486,14 +509,16 @@ UniValue listaddressgroupings(const JSONRPCRequest& request) UniValue signmessage(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 2) - throw runtime_error( + throw std::runtime_error( "signmessage \"address\" \"message\"\n" "\nSign a message with the private key of an address" - + HelpRequiringPassphrase() + "\n" + + HelpRequiringPassphrase(pwallet) + "\n" "\nArguments:\n" "1. \"address\" (string, required) The bitcoin address to use for the private key.\n" "2. \"message\" (string, required) The message to create a signature of.\n" @@ -510,12 +535,12 @@ UniValue signmessage(const JSONRPCRequest& request) + HelpExampleRpc("signmessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\", \"my message\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - EnsureWalletIsUnlocked(); + EnsureWalletIsUnlocked(pwallet); - string strAddress = request.params[0].get_str(); - string strMessage = request.params[1].get_str(); + std::string strAddress = request.params[0].get_str(); + std::string strMessage = request.params[1].get_str(); CBitcoinAddress addr(strAddress); if (!addr.IsValid()) @@ -526,14 +551,15 @@ UniValue signmessage(const JSONRPCRequest& request) throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); CKey key; - if (!pwalletMain->GetKey(keyID, key)) + if (!pwallet->GetKey(keyID, key)) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key not available"); + } CHashWriter ss(SER_GETHASH, 0); ss << strMessageMagic; ss << strMessage; - vector<unsigned char> vchSig; + std::vector<unsigned char> vchSig; if (!key.SignCompact(ss.GetHash(), vchSig)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Sign failed"); @@ -542,11 +568,13 @@ UniValue signmessage(const JSONRPCRequest& request) UniValue getreceivedbyaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "getreceivedbyaddress \"address\" ( minconf )\n" "\nReturns the total amount received by the given address in transactions with at least minconf confirmations.\n" "\nArguments:\n" @@ -565,15 +593,16 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) + HelpExampleRpc("getreceivedbyaddress", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\", 6") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); // Bitcoin address CBitcoinAddress address = CBitcoinAddress(request.params[0].get_str()); if (!address.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); CScript scriptPubKey = GetScriptForDestination(address.Get()); - if (!IsMine(*pwalletMain, scriptPubKey)) + if (!IsMine(*pwallet, scriptPubKey)) { return ValueFromAmount(0); + } // Minimum confirmations int nMinDepth = 1; @@ -582,9 +611,8 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) // Tally CAmount nAmount = 0; - for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; + for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; @@ -600,11 +628,13 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) UniValue getreceivedbyaccount(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "getreceivedbyaccount \"account\" ( minconf )\n" "\nDEPRECATED. Returns the total amount received by addresses with <account> in transactions with at least [minconf] confirmations.\n" "\nArguments:\n" @@ -623,7 +653,7 @@ UniValue getreceivedbyaccount(const JSONRPCRequest& request) + HelpExampleRpc("getreceivedbyaccount", "\"tabby\", 6") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); // Minimum confirmations int nMinDepth = 1; @@ -631,23 +661,23 @@ UniValue getreceivedbyaccount(const JSONRPCRequest& request) nMinDepth = request.params[1].get_int(); // Get the set of pub keys assigned to account - string strAccount = AccountFromValue(request.params[0]); - set<CTxDestination> setAddress = pwalletMain->GetAccountAddresses(strAccount); + std::string strAccount = AccountFromValue(request.params[0]); + std::set<CTxDestination> setAddress = pwallet->GetAccountAddresses(strAccount); // Tally CAmount nAmount = 0; - for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; + for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { CTxDestination address; - if (ExtractDestination(txout.scriptPubKey, address) && IsMine(*pwalletMain, address) && setAddress.count(address)) + if (ExtractDestination(txout.scriptPubKey, address) && IsMine(*pwallet, address) && setAddress.count(address)) { if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; + } } } @@ -657,11 +687,13 @@ UniValue getreceivedbyaccount(const JSONRPCRequest& request) UniValue getbalance(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 3) - throw runtime_error( + throw std::runtime_error( "getbalance ( \"account\" minconf include_watchonly )\n" "\nIf account is not specified, returns the server's total available balance.\n" "If account is specified (DEPRECATED), returns the balance in the account.\n" @@ -693,10 +725,13 @@ UniValue getbalance(const JSONRPCRequest& request) + HelpExampleRpc("getbalance", "\"*\", 6") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); if (request.params.size() == 0) - return ValueFromAmount(pwalletMain->GetBalance()); + return ValueFromAmount(pwallet->GetBalance()); + + const std::string& account_param = request.params[0].get_str(); + const std::string* account = account_param != "*" ? &account_param : nullptr; int nMinDepth = 1; if (request.params.size() > 1) @@ -706,67 +741,36 @@ UniValue getbalance(const JSONRPCRequest& request) if(request.params[2].get_bool()) filter = filter | ISMINE_WATCH_ONLY; - if (request.params[0].get_str() == "*") { - // Calculate total balance in a very different way from GetBalance(). - // The biggest difference is that GetBalance() sums up all unspent - // TxOuts paying to the wallet, while this sums up both spent and - // unspent TxOuts paying to the wallet, and then subtracts the values of - // TxIns spending from the wallet. This also has fewer restrictions on - // which unconfirmed transactions are considered trusted. - CAmount nBalance = 0; - for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount allFee; - string strSentAccount; - list<COutputEntry> listReceived; - list<COutputEntry> listSent; - wtx.GetAmounts(listReceived, listSent, allFee, strSentAccount, filter); - if (wtx.GetDepthInMainChain() >= nMinDepth) - { - BOOST_FOREACH(const COutputEntry& r, listReceived) - nBalance += r.amount; - } - BOOST_FOREACH(const COutputEntry& s, listSent) - nBalance -= s.amount; - nBalance -= allFee; - } - return ValueFromAmount(nBalance); - } - - string strAccount = AccountFromValue(request.params[0]); - - CAmount nBalance = pwalletMain->GetAccountBalance(strAccount, nMinDepth, filter); - - return ValueFromAmount(nBalance); + return ValueFromAmount(pwallet->GetLegacyBalance(filter, nMinDepth, account)); } UniValue getunconfirmedbalance(const JSONRPCRequest &request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 0) - throw runtime_error( + throw std::runtime_error( "getunconfirmedbalance\n" "Returns the server's total unconfirmed balance\n"); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - return ValueFromAmount(pwalletMain->GetUnconfirmedBalance()); + return ValueFromAmount(pwallet->GetUnconfirmedBalance()); } UniValue movecmd(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 3 || request.params.size() > 5) - throw runtime_error( + throw std::runtime_error( "move \"fromaccount\" \"toaccount\" amount ( minconf \"comment\" )\n" "\nDEPRECATED. Move a specified amount from one account in your wallet to another.\n" "\nArguments:\n" @@ -786,22 +790,23 @@ UniValue movecmd(const JSONRPCRequest& request) + HelpExampleRpc("move", "\"timotei\", \"akiko\", 0.01, 6, \"happy birthday!\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - string strFrom = AccountFromValue(request.params[0]); - string strTo = AccountFromValue(request.params[1]); + std::string strFrom = AccountFromValue(request.params[0]); + std::string strTo = AccountFromValue(request.params[1]); CAmount nAmount = AmountFromValue(request.params[2]); if (nAmount <= 0) throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount for send"); if (request.params.size() > 3) // unused parameter, used to be nMinDepth, keep type-checking it though (void)request.params[3].get_int(); - string strComment; + std::string strComment; if (request.params.size() > 4) strComment = request.params[4].get_str(); - if (!pwalletMain->AccountMove(strFrom, strTo, nAmount, strComment)) + if (!pwallet->AccountMove(strFrom, strTo, nAmount, strComment)) { throw JSONRPCError(RPC_DATABASE_ERROR, "database error"); + } return true; } @@ -809,14 +814,16 @@ UniValue movecmd(const JSONRPCRequest& request) UniValue sendfrom(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 3 || request.params.size() > 6) - throw runtime_error( + throw std::runtime_error( "sendfrom \"fromaccount\" \"toaddress\" amount ( minconf \"comment\" \"comment_to\" )\n" "\nDEPRECATED (use sendtoaddress). Sent an amount from an account to a bitcoin address." - + HelpRequiringPassphrase() + "\n" + + HelpRequiringPassphrase(pwallet) + "\n" "\nArguments:\n" "1. \"fromaccount\" (string, required) The name of the account to send funds from. May be the default account using \"\".\n" " Specifying an account does not influence coin selection, but it does associate the newly created\n" @@ -841,9 +848,9 @@ UniValue sendfrom(const JSONRPCRequest& request) + HelpExampleRpc("sendfrom", "\"tabby\", \"1M72Sfpbz1BPpXFHz9m3CdqATR44Jvaydd\", 0.01, 6, \"donation\", \"seans outpost\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - string strAccount = AccountFromValue(request.params[0]); + std::string strAccount = AccountFromValue(request.params[0]); CBitcoinAddress address(request.params[1].get_str()); if (!address.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); @@ -861,14 +868,14 @@ UniValue sendfrom(const JSONRPCRequest& request) if (request.params.size() > 5 && !request.params[5].isNull() && !request.params[5].get_str().empty()) wtx.mapValue["to"] = request.params[5].get_str(); - EnsureWalletIsUnlocked(); + EnsureWalletIsUnlocked(pwallet); // Check funds - CAmount nBalance = pwalletMain->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth, &strAccount); if (nAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); - SendMoney(address.Get(), nAmount, false, wtx); + SendMoney(pwallet, address.Get(), nAmount, false, wtx); return wtx.GetHash().GetHex(); } @@ -876,14 +883,16 @@ UniValue sendfrom(const JSONRPCRequest& request) UniValue sendmany(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 2 || request.params.size() > 5) - throw runtime_error( + throw std::runtime_error( "sendmany \"fromaccount\" {\"address\":amount,...} ( minconf \"comment\" [\"address\",...] )\n" "\nSend multiple times. Amounts are double-precision floating point numbers." - + HelpRequiringPassphrase() + "\n" + + HelpRequiringPassphrase(pwallet) + "\n" "\nArguments:\n" "1. \"fromaccount\" (string, required) DEPRECATED. The account to send the funds from. Should be \"\" for the default account\n" "2. \"amounts\" (string, required) A json object with addresses and amounts\n" @@ -915,12 +924,13 @@ UniValue sendmany(const JSONRPCRequest& request) + HelpExampleRpc("sendmany", "\"\", \"{\\\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\\\":0.01,\\\"1353tsE8YMTA4EuV7dgUXGjNFf9KpVvKHz\\\":0.02}\", 6, \"testing\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - if (pwalletMain->GetBroadcastTransactions() && !g_connman) + if (pwallet->GetBroadcastTransactions() && !g_connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); + } - string strAccount = AccountFromValue(request.params[0]); + std::string strAccount = AccountFromValue(request.params[0]); UniValue sendTo = request.params[1].get_obj(); int nMinDepth = 1; if (request.params.size() > 2) @@ -935,19 +945,19 @@ UniValue sendmany(const JSONRPCRequest& request) if (request.params.size() > 4) subtractFeeFromAmount = request.params[4].get_array(); - set<CBitcoinAddress> setAddress; - vector<CRecipient> vecSend; + std::set<CBitcoinAddress> setAddress; + std::vector<CRecipient> vecSend; CAmount totalAmount = 0; - vector<string> keys = sendTo.getKeys(); - BOOST_FOREACH(const string& name_, keys) + std::vector<std::string> keys = sendTo.getKeys(); + BOOST_FOREACH(const std::string& name_, keys) { CBitcoinAddress address(name_); if (!address.IsValid()) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, string("Invalid Bitcoin address: ")+name_); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ")+name_); if (setAddress.count(address)) - throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid parameter, duplicated address: ")+name_); + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ")+name_); setAddress.insert(address); CScript scriptPubKey = GetScriptForDestination(address.Get()); @@ -967,23 +977,23 @@ UniValue sendmany(const JSONRPCRequest& request) vecSend.push_back(recipient); } - EnsureWalletIsUnlocked(); + EnsureWalletIsUnlocked(pwallet); // Check funds - CAmount nBalance = pwalletMain->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth, &strAccount); if (totalAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); // Send - CReserveKey keyChange(pwalletMain); + CReserveKey keyChange(pwallet); CAmount nFeeRequired = 0; int nChangePosRet = -1; - string strFailReason; - bool fCreated = pwalletMain->CreateTransaction(vecSend, wtx, keyChange, nFeeRequired, nChangePosRet, strFailReason); + std::string strFailReason; + bool fCreated = pwallet->CreateTransaction(vecSend, wtx, keyChange, nFeeRequired, nChangePosRet, strFailReason); if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); CValidationState state; - if (!pwalletMain->CommitTransaction(wtx, keyChange, g_connman.get(), state)) { + if (!pwallet->CommitTransaction(wtx, keyChange, g_connman.get(), state)) { strFailReason = strprintf("Transaction commit failed:: %s", state.GetRejectReason()); throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); } @@ -992,16 +1002,18 @@ UniValue sendmany(const JSONRPCRequest& request) } // Defined in rpc/misc.cpp -extern CScript _createmultisig_redeemScript(const UniValue& params); +extern CScript _createmultisig_redeemScript(CWallet * const pwallet, const UniValue& params); UniValue addmultisigaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 2 || request.params.size() > 3) { - string msg = "addmultisigaddress nrequired [\"key\",...] ( \"account\" )\n" + std::string msg = "addmultisigaddress nrequired [\"key\",...] ( \"account\" )\n" "\nAdd a nrequired-to-sign multisignature address to the wallet.\n" "Each key is a Bitcoin address or hex-encoded public key.\n" "If 'account' is specified (DEPRECATED), assign address to that account.\n" @@ -1024,41 +1036,44 @@ UniValue addmultisigaddress(const JSONRPCRequest& request) "\nAs json rpc call\n" + HelpExampleRpc("addmultisigaddress", "2, \"[\\\"16sSauSf5pF2UkUwvKGq4qjNRzBZYqgEL5\\\",\\\"171sgjn4YtPu27adkKGrdDwzRTxnRkBfKV\\\"]\"") ; - throw runtime_error(msg); + throw std::runtime_error(msg); } - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - string strAccount; + std::string strAccount; if (request.params.size() > 2) strAccount = AccountFromValue(request.params[2]); // Construct using pay-to-script-hash: - CScript inner = _createmultisig_redeemScript(request.params); + CScript inner = _createmultisig_redeemScript(pwallet, request.params); CScriptID innerID(inner); - pwalletMain->AddCScript(inner); + pwallet->AddCScript(inner); - pwalletMain->SetAddressBook(innerID, strAccount, "send"); + pwallet->SetAddressBook(innerID, strAccount, "send"); return CBitcoinAddress(innerID).ToString(); } class Witnessifier : public boost::static_visitor<bool> { public: + CWallet * const pwallet; CScriptID result; + Witnessifier(CWallet *_pwallet) : pwallet(_pwallet) {} + bool operator()(const CNoDestination &dest) const { return false; } bool operator()(const CKeyID &keyID) { CPubKey pubkey; - if (pwalletMain) { + if (pwallet) { CScript basescript = GetScriptForDestination(keyID); isminetype typ; - typ = IsMine(*pwalletMain, basescript, SIGVERSION_WITNESS_V0); + typ = IsMine(*pwallet, basescript, SIGVERSION_WITNESS_V0); if (typ != ISMINE_SPENDABLE && typ != ISMINE_WATCH_SOLVABLE) return false; CScript witscript = GetScriptForWitness(basescript); - pwalletMain->AddCScript(witscript); + pwallet->AddCScript(witscript); result = CScriptID(witscript); return true; } @@ -1067,7 +1082,7 @@ public: bool operator()(const CScriptID &scriptID) { CScript subscript; - if (pwalletMain && pwalletMain->GetCScript(scriptID, subscript)) { + if (pwallet && pwallet->GetCScript(scriptID, subscript)) { int witnessversion; std::vector<unsigned char> witprog; if (subscript.IsWitnessProgram(witnessversion, witprog)) { @@ -1075,11 +1090,11 @@ public: return true; } isminetype typ; - typ = IsMine(*pwalletMain, subscript, SIGVERSION_WITNESS_V0); + typ = IsMine(*pwallet, subscript, SIGVERSION_WITNESS_V0); if (typ != ISMINE_SPENDABLE && typ != ISMINE_WATCH_SOLVABLE) return false; CScript witscript = GetScriptForWitness(subscript); - pwalletMain->AddCScript(witscript); + pwallet->AddCScript(witscript); result = CScriptID(witscript); return true; } @@ -1089,12 +1104,14 @@ public: UniValue addwitnessaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 1) { - string msg = "addwitnessaddress \"address\"\n" + std::string msg = "addwitnessaddress \"address\"\n" "\nAdd a witness address for a script (with pubkey or redeemscript known).\n" "It returns the witness script.\n" @@ -1105,7 +1122,7 @@ UniValue addwitnessaddress(const JSONRPCRequest& request) "\"witnessaddress\", (string) The value of the new address (P2SH of witness script).\n" "}\n" ; - throw runtime_error(msg); + throw std::runtime_error(msg); } { @@ -1119,14 +1136,14 @@ UniValue addwitnessaddress(const JSONRPCRequest& request) if (!address.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); - Witnessifier w; + Witnessifier w(pwallet); CTxDestination dest = address.Get(); bool ret = boost::apply_visitor(w, dest); if (!ret) { throw JSONRPCError(RPC_WALLET_ERROR, "Public key or redeemscript not known to wallet, or the key is uncompressed"); } - pwalletMain->SetAddressBook(w.result, "", "receive"); + pwallet->SetAddressBook(w.result, "", "receive"); return CBitcoinAddress(w.result).ToString(); } @@ -1135,7 +1152,7 @@ struct tallyitem { CAmount nAmount; int nConf; - vector<uint256> txids; + std::vector<uint256> txids; bool fIsWatchonly; tallyitem() { @@ -1145,7 +1162,7 @@ struct tallyitem } }; -UniValue ListReceived(const UniValue& params, bool fByAccounts) +UniValue ListReceived(CWallet * const pwallet, const UniValue& params, bool fByAccounts) { // Minimum confirmations int nMinDepth = 1; @@ -1163,10 +1180,9 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) filter = filter | ISMINE_WATCH_ONLY; // Tally - map<CBitcoinAddress, tallyitem> mapTally; - for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; + std::map<CBitcoinAddress, tallyitem> mapTally; + for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; @@ -1181,13 +1197,13 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) if (!ExtractDestination(txout.scriptPubKey, address)) continue; - isminefilter mine = IsMine(*pwalletMain, address); + isminefilter mine = IsMine(*pwallet, address); if(!(mine & filter)) continue; tallyitem& item = mapTally[address]; item.nAmount += txout.nValue; - item.nConf = min(item.nConf, nDepth); + item.nConf = std::min(item.nConf, nDepth); item.txids.push_back(wtx.GetHash()); if (mine & ISMINE_WATCH_ONLY) item.fIsWatchonly = true; @@ -1196,12 +1212,11 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) // Reply UniValue ret(UniValue::VARR); - map<string, tallyitem> mapAccountTally; - BOOST_FOREACH(const PAIRTYPE(CBitcoinAddress, CAddressBookData)& item, pwalletMain->mapAddressBook) - { + std::map<std::string, tallyitem> mapAccountTally; + for (const std::pair<CBitcoinAddress, CAddressBookData>& item : pwallet->mapAddressBook) { const CBitcoinAddress& address = item.first; - const string& strAccount = item.second.name; - map<CBitcoinAddress, tallyitem>::iterator it = mapTally.find(address); + const std::string& strAccount = item.second.name; + std::map<CBitcoinAddress, tallyitem>::iterator it = mapTally.find(address); if (it == mapTally.end() && !fIncludeEmpty) continue; @@ -1219,7 +1234,7 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) { tallyitem& _item = mapAccountTally[strAccount]; _item.nAmount += nAmount; - _item.nConf = min(_item.nConf, nConf); + _item.nConf = std::min(_item.nConf, nConf); _item.fIsWatchonly = fIsWatchonly; } else @@ -1248,7 +1263,7 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) if (fByAccounts) { - for (map<string, tallyitem>::iterator it = mapAccountTally.begin(); it != mapAccountTally.end(); ++it) + for (std::map<std::string, tallyitem>::iterator it = mapAccountTally.begin(); it != mapAccountTally.end(); ++it) { CAmount nAmount = (*it).second.nAmount; int nConf = (*it).second.nConf; @@ -1267,11 +1282,13 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) UniValue listreceivedbyaddress(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 3) - throw runtime_error( + throw std::runtime_error( "listreceivedbyaddress ( minconf include_empty include_watchonly)\n" "\nList balances by receiving address.\n" "\nArguments:\n" @@ -1302,18 +1319,20 @@ UniValue listreceivedbyaddress(const JSONRPCRequest& request) + HelpExampleRpc("listreceivedbyaddress", "6, true, true") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - return ListReceived(request.params, false); + return ListReceived(pwallet, request.params, false); } UniValue listreceivedbyaccount(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 3) - throw runtime_error( + throw std::runtime_error( "listreceivedbyaccount ( minconf include_empty include_watchonly)\n" "\nDEPRECATED. List balances by account.\n" "\nArguments:\n" @@ -1339,9 +1358,9 @@ UniValue listreceivedbyaccount(const JSONRPCRequest& request) + HelpExampleRpc("listreceivedbyaccount", "6, true, true") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - return ListReceived(request.params, true); + return ListReceived(pwallet, request.params, true); } static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) @@ -1351,16 +1370,16 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) entry.push_back(Pair("address", addr.ToString())); } -void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter) +void ListTransactions(CWallet* const pwallet, const CWalletTx& wtx, const std::string& strAccount, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter) { CAmount nFee; - string strSentAccount; - list<COutputEntry> listReceived; - list<COutputEntry> listSent; + std::string strSentAccount; + std::list<COutputEntry> listReceived; + std::list<COutputEntry> listSent; wtx.GetAmounts(listReceived, listSent, nFee, strSentAccount, filter); - bool fAllAccounts = (strAccount == string("*")); + bool fAllAccounts = (strAccount == std::string("*")); bool involvesWatchonly = wtx.IsFromMe(ISMINE_WATCH_ONLY); // Sent @@ -1369,14 +1388,16 @@ void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDe BOOST_FOREACH(const COutputEntry& s, listSent) { UniValue entry(UniValue::VOBJ); - if(involvesWatchonly || (::IsMine(*pwalletMain, s.destination) & ISMINE_WATCH_ONLY)) + if (involvesWatchonly || (::IsMine(*pwallet, s.destination) & ISMINE_WATCH_ONLY)) { entry.push_back(Pair("involvesWatchonly", true)); + } entry.push_back(Pair("account", strSentAccount)); MaybePushAddress(entry, s.destination); entry.push_back(Pair("category", "send")); entry.push_back(Pair("amount", ValueFromAmount(-s.amount))); - if (pwalletMain->mapAddressBook.count(s.destination)) - entry.push_back(Pair("label", pwalletMain->mapAddressBook[s.destination].name)); + if (pwallet->mapAddressBook.count(s.destination)) { + entry.push_back(Pair("label", pwallet->mapAddressBook[s.destination].name)); + } entry.push_back(Pair("vout", s.vout)); entry.push_back(Pair("fee", ValueFromAmount(-nFee))); if (fLong) @@ -1391,14 +1412,16 @@ void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDe { BOOST_FOREACH(const COutputEntry& r, listReceived) { - string account; - if (pwalletMain->mapAddressBook.count(r.destination)) - account = pwalletMain->mapAddressBook[r.destination].name; + std::string account; + if (pwallet->mapAddressBook.count(r.destination)) { + account = pwallet->mapAddressBook[r.destination].name; + } if (fAllAccounts || (account == strAccount)) { UniValue entry(UniValue::VOBJ); - if(involvesWatchonly || (::IsMine(*pwalletMain, r.destination) & ISMINE_WATCH_ONLY)) + if (involvesWatchonly || (::IsMine(*pwallet, r.destination) & ISMINE_WATCH_ONLY)) { entry.push_back(Pair("involvesWatchonly", true)); + } entry.push_back(Pair("account", account)); MaybePushAddress(entry, r.destination); if (wtx.IsCoinBase()) @@ -1415,8 +1438,9 @@ void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDe entry.push_back(Pair("category", "receive")); } entry.push_back(Pair("amount", ValueFromAmount(r.amount))); - if (pwalletMain->mapAddressBook.count(r.destination)) + if (pwallet->mapAddressBook.count(r.destination)) { entry.push_back(Pair("label", account)); + } entry.push_back(Pair("vout", r.vout)); if (fLong) WalletTxToJSON(wtx, entry); @@ -1426,9 +1450,9 @@ void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDe } } -void AcentryToJSON(const CAccountingEntry& acentry, const string& strAccount, UniValue& ret) +void AcentryToJSON(const CAccountingEntry& acentry, const std::string& strAccount, UniValue& ret) { - bool fAllAccounts = (strAccount == string("*")); + bool fAllAccounts = (strAccount == std::string("*")); if (fAllAccounts || acentry.strAccount == strAccount) { @@ -1445,11 +1469,13 @@ void AcentryToJSON(const CAccountingEntry& acentry, const string& strAccount, Un UniValue listtransactions(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 4) - throw runtime_error( + throw std::runtime_error( "listtransactions ( \"account\" count skip include_watchonly)\n" "\nReturns up to 'count' most recent transactions skipping the first 'from' transactions for account 'account'.\n" "\nArguments:\n" @@ -1508,9 +1534,9 @@ UniValue listtransactions(const JSONRPCRequest& request) + HelpExampleRpc("listtransactions", "\"*\", 20, 100") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - string strAccount = "*"; + std::string strAccount = "*"; if (request.params.size() > 0) strAccount = request.params[0].get_str(); int nCount = 10; @@ -1531,14 +1557,14 @@ UniValue listtransactions(const JSONRPCRequest& request) UniValue ret(UniValue::VARR); - const CWallet::TxItems & txOrdered = pwalletMain->wtxOrdered; + const CWallet::TxItems & txOrdered = pwallet->wtxOrdered; // iterate backwards until we have nCount items to return: for (CWallet::TxItems::const_reverse_iterator it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { CWalletTx *const pwtx = (*it).second.first; if (pwtx != 0) - ListTransactions(*pwtx, strAccount, 0, true, ret, filter); + ListTransactions(pwallet, *pwtx, strAccount, 0, true, ret, filter); CAccountingEntry *const pacentry = (*it).second.second; if (pacentry != 0) AcentryToJSON(*pacentry, strAccount, ret); @@ -1552,11 +1578,11 @@ UniValue listtransactions(const JSONRPCRequest& request) if ((nFrom + nCount) > (int)ret.size()) nCount = ret.size() - nFrom; - vector<UniValue> arrTmp = ret.getValues(); + std::vector<UniValue> arrTmp = ret.getValues(); - vector<UniValue>::iterator first = arrTmp.begin(); + std::vector<UniValue>::iterator first = arrTmp.begin(); std::advance(first, nFrom); - vector<UniValue>::iterator last = arrTmp.begin(); + std::vector<UniValue>::iterator last = arrTmp.begin(); std::advance(last, nFrom+nCount); if (last != arrTmp.end()) arrTmp.erase(last, arrTmp.end()); @@ -1573,11 +1599,13 @@ UniValue listtransactions(const JSONRPCRequest& request) UniValue listaccounts(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "listaccounts ( minconf include_watchonly)\n" "\nDEPRECATED. Returns Object that has account names as keys, account balances as values.\n" "\nArguments:\n" @@ -1599,7 +1627,7 @@ UniValue listaccounts(const JSONRPCRequest& request) + HelpExampleRpc("listaccounts", "6") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); int nMinDepth = 1; if (request.params.size() > 0) @@ -1609,19 +1637,19 @@ UniValue listaccounts(const JSONRPCRequest& request) if(request.params[1].get_bool()) includeWatchonly = includeWatchonly | ISMINE_WATCH_ONLY; - map<string, CAmount> mapAccountBalances; - BOOST_FOREACH(const PAIRTYPE(CTxDestination, CAddressBookData)& entry, pwalletMain->mapAddressBook) { - if (IsMine(*pwalletMain, entry.first) & includeWatchonly) // This address belongs to me + std::map<std::string, CAmount> mapAccountBalances; + for (const std::pair<CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { + if (IsMine(*pwallet, entry.first) & includeWatchonly) { // This address belongs to me mapAccountBalances[entry.second.name] = 0; + } } - for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; + for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + const CWalletTx& wtx = pairWtx.second; CAmount nFee; - string strSentAccount; - list<COutputEntry> listReceived; - list<COutputEntry> listSent; + std::string strSentAccount; + std::list<COutputEntry> listReceived; + std::list<COutputEntry> listSent; int nDepth = wtx.GetDepthInMainChain(); if (wtx.GetBlocksToMaturity() > 0 || nDepth < 0) continue; @@ -1632,19 +1660,20 @@ UniValue listaccounts(const JSONRPCRequest& request) if (nDepth >= nMinDepth) { BOOST_FOREACH(const COutputEntry& r, listReceived) - if (pwalletMain->mapAddressBook.count(r.destination)) - mapAccountBalances[pwalletMain->mapAddressBook[r.destination].name] += r.amount; + if (pwallet->mapAddressBook.count(r.destination)) { + mapAccountBalances[pwallet->mapAddressBook[r.destination].name] += r.amount; + } else mapAccountBalances[""] += r.amount; } } - const list<CAccountingEntry> & acentries = pwalletMain->laccentries; + const std::list<CAccountingEntry>& acentries = pwallet->laccentries; BOOST_FOREACH(const CAccountingEntry& entry, acentries) mapAccountBalances[entry.strAccount] += entry.nCreditDebit; UniValue ret(UniValue::VOBJ); - BOOST_FOREACH(const PAIRTYPE(string, CAmount)& accountBalance, mapAccountBalances) { + BOOST_FOREACH(const PAIRTYPE(std::string, CAmount)& accountBalance, mapAccountBalances) { ret.push_back(Pair(accountBalance.first, ValueFromAmount(accountBalance.second))); } return ret; @@ -1652,11 +1681,13 @@ UniValue listaccounts(const JSONRPCRequest& request) UniValue listsinceblock(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp) - throw runtime_error( + throw std::runtime_error( "listsinceblock ( \"blockhash\" target_confirmations include_watchonly)\n" "\nGet all transactions in blocks since block [blockhash], or all transactions if omitted\n" "\nArguments:\n" @@ -1696,7 +1727,7 @@ UniValue listsinceblock(const JSONRPCRequest& request) + HelpExampleRpc("listsinceblock", "\"000000000000000bacf66f7497b7dc45ef753ee9a7d38571037cdb1a57f663ad\", 6") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); const CBlockIndex *pindex = NULL; int target_confirms = 1; @@ -1738,12 +1769,11 @@ UniValue listsinceblock(const JSONRPCRequest& request) UniValue transactions(UniValue::VARR); - for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); it++) - { - CWalletTx tx = (*it).second; + for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + CWalletTx tx = pairWtx.second; if (depth == -1 || tx.GetDepthInMainChain() < depth) - ListTransactions(tx, "*", 0, true, transactions, filter); + ListTransactions(pwallet, tx, "*", 0, true, transactions, filter); } CBlockIndex *pblockLast = chainActive[chainActive.Height() + 1 - target_confirms]; @@ -1758,11 +1788,13 @@ UniValue listsinceblock(const JSONRPCRequest& request) UniValue gettransaction(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "gettransaction \"txid\" ( include_watchonly )\n" "\nGet detailed information about in-wallet transaction <txid>\n" "\nArguments:\n" @@ -1793,7 +1825,7 @@ UniValue gettransaction(const JSONRPCRequest& request) " \"fee\": x.xxx, (numeric) The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the \n" " 'send' category of transactions.\n" " \"abandoned\": xxx (bool) 'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" - " 'send' category of transactions.\n" + " 'send' category of transactions.\n" " }\n" " ,...\n" " ],\n" @@ -1806,7 +1838,7 @@ UniValue gettransaction(const JSONRPCRequest& request) + HelpExampleRpc("gettransaction", "\"1075db55d416d3ca199f55b6084e2115b9345e16c5cf302fc80e9d5fbf5d48d\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; hash.SetHex(request.params[0].get_str()); @@ -1817,9 +1849,10 @@ UniValue gettransaction(const JSONRPCRequest& request) filter = filter | ISMINE_WATCH_ONLY; UniValue entry(UniValue::VOBJ); - if (!pwalletMain->mapWallet.count(hash)) + if (!pwallet->mapWallet.count(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid or non-wallet transaction id"); - const CWalletTx& wtx = pwalletMain->mapWallet[hash]; + } + const CWalletTx& wtx = pwallet->mapWallet[hash]; CAmount nCredit = wtx.GetCredit(filter); CAmount nDebit = wtx.GetDebit(filter); @@ -1833,10 +1866,10 @@ UniValue gettransaction(const JSONRPCRequest& request) WalletTxToJSON(wtx, entry); UniValue details(UniValue::VARR); - ListTransactions(wtx, "*", 0, false, details, filter); + ListTransactions(pwallet, wtx, "*", 0, false, details, filter); entry.push_back(Pair("details", details)); - string strHex = EncodeHexTx(static_cast<CTransaction>(wtx), RPCSerializationFlags()); + std::string strHex = EncodeHexTx(static_cast<CTransaction>(wtx), RPCSerializationFlags()); entry.push_back(Pair("hex", strHex)); return entry; @@ -1844,11 +1877,13 @@ UniValue gettransaction(const JSONRPCRequest& request) UniValue abandontransaction(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "abandontransaction \"txid\"\n" "\nMark in-wallet transaction <txid> as abandoned\n" "This will mark this transaction and all its in-wallet descendants as abandoned which will allow\n" @@ -1863,15 +1898,17 @@ UniValue abandontransaction(const JSONRPCRequest& request) + HelpExampleRpc("abandontransaction", "\"1075db55d416d3ca199f55b6084e2115b9345e16c5cf302fc80e9d5fbf5d48d\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; hash.SetHex(request.params[0].get_str()); - if (!pwalletMain->mapWallet.count(hash)) + if (!pwallet->mapWallet.count(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid or non-wallet transaction id"); - if (!pwalletMain->AbandonTransaction(hash)) + } + if (!pwallet->AbandonTransaction(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not eligible for abandonment"); + } return NullUniValue; } @@ -1879,11 +1916,13 @@ UniValue abandontransaction(const JSONRPCRequest& request) UniValue backupwallet(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "backupwallet \"destination\"\n" "\nSafely copies current wallet file to destination, which can be a directory or a path with filename.\n" "\nArguments:\n" @@ -1893,11 +1932,12 @@ UniValue backupwallet(const JSONRPCRequest& request) + HelpExampleRpc("backupwallet", "\"backup.dat\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - string strDest = request.params[0].get_str(); - if (!pwalletMain->BackupWallet(strDest)) + std::string strDest = request.params[0].get_str(); + if (!pwallet->BackupWallet(strDest)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet backup failed!"); + } return NullUniValue; } @@ -1905,14 +1945,16 @@ UniValue backupwallet(const JSONRPCRequest& request) UniValue keypoolrefill(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "keypoolrefill ( newsize )\n" "\nFills the keypool." - + HelpRequiringPassphrase() + "\n" + + HelpRequiringPassphrase(pwallet) + "\n" "\nArguments\n" "1. newsize (numeric, optional, default=100) The new keypool size\n" "\nExamples:\n" @@ -1920,7 +1962,7 @@ UniValue keypoolrefill(const JSONRPCRequest& request) + HelpExampleRpc("keypoolrefill", "") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); // 0 is interpreted by TopUpKeyPool() as the default keypool size given by -keypool unsigned int kpSize = 0; @@ -1930,11 +1972,12 @@ UniValue keypoolrefill(const JSONRPCRequest& request) kpSize = (unsigned int)request.params[0].get_int(); } - EnsureWalletIsUnlocked(); - pwalletMain->TopUpKeyPool(kpSize); + EnsureWalletIsUnlocked(pwallet); + pwallet->TopUpKeyPool(kpSize); - if (pwalletMain->GetKeyPoolSize() < kpSize) + if (pwallet->GetKeyPoolSize() < kpSize) { throw JSONRPCError(RPC_WALLET_ERROR, "Error refreshing keypool."); + } return NullUniValue; } @@ -1942,18 +1985,20 @@ UniValue keypoolrefill(const JSONRPCRequest& request) static void LockWallet(CWallet* pWallet) { - LOCK(cs_nWalletUnlockTime); - nWalletUnlockTime = 0; + LOCK(pWallet->cs_wallet); + pWallet->nRelockTime = 0; pWallet->Lock(); } UniValue walletpassphrase(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } - if (pwalletMain->IsCrypted() && (request.fHelp || request.params.size() != 2)) - throw runtime_error( + if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 2)) { + throw std::runtime_error( "walletpassphrase \"passphrase\" timeout\n" "\nStores the wallet decryption key in memory for 'timeout' seconds.\n" "This is needed prior to performing transactions related to private keys such as sending bitcoins\n" @@ -1971,13 +2016,15 @@ UniValue walletpassphrase(const JSONRPCRequest& request) "\nAs json rpc call\n" + HelpExampleRpc("walletpassphrase", "\"my pass phrase\", 60") ); + } - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); if (request.fHelp) return true; - if (!pwalletMain->IsCrypted()) + if (!pwallet->IsCrypted()) { throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletpassphrase was called."); + } // Note that the walletpassphrase is stored in request.params[0] which is not mlock()ed SecureString strWalletPass; @@ -1988,20 +2035,20 @@ UniValue walletpassphrase(const JSONRPCRequest& request) if (strWalletPass.length() > 0) { - if (!pwalletMain->Unlock(strWalletPass)) + if (!pwallet->Unlock(strWalletPass)) { throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); + } } else - throw runtime_error( + throw std::runtime_error( "walletpassphrase <passphrase> <timeout>\n" "Stores the wallet decryption key in memory for <timeout> seconds."); - pwalletMain->TopUpKeyPool(); + pwallet->TopUpKeyPool(); int64_t nSleepTime = request.params[1].get_int64(); - LOCK(cs_nWalletUnlockTime); - nWalletUnlockTime = GetTime() + nSleepTime; - RPCRunLater("lockwallet", boost::bind(LockWallet, pwalletMain), nSleepTime); + pwallet->nRelockTime = GetTime() + nSleepTime; + RPCRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), boost::bind(LockWallet, pwallet), nSleepTime); return NullUniValue; } @@ -2009,11 +2056,13 @@ UniValue walletpassphrase(const JSONRPCRequest& request) UniValue walletpassphrasechange(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } - if (pwalletMain->IsCrypted() && (request.fHelp || request.params.size() != 2)) - throw runtime_error( + if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 2)) { + throw std::runtime_error( "walletpassphrasechange \"oldpassphrase\" \"newpassphrase\"\n" "\nChanges the wallet passphrase from 'oldpassphrase' to 'newpassphrase'.\n" "\nArguments:\n" @@ -2023,13 +2072,15 @@ UniValue walletpassphrasechange(const JSONRPCRequest& request) + HelpExampleCli("walletpassphrasechange", "\"old one\" \"new one\"") + HelpExampleRpc("walletpassphrasechange", "\"old one\", \"new one\"") ); + } - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); if (request.fHelp) return true; - if (!pwalletMain->IsCrypted()) + if (!pwallet->IsCrypted()) { throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletpassphrasechange was called."); + } // TODO: get rid of these .c_str() calls by implementing SecureString::operator=(std::string) // Alternately, find a way to make request.params[0] mlock()'d to begin with. @@ -2042,12 +2093,13 @@ UniValue walletpassphrasechange(const JSONRPCRequest& request) strNewWalletPass = request.params[1].get_str().c_str(); if (strOldWalletPass.length() < 1 || strNewWalletPass.length() < 1) - throw runtime_error( + throw std::runtime_error( "walletpassphrasechange <oldpassphrase> <newpassphrase>\n" "Changes the wallet passphrase from <oldpassphrase> to <newpassphrase>."); - if (!pwalletMain->ChangeWalletPassphrase(strOldWalletPass, strNewWalletPass)) + if (!pwallet->ChangeWalletPassphrase(strOldWalletPass, strNewWalletPass)) { throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); + } return NullUniValue; } @@ -2055,11 +2107,13 @@ UniValue walletpassphrasechange(const JSONRPCRequest& request) UniValue walletlock(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } - if (pwalletMain->IsCrypted() && (request.fHelp || request.params.size() != 0)) - throw runtime_error( + if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 0)) { + throw std::runtime_error( "walletlock\n" "\nRemoves the wallet encryption key from memory, locking the wallet.\n" "After calling this method, you will need to call walletpassphrase again\n" @@ -2074,31 +2128,32 @@ UniValue walletlock(const JSONRPCRequest& request) "\nAs json rpc call\n" + HelpExampleRpc("walletlock", "") ); + } - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); if (request.fHelp) return true; - if (!pwalletMain->IsCrypted()) + if (!pwallet->IsCrypted()) { throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletlock was called."); - - { - LOCK(cs_nWalletUnlockTime); - pwalletMain->Lock(); - nWalletUnlockTime = 0; } + pwallet->Lock(); + pwallet->nRelockTime = 0; + return NullUniValue; } UniValue encryptwallet(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } - if (!pwalletMain->IsCrypted() && (request.fHelp || request.params.size() != 1)) - throw runtime_error( + if (!pwallet->IsCrypted() && (request.fHelp || request.params.size() != 1)) { + throw std::runtime_error( "encryptwallet \"passphrase\"\n" "\nEncrypts the wallet with 'passphrase'. This is for first time encryption.\n" "After this, any calls that interact with private keys such as sending or signing \n" @@ -2120,13 +2175,15 @@ UniValue encryptwallet(const JSONRPCRequest& request) "\nAs a json rpc call\n" + HelpExampleRpc("encryptwallet", "\"my pass phrase\"") ); + } - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); if (request.fHelp) return true; - if (pwalletMain->IsCrypted()) + if (pwallet->IsCrypted()) { throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an encrypted wallet, but encryptwallet was called."); + } // TODO: get rid of this .c_str() by implementing SecureString::operator=(std::string) // Alternately, find a way to make request.params[0] mlock()'d to begin with. @@ -2135,12 +2192,13 @@ UniValue encryptwallet(const JSONRPCRequest& request) strWalletPass = request.params[0].get_str().c_str(); if (strWalletPass.length() < 1) - throw runtime_error( + throw std::runtime_error( "encryptwallet <passphrase>\n" "Encrypts the wallet with <passphrase>."); - if (!pwalletMain->EncryptWallet(strWalletPass)) + if (!pwallet->EncryptWallet(strWalletPass)) { throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, "Error: Failed to encrypt the wallet."); + } // BDB seems to have a bad habit of writing old data into // slack space in .dat files; that is bad if the old data is @@ -2151,11 +2209,13 @@ UniValue encryptwallet(const JSONRPCRequest& request) UniValue lockunspent(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "lockunspent unlock ([{\"txid\":\"txid\",\"vout\":n},...])\n" "\nUpdates list of temporarily unspendable outputs.\n" "Temporarily lock (unlock=false) or unlock (unlock=true) specified transaction outputs.\n" @@ -2191,7 +2251,7 @@ UniValue lockunspent(const JSONRPCRequest& request) + HelpExampleRpc("lockunspent", "false, \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\"") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); if (request.params.size() == 1) RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VBOOL)); @@ -2202,7 +2262,7 @@ UniValue lockunspent(const JSONRPCRequest& request) if (request.params.size() == 1) { if (fUnlock) - pwalletMain->UnlockAllCoins(); + pwallet->UnlockAllCoins(); return true; } @@ -2219,7 +2279,7 @@ UniValue lockunspent(const JSONRPCRequest& request) {"vout", UniValueType(UniValue::VNUM)}, }); - string txid = find_value(o, "txid").get_str(); + std::string txid = find_value(o, "txid").get_str(); if (!IsHex(txid)) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected hex txid"); @@ -2230,9 +2290,9 @@ UniValue lockunspent(const JSONRPCRequest& request) COutPoint outpt(uint256S(txid), nOutput); if (fUnlock) - pwalletMain->UnlockCoin(outpt); + pwallet->UnlockCoin(outpt); else - pwalletMain->LockCoin(outpt); + pwallet->LockCoin(outpt); } return true; @@ -2240,11 +2300,13 @@ UniValue lockunspent(const JSONRPCRequest& request) UniValue listlockunspent(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() > 0) - throw runtime_error( + throw std::runtime_error( "listlockunspent\n" "\nReturns list of temporarily unspendable outputs.\n" "See the lockunspent call to lock and unlock transactions for spending.\n" @@ -2269,10 +2331,10 @@ UniValue listlockunspent(const JSONRPCRequest& request) + HelpExampleRpc("listlockunspent", "") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - vector<COutPoint> vOutpts; - pwalletMain->ListLockedCoins(vOutpts); + std::vector<COutPoint> vOutpts; + pwallet->ListLockedCoins(vOutpts); UniValue ret(UniValue::VARR); @@ -2289,11 +2351,13 @@ UniValue listlockunspent(const JSONRPCRequest& request) UniValue settxfee(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "settxfee amount\n" "\nSet the transaction fee per kB. Overwrites the paytxfee parameter.\n" "\nArguments:\n" @@ -2305,7 +2369,7 @@ UniValue settxfee(const JSONRPCRequest& request) + HelpExampleRpc("settxfee", "0.00001") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); // Amount CAmount nAmount = AmountFromValue(request.params[0]); @@ -2316,45 +2380,54 @@ UniValue settxfee(const JSONRPCRequest& request) UniValue getwalletinfo(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getwalletinfo\n" "Returns an object containing various wallet state info.\n" "\nResult:\n" "{\n" - " \"walletversion\": xxxxx, (numeric) the wallet version\n" - " \"balance\": xxxxxxx, (numeric) the total confirmed balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" - " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" - " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" - " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" - " \"hdmasterkeyid\": \"<hash160>\" (string) the Hash160 of the HD master pubkey\n" + " \"walletversion\": xxxxx, (numeric) the wallet version\n" + " \"balance\": xxxxxxx, (numeric) the total confirmed balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated (only counts external keys)\n" + " \"keypoolsize_hd_internal\": xxxx, (numeric) how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)\n" + " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" + " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" + " \"hdmasterkeyid\": \"<hash160>\" (string) the Hash160 of the HD master pubkey\n" "}\n" "\nExamples:\n" + HelpExampleCli("getwalletinfo", "") + HelpExampleRpc("getwalletinfo", "") ); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); UniValue obj(UniValue::VOBJ); - obj.push_back(Pair("walletversion", pwalletMain->GetVersion())); - obj.push_back(Pair("balance", ValueFromAmount(pwalletMain->GetBalance()))); - obj.push_back(Pair("unconfirmed_balance", ValueFromAmount(pwalletMain->GetUnconfirmedBalance()))); - obj.push_back(Pair("immature_balance", ValueFromAmount(pwalletMain->GetImmatureBalance()))); - obj.push_back(Pair("txcount", (int)pwalletMain->mapWallet.size())); - obj.push_back(Pair("keypoololdest", pwalletMain->GetOldestKeyPoolTime())); - obj.push_back(Pair("keypoolsize", (int)pwalletMain->GetKeyPoolSize())); - if (pwalletMain->IsCrypted()) - obj.push_back(Pair("unlocked_until", nWalletUnlockTime)); + + size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); + obj.push_back(Pair("walletversion", pwallet->GetVersion())); + obj.push_back(Pair("balance", ValueFromAmount(pwallet->GetBalance()))); + obj.push_back(Pair("unconfirmed_balance", ValueFromAmount(pwallet->GetUnconfirmedBalance()))); + obj.push_back(Pair("immature_balance", ValueFromAmount(pwallet->GetImmatureBalance()))); + obj.push_back(Pair("txcount", (int)pwallet->mapWallet.size())); + obj.push_back(Pair("keypoololdest", pwallet->GetOldestKeyPoolTime())); + obj.push_back(Pair("keypoolsize", (int64_t)kpExternalSize)); + CKeyID masterKeyID = pwallet->GetHDChain().masterKeyID; + if (!masterKeyID.IsNull() && pwallet->CanSupportFeature(FEATURE_HD_SPLIT)) { + obj.push_back(Pair("keypoolsize_hd_internal", (int64_t)(pwallet->GetKeyPoolSize() - kpExternalSize))); + } + if (pwallet->IsCrypted()) { + obj.push_back(Pair("unlocked_until", pwallet->nRelockTime)); + } obj.push_back(Pair("paytxfee", ValueFromAmount(payTxFee.GetFeePerK()))); - CKeyID masterKeyID = pwalletMain->GetHDChain().masterKeyID; if (!masterKeyID.IsNull()) obj.push_back(Pair("hdmasterkeyid", masterKeyID.GetHex())); return obj; @@ -2362,11 +2435,13 @@ UniValue getwalletinfo(const JSONRPCRequest& request) UniValue resendwallettransactions(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "resendwallettransactions\n" "Immediately re-broadcast unconfirmed wallet transactions to all peers.\n" "Intended only for testing; the wallet code periodically re-broadcasts\n" @@ -2377,9 +2452,9 @@ UniValue resendwallettransactions(const JSONRPCRequest& request) if (!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - LOCK2(cs_main, pwalletMain->cs_wallet); + LOCK2(cs_main, pwallet->cs_wallet); - std::vector<uint256> txids = pwalletMain->ResendWalletTransactionsBefore(GetTime(), g_connman.get()); + std::vector<uint256> txids = pwallet->ResendWalletTransactionsBefore(GetTime(), g_connman.get()); UniValue result(UniValue::VARR); BOOST_FOREACH(const uint256& txid, txids) { @@ -2390,27 +2465,34 @@ UniValue resendwallettransactions(const JSONRPCRequest& request) UniValue listunspent(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } - if (request.fHelp || request.params.size() > 4) - throw runtime_error( - "listunspent ( minconf maxconf [\"addresses\",...] [include_unsafe] )\n" + if (request.fHelp || request.params.size() > 5) + throw std::runtime_error( + "listunspent ( minconf maxconf [\"addresses\",...] [include_unsafe] [query_options])\n" "\nReturns array of unspent transaction outputs\n" "with between minconf and maxconf (inclusive) confirmations.\n" "Optionally filter to only include txouts paid to specified addresses.\n" "\nArguments:\n" "1. minconf (numeric, optional, default=1) The minimum confirmations to filter\n" "2. maxconf (numeric, optional, default=9999999) The maximum confirmations to filter\n" - "3. \"addresses\" (string) A json array of bitcoin addresses to filter\n" + "3. \"addresses\" (string) A json array of bitcoin addresses to filter\n" " [\n" - " \"address\" (string) bitcoin address\n" + " \"address\" (string) bitcoin address\n" " ,...\n" " ]\n" "4. include_unsafe (bool, optional, default=true) Include outputs that are not safe to spend\n" - " because they come from unconfirmed untrusted transactions or unconfirmed\n" - " replacement transactions (cases where we are less sure that a conflicting\n" - " transaction won't be mined).\n" + " See description of \"safe\" attribute below.\n" + "5. query_options (json, optional) JSON with query options\n" + " {\n" + " \"minimumAmount\" (numeric or string, default=0) Minimum value of each UTXO in " + CURRENCY_UNIT + "\n" + " \"maximumAmount\" (numeric or string, default=unlimited) Maximum value of each UTXO in " + CURRENCY_UNIT + "\n" + " \"maximumCount\" (numeric or string, default=unlimited) Maximum number of UTXOs\n" + " \"minimumSumAmount\" (numeric or string, default=unlimited) Minimum sum value of all UTXOs in " + CURRENCY_UNIT + "\n" + " }\n" "\nResult\n" "[ (array of json object)\n" " {\n" @@ -2423,7 +2505,10 @@ UniValue listunspent(const JSONRPCRequest& request) " \"confirmations\" : n, (numeric) The number of confirmations\n" " \"redeemScript\" : n (string) The redeemScript if scriptPubKey is P2SH\n" " \"spendable\" : xxx, (bool) Whether we have the private keys to spend this output\n" - " \"solvable\" : xxx (bool) Whether we know how to spend this output, ignoring the lack of keys\n" + " \"solvable\" : xxx, (bool) Whether we know how to spend this output, ignoring the lack of keys\n" + " \"safe\" : xxx (bool) Whether this output is considered safe to spend. Unconfirmed transactions\n" + " from outside keys and unconfirmed replacement transactions are considered unsafe\n" + " and are not eligible for spending by fundrawtransaction and sendtoaddress.\n" " }\n" " ,...\n" "]\n" @@ -2432,6 +2517,8 @@ UniValue listunspent(const JSONRPCRequest& request) + HelpExampleCli("listunspent", "") + HelpExampleCli("listunspent", "6 9999999 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + HelpExampleRpc("listunspent", "6, 9999999 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + + HelpExampleCli("listunspent", "6 9999999 '[]' true '{ \"minimumAmount\": 0.005 }'") + + HelpExampleRpc("listunspent", "6, 9999999, [] , true, { \"minimumAmount\": 0.005 } ") ); int nMinDepth = 1; @@ -2446,7 +2533,7 @@ UniValue listunspent(const JSONRPCRequest& request) nMaxDepth = request.params[1].get_int(); } - set<CBitcoinAddress> setAddress; + std::set<CBitcoinAddress> setAddress; if (request.params.size() > 2 && !request.params[2].isNull()) { RPCTypeCheckArgument(request.params[2], UniValue::VARR); UniValue inputs = request.params[2].get_array(); @@ -2454,9 +2541,9 @@ UniValue listunspent(const JSONRPCRequest& request) const UniValue& input = inputs[idx]; CBitcoinAddress address(input.get_str()); if (!address.IsValid()) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, string("Invalid Bitcoin address: ")+input.get_str()); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ")+input.get_str()); if (setAddress.count(address)) - throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid parameter, duplicated address: ")+input.get_str()); + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ")+input.get_str()); setAddress.insert(address); } } @@ -2467,15 +2554,34 @@ UniValue listunspent(const JSONRPCRequest& request) include_unsafe = request.params[3].get_bool(); } + CAmount nMinimumAmount = 0; + CAmount nMaximumAmount = MAX_MONEY; + CAmount nMinimumSumAmount = MAX_MONEY; + uint64_t nMaximumCount = 0; + + if (request.params.size() > 4) { + const UniValue& options = request.params[4].get_obj(); + + if (options.exists("minimumAmount")) + nMinimumAmount = AmountFromValue(options["minimumAmount"]); + + if (options.exists("maximumAmount")) + nMaximumAmount = AmountFromValue(options["maximumAmount"]); + + if (options.exists("minimumSumAmount")) + nMinimumSumAmount = AmountFromValue(options["minimumSumAmount"]); + + if (options.exists("maximumCount")) + nMaximumCount = options["maximumCount"].get_int64(); + } + UniValue results(UniValue::VARR); - vector<COutput> vecOutputs; - assert(pwalletMain != NULL); - LOCK2(cs_main, pwalletMain->cs_wallet); - pwalletMain->AvailableCoins(vecOutputs, !include_unsafe, NULL, true); - BOOST_FOREACH(const COutput& out, vecOutputs) { - if (out.nDepth < nMinDepth || out.nDepth > nMaxDepth) - continue; + std::vector<COutput> vecOutputs; + assert(pwallet != NULL); + LOCK2(cs_main, pwallet->cs_wallet); + pwallet->AvailableCoins(vecOutputs, !include_unsafe, NULL, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, nMinDepth, nMaxDepth); + BOOST_FOREACH(const COutput& out, vecOutputs) { CTxDestination address; const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); @@ -2490,14 +2596,16 @@ UniValue listunspent(const JSONRPCRequest& request) if (fValidAddress) { entry.push_back(Pair("address", CBitcoinAddress(address).ToString())); - if (pwalletMain->mapAddressBook.count(address)) - entry.push_back(Pair("account", pwalletMain->mapAddressBook[address].name)); + if (pwallet->mapAddressBook.count(address)) { + entry.push_back(Pair("account", pwallet->mapAddressBook[address].name)); + } if (scriptPubKey.IsPayToScriptHash()) { const CScriptID& hash = boost::get<CScriptID>(address); CScript redeemScript; - if (pwalletMain->GetCScript(hash, redeemScript)) + if (pwallet->GetCScript(hash, redeemScript)) { entry.push_back(Pair("redeemScript", HexStr(redeemScript.begin(), redeemScript.end()))); + } } } @@ -2506,6 +2614,7 @@ UniValue listunspent(const JSONRPCRequest& request) entry.push_back(Pair("confirmations", out.nDepth)); entry.push_back(Pair("spendable", out.fSpendable)); entry.push_back(Pair("solvable", out.fSolvable)); + entry.push_back(Pair("safe", out.fSafe)); results.push_back(entry); } @@ -2514,11 +2623,13 @@ UniValue listunspent(const JSONRPCRequest& request) UniValue fundrawtransaction(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { return NullUniValue; + } if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "fundrawtransaction \"hexstring\" ( options )\n" "\nAdd inputs to a transaction until it has enough in value to meet its out value.\n" "This will not modify existing inputs, and will add at most one change output to the outputs.\n" @@ -2575,7 +2686,7 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) CFeeRate feeRate = CFeeRate(0); bool overrideEstimatedFeerate = false; UniValue subtractFeeFromOutputs; - set<int> setSubtractFeeFromOutputs; + std::set<int> setSubtractFeeFromOutputs; if (request.params.size() > 1) { if (request.params[1].type() == UniValue::VBOOL) { @@ -2603,7 +2714,7 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) CBitcoinAddress address(options["changeAddress"].get_str()); if (!address.IsValid()) - throw JSONRPCError(RPC_INVALID_PARAMETER, "changeAddress must be a valid bitcoin address"); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "changeAddress must be a valid bitcoin address"); changeAddress = address.Get(); } @@ -2654,10 +2765,11 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) } CAmount nFeeOut; - string strFailReason; + std::string strFailReason; - if(!pwalletMain->FundTransaction(tx, nFeeOut, overrideEstimatedFeerate, feeRate, changePosition, strFailReason, includeWatching, lockUnspents, setSubtractFeeFromOutputs, reserveChangeKey, changeAddress)) - throw JSONRPCError(RPC_INTERNAL_ERROR, strFailReason); + if (!pwallet->FundTransaction(tx, nFeeOut, overrideEstimatedFeerate, feeRate, changePosition, strFailReason, includeWatching, lockUnspents, setSubtractFeeFromOutputs, reserveChangeKey, changeAddress)) { + throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); + } UniValue result(UniValue::VOBJ); result.push_back(Pair("hex", EncodeHexTx(tx))); @@ -2667,41 +2779,15 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) return result; } -// Calculate the size of the transaction assuming all signatures are max size -// Use DummySignatureCreator, which inserts 72 byte signatures everywhere. -// TODO: re-use this in CWallet::CreateTransaction (right now -// CreateTransaction uses the constructed dummy-signed tx to do a priority -// calculation, but we should be able to refactor after priority is removed). -// NOTE: this requires that all inputs must be in mapWallet (eg the tx should -// be IsAllFromMe). -int64_t CalculateMaximumSignedTxSize(const CTransaction &tx) -{ - CMutableTransaction txNew(tx); - std::vector<pair<CWalletTx *, unsigned int>> vCoins; - // Look up the inputs. We should have already checked that this transaction - // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our - // wallet, with a valid index into the vout array. - for (auto& input : tx.vin) { - const auto mi = pwalletMain->mapWallet.find(input.prevout.hash); - assert(mi != pwalletMain->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); - vCoins.emplace_back(make_pair(&(mi->second), input.prevout.n)); - } - if (!pwalletMain->DummySignTx(txNew, vCoins)) { - // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) - // implies that we can sign for every input. - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction contains inputs that cannot be signed"); - } - return GetVirtualTransactionSize(txNew); -} - UniValue bumpfee(const JSONRPCRequest& request) { - if (!EnsureWalletIsAvailable(request.fHelp)) { + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) return NullUniValue; - } if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw runtime_error( + throw std::runtime_error( "bumpfee \"txid\" ( options ) \n" "\nBumps the fee of an opt-in-RBF transaction T, replacing it with a new transaction B.\n" "An opt-in RBF transaction with the given txid must be in the wallet.\n" @@ -2711,7 +2797,7 @@ UniValue bumpfee(const JSONRPCRequest& request) "The command will fail if the wallet or mempool contains a transaction that spends one of T's outputs.\n" "By default, the new fee will be calculated automatically using estimatefee.\n" "The user can specify a confirmation target for estimatefee.\n" - "Alternatively, the user can specify totalFee, or use RPC setpaytxfee to set a higher fee rate.\n" + "Alternatively, the user can specify totalFee, or use RPC settxfee to set a higher fee rate.\n" "At a minimum, the new fee rate must be high enough to pay an additional new relay fee (incrementalfee\n" "returned by getnetworkinfo) to enter the node's mempool.\n" "\nArguments:\n" @@ -2747,65 +2833,8 @@ UniValue bumpfee(const JSONRPCRequest& request) uint256 hash; hash.SetHex(request.params[0].get_str()); - // retrieve the original tx from the wallet - LOCK2(cs_main, pwalletMain->cs_wallet); - EnsureWalletIsUnlocked(); - if (!pwalletMain->mapWallet.count(hash)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid or non-wallet transaction id"); - } - CWalletTx& wtx = pwalletMain->mapWallet[hash]; - - if (pwalletMain->HasWalletSpend(hash)) { - throw JSONRPCError(RPC_MISC_ERROR, "Transaction has descendants in the wallet"); - } - - { - LOCK(mempool.cs); - auto it = mempool.mapTx.find(hash); - if (it != mempool.mapTx.end() && it->GetCountWithDescendants() > 1) { - throw JSONRPCError(RPC_MISC_ERROR, "Transaction has descendants in the mempool"); - } - } - - if (wtx.GetDepthInMainChain() != 0) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction has been mined, or is conflicted with a mined transaction"); - } - - if (!SignalsOptInRBF(wtx)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction is not BIP 125 replaceable"); - } - - if (wtx.mapValue.count("replaced_by_txid")) { - throw JSONRPCError(RPC_INVALID_REQUEST, strprintf("Cannot bump transaction %s which was already bumped by transaction %s", hash.ToString(), wtx.mapValue.at("replaced_by_txid"))); - } - - // check that original tx consists entirely of our inputs - // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) - if (!pwalletMain->IsAllFromMe(wtx, ISMINE_SPENDABLE)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction contains inputs that don't belong to this wallet"); - } - - // figure out which output was change - // if there was no change output or multiple change outputs, fail - int nOutput = -1; - for (size_t i = 0; i < wtx.tx->vout.size(); ++i) { - if (pwalletMain->IsChange(wtx.tx->vout[i])) { - if (nOutput != -1) { - throw JSONRPCError(RPC_MISC_ERROR, "Transaction has multiple change outputs"); - } - nOutput = i; - } - } - if (nOutput == -1) { - throw JSONRPCError(RPC_MISC_ERROR, "Transaction does not have a change output"); - } - - // Calculate the expected size of the new transaction. - int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); - const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx); - // optional parameters - bool specifiedConfirmTarget = false; + bool ignoreGlobalPayTxFee = false; int newConfirmTarget = nTxConfirmTarget; CAmount totalFee = 0; bool replaceable = true; @@ -2822,18 +2851,18 @@ UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("confTarget") && options.exists("totalFee")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget and totalFee options should not both be set. Please provide either a confirmation target for fee estimation or an explicit total fee for the transaction."); } else if (options.exists("confTarget")) { - specifiedConfirmTarget = true; + // If the user has explicitly set a confTarget in this rpc call, + // then override the default logic that uses the global payTxFee + // instead of the confirmation target. + ignoreGlobalPayTxFee = true; newConfirmTarget = options["confTarget"].get_int(); if (newConfirmTarget <= 0) { // upper-bound will be checked by estimatefee/smartfee throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid confTarget (cannot be <= 0)"); } } else if (options.exists("totalFee")) { totalFee = options["totalFee"].get_int64(); - CAmount requiredFee = CWallet::GetRequiredFee(maxNewTxSize); - if (totalFee < requiredFee ) { - throw JSONRPCError(RPC_INVALID_PARAMETER, - strprintf("Insufficient totalFee (cannot be less than required fee %s)", - FormatMoney(requiredFee))); + if (totalFee <= 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid totalFee %s (must be greater than 0)", FormatMoney(totalFee))); } } @@ -2842,148 +2871,53 @@ UniValue bumpfee(const JSONRPCRequest& request) } } - // calculate the old fee and fee-rate - CAmount nOldFee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); - CFeeRate nOldFeeRate(nOldFee, txSize); - CAmount nNewFee; - CFeeRate nNewFeeRate; - // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to - // future proof against changes to network wide policy for incremental relay - // fee that our node may not be aware of. - CFeeRate walletIncrementalRelayFee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); - if (::incrementalRelayFee > walletIncrementalRelayFee) { - walletIncrementalRelayFee = ::incrementalRelayFee; - } - - if (totalFee > 0) { - CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); - if (totalFee < minTotalFee) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", - FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); - } - nNewFee = totalFee; - nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); - } else { - // if user specified a confirm target then don't consider any global payTxFee - if (specifiedConfirmTarget) { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, CAmount(0)); - } - // otherwise use the regular wallet logic to select payTxFee or default confirm target - else { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool); - } - - nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); - - // New fee rate must be at least old rate + minimum incremental relay rate - // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized - // in that unit (fee per kb). - // However, nOldFeeRate is a calculated value from the tx fee/size, so - // add 1 satoshi to the result, because it may have been rounded down. - if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { - nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); - nNewFee = nNewFeeRate.GetFee(maxNewTxSize); - } - } - - // Check that in all cases the new fee doesn't violate maxTxFee - if (nNewFee > maxTxFee) { - throw JSONRPCError(RPC_MISC_ERROR, - strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", - FormatMoney(nNewFee), FormatMoney(maxTxFee))); - } - - // check that fee rate is higher than mempool's minimum fee - // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) - // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, - // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a - // moment earlier. In this case, we report an error to the user, who may use totalFee to make an adjustment. - CFeeRate minMempoolFeeRate = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); - if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { - throw JSONRPCError(RPC_MISC_ERROR, strprintf("New fee rate (%s) is less than the minimum fee rate (%s) to get into the mempool. totalFee value should to be at least %s or settxfee value should be at least %s to add transaction.", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), FormatMoney(minMempoolFeeRate.GetFeePerK()))); - } - - // Now modify the output to increase the fee. - // If the output is not large enough to pay the fee, fail. - CAmount nDelta = nNewFee - nOldFee; - assert(nDelta > 0); - CMutableTransaction tx(*(wtx.tx)); - CTxOut* poutput = &(tx.vout[nOutput]); - if (poutput->nValue < nDelta) { - throw JSONRPCError(RPC_MISC_ERROR, "Change output is too small to bump the fee"); - } - - // If the output would become dust, discard it (converting the dust to fee) - poutput->nValue -= nDelta; - if (poutput->nValue <= poutput->GetDustThreshold(::dustRelayFee)) { - LogPrint("rpc", "Bumping fee and discarding dust output\n"); - nNewFee += poutput->nValue; - tx.vout.erase(tx.vout.begin() + nOutput); - } - - // Mark new tx not replaceable, if requested. - if (!replaceable) { - for (auto& input : tx.vin) { - if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; - } - } + LOCK2(cs_main, pwallet->cs_wallet); + EnsureWalletIsUnlocked(pwallet); - // sign the new tx - CTransaction txNewConst(tx); - int nIn = 0; - for (auto& input : tx.vin) { - std::map<uint256, CWalletTx>::const_iterator mi = pwalletMain->mapWallet.find(input.prevout.hash); - assert(mi != pwalletMain->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); - const CScript& scriptPubKey = mi->second.tx->vout[input.prevout.n].scriptPubKey; - const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; - SignatureData sigdata; - if (!ProduceSignature(TransactionSignatureCreator(pwalletMain, &txNewConst, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); + CFeeBumper feeBump(pwallet, hash, newConfirmTarget, ignoreGlobalPayTxFee, totalFee, replaceable); + BumpFeeResult res = feeBump.getResult(); + if (res != BumpFeeResult::OK) + { + switch(res) { + case BumpFeeResult::INVALID_ADDRESS_OR_KEY: + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, feeBump.getErrors()[0]); + break; + case BumpFeeResult::INVALID_REQUEST: + throw JSONRPCError(RPC_INVALID_REQUEST, feeBump.getErrors()[0]); + break; + case BumpFeeResult::INVALID_PARAMETER: + throw JSONRPCError(RPC_INVALID_PARAMETER, feeBump.getErrors()[0]); + break; + case BumpFeeResult::WALLET_ERROR: + throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); + break; + default: + throw JSONRPCError(RPC_MISC_ERROR, feeBump.getErrors()[0]); + break; } - UpdateTransaction(tx, nIn, sigdata); - nIn++; - } - - // commit/broadcast the tx - CReserveKey reservekey(pwalletMain); - CWalletTx wtxBumped(pwalletMain, MakeTransactionRef(std::move(tx))); - wtxBumped.mapValue = wtx.mapValue; - wtxBumped.mapValue["replaces_txid"] = hash.ToString(); - wtxBumped.vOrderForm = wtx.vOrderForm; - wtxBumped.strFromAccount = wtx.strFromAccount; - wtxBumped.fTimeReceivedIsTxTime = true; - wtxBumped.fFromMe = true; - CValidationState state; - if (!pwalletMain->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { - // NOTE: CommitTransaction never returns false, so this should never happen. - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); } - UniValue vErrors(UniValue::VARR); - if (state.IsInvalid()) { - // This can happen if the mempool rejected the transaction. Report - // what happened in the "errors" response. - vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); + // sign bumped transaction + if (!feeBump.signTransaction(pwallet)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } - - // mark the original tx as bumped - if (!pwalletMain->MarkReplaced(wtx.GetHash(), wtxBumped.GetHash())) { - // TODO: see if JSON-RPC has a standard way of returning a response - // along with an exception. It would be good to return information about - // wtxBumped to the caller even if marking the original transaction - // replaced does not succeed for some reason. - vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); + // commit the bumped transaction + if(!feeBump.commit(pwallet)) { + throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); } - UniValue result(UniValue::VOBJ); - result.push_back(Pair("txid", wtxBumped.GetHash().GetHex())); - result.push_back(Pair("origfee", ValueFromAmount(nOldFee))); - result.push_back(Pair("fee", ValueFromAmount(nNewFee))); - result.push_back(Pair("errors", vErrors)); + result.push_back(Pair("txid", feeBump.getBumpedTxId().GetHex())); + result.push_back(Pair("origfee", ValueFromAmount(feeBump.getOldFee()))); + result.push_back(Pair("fee", ValueFromAmount(feeBump.getNewFee()))); + UniValue errors(UniValue::VARR); + for (const std::string& err: feeBump.getErrors()) + errors.push_back(err); + result.push_back(Pair("errors", errors)); return result; } +extern UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue importprivkey(const JSONRPCRequest& request); extern UniValue importaddress(const JSONRPCRequest& request); @@ -3000,6 +2934,7 @@ static const CRPCCommand commands[] = { "rawtransactions", "fundrawtransaction", &fundrawtransaction, false, {"hexstring","options"} }, { "hidden", "resendwallettransactions", &resendwallettransactions, true, {} }, { "wallet", "abandontransaction", &abandontransaction, false, {"txid"} }, + { "wallet", "abortrescan", &abortrescan, false, {} }, { "wallet", "addmultisigaddress", &addmultisigaddress, true, {"nrequired","keys","account"} }, { "wallet", "addwitnessaddress", &addwitnessaddress, true, {"address"} }, { "wallet", "backupwallet", &backupwallet, true, {"destination"} }, @@ -3032,7 +2967,7 @@ static const CRPCCommand commands[] = { "wallet", "listreceivedbyaddress", &listreceivedbyaddress, false, {"minconf","include_empty","include_watchonly"} }, { "wallet", "listsinceblock", &listsinceblock, false, {"blockhash","target_confirmations","include_watchonly"} }, { "wallet", "listtransactions", &listtransactions, false, {"account","count","skip","include_watchonly"} }, - { "wallet", "listunspent", &listunspent, false, {"minconf","maxconf","addresses","include_unsafe"} }, + { "wallet", "listunspent", &listunspent, false, {"minconf","maxconf","addresses","include_unsafe","query_options"} }, { "wallet", "lockunspent", &lockunspent, true, {"unlock","transactions"} }, { "wallet", "move", &movecmd, false, {"fromaccount","toaccount","amount","minconf","comment"} }, { "wallet", "sendfrom", &sendfrom, false, {"fromaccount","toaddress","amount","minconf","comment","comment_to"} }, diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 3a68ccf1b2..bd5dad18ca 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -6,7 +6,20 @@ #define BITCOIN_WALLET_RPCWALLET_H class CRPCTable; +class JSONRPCRequest; void RegisterWalletRPCCommands(CRPCTable &t); +/** + * Figures out what wallet, if any, to use for a JSONRPCRequest. + * + * @param[in] request JSONRPCRequest that wishes to access a wallet + * @return NULL if no wallet should be used, or a pointer to the CWallet + */ +CWallet *GetWalletForJSONRPCRequest(const JSONRPCRequest&); + +std::string HelpRequiringPassphrase(CWallet *); +void EnsureWalletIsUnlocked(CWallet *); +bool EnsureWalletIsAvailable(CWallet *, bool avoidException); + #endif //BITCOIN_WALLET_RPCWALLET_H diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index a76db37617..1989bf8d9b 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -14,7 +14,8 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName): bitdb.MakeMock(); bool fFirstRun; - pwalletMain = new CWallet("wallet_test.dat"); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); + pwalletMain = new CWallet(std::move(dbw)); pwalletMain->LoadWallet(fFirstRun); RegisterValidationInterface(pwalletMain); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 7ac2112dd2..b30748d66b 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -9,6 +9,7 @@ #include <utility> #include <vector> +#include "consensus/validation.h" #include "rpc/server.h" #include "test/test_bitcoin.h" #include "validation.h" @@ -19,6 +20,8 @@ #include <univalue.h> extern UniValue importmulti(const JSONRPCRequest& request); +extern UniValue dumpwallet(const JSONRPCRequest& request); +extern UniValue importwallet(const JSONRPCRequest& request); // how many times to run all the tests to have a chance to catch errors that only show up with particular random shuffles #define RUN_TESTS 100 @@ -27,16 +30,14 @@ extern UniValue importmulti(const JSONRPCRequest& request); // we repeat those tests this many times and only complain if all iterations of the test fail #define RANDOM_REPEATS 5 -using namespace std; - std::vector<std::unique_ptr<CWalletTx>> wtxn; -typedef set<pair<const CWalletTx*,unsigned int> > CoinSet; +typedef std::set<CInputCoin> CoinSet; BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) -static const CWallet wallet; -static vector<COutput> vCoins; +static const CWallet testWallet; +static std::vector<COutput> vCoins; static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0) { @@ -50,13 +51,13 @@ static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = fa // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() tx.vin.resize(1); } - std::unique_ptr<CWalletTx> wtx(new CWalletTx(&wallet, MakeTransactionRef(std::move(tx)))); + std::unique_ptr<CWalletTx> wtx(new CWalletTx(&testWallet, MakeTransactionRef(std::move(tx)))); if (fIsFromMe) { wtx->fDebitCached = true; wtx->nDebitCached = 1; } - COutput output(wtx.get(), nInput, nAge, true, true); + COutput output(wtx.get(), nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); vCoins.push_back(output); wtxn.emplace_back(std::move(wtx)); } @@ -69,7 +70,7 @@ static void empty_wallet(void) static bool equal_sets(CoinSet a, CoinSet b) { - pair<CoinSet::iterator, CoinSet::iterator> ret = mismatch(a.begin(), a.end(), b.begin()); + std::pair<CoinSet::iterator, CoinSet::iterator> ret = mismatch(a.begin(), a.end(), b.begin()); return ret.first == a.end() && ret.second == b.end(); } @@ -78,7 +79,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) CoinSet setCoinsRet, setCoinsRet2; CAmount nValueRet; - LOCK(wallet.cs_wallet); + LOCK(testWallet.cs_wallet); // test multiple times to allow for differences in the shuffle order for (int i = 0; i < RUN_TESTS; i++) @@ -86,24 +87,24 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) empty_wallet(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!wallet.SelectCoinsMinConf( 1 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); add_coin(1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!wallet.SelectCoinsMinConf( 1 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); // but we can find a new 1 cent - BOOST_CHECK( wallet.SelectCoinsMinConf( 1 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 1 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); add_coin(2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!wallet.SelectCoinsMinConf( 3 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 3 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); // we can make 3 cents of new coins - BOOST_CHECK( wallet.SelectCoinsMinConf( 3 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 3 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 3 * CENT); add_coin(5*CENT); // add a mature 5 cent coin, @@ -113,33 +114,33 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!wallet.SelectCoinsMinConf(38 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!wallet.SelectCoinsMinConf(38 * CENT, 6, 6, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, 6, 6, 0, vCoins, setCoinsRet, nValueRet)); // but we can make 37 cents if we accept new coins from ourself - BOOST_CHECK( wallet.SelectCoinsMinConf(37 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(37 * CENT, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 37 * CENT); // and we can make 38 cents if we accept all new coins - BOOST_CHECK( wallet.SelectCoinsMinConf(38 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(38 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - BOOST_CHECK( wallet.SelectCoinsMinConf(34 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(34 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - BOOST_CHECK( wallet.SelectCoinsMinConf( 7 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 7 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 7 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - BOOST_CHECK( wallet.SelectCoinsMinConf( 8 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 8 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK(nValueRet == 8 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - BOOST_CHECK( wallet.SelectCoinsMinConf( 9 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 9 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 10 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -153,30 +154,30 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) add_coin(30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - BOOST_CHECK( wallet.SelectCoinsMinConf(71 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); - BOOST_CHECK(!wallet.SelectCoinsMinConf(72 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(71 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(72 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - BOOST_CHECK( wallet.SelectCoinsMinConf(16 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); add_coin( 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - BOOST_CHECK( wallet.SelectCoinsMinConf(16 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); add_coin( 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - BOOST_CHECK( wallet.SelectCoinsMinConf(16 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - BOOST_CHECK( wallet.SelectCoinsMinConf(11 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(11 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 11 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -185,11 +186,11 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) add_coin( 2*COIN); add_coin( 3*COIN); add_coin( 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - BOOST_CHECK( wallet.SelectCoinsMinConf(95 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(95 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - BOOST_CHECK( wallet.SelectCoinsMinConf(195 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(195 * CENT, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -204,14 +205,14 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - BOOST_CHECK( wallet.SelectCoinsMinConf(MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // but if we add a bigger coin, small change is avoided add_coin(1111*MIN_CHANGE); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - BOOST_CHECK( wallet.SelectCoinsMinConf(1 * MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // if we add more small coins: @@ -219,7 +220,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) add_coin(MIN_CHANGE * 7 / 10); // and try again to make 1.0 * MIN_CHANGE - BOOST_CHECK( wallet.SelectCoinsMinConf(1 * MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // run the 'mtgox' test (see http://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) @@ -228,7 +229,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) for (int j = 0; j < 20; j++) add_coin(50000 * COIN); - BOOST_CHECK( wallet.SelectCoinsMinConf(500000 * COIN, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(500000 * COIN, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 10U); // in ten coins @@ -241,7 +242,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 7 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( wallet.SelectCoinsMinConf(1 * MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1111 * MIN_CHANGE); // we get the bigger coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -251,7 +252,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 8 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( wallet.SelectCoinsMinConf(MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // in two coins 0.4+0.6 @@ -262,12 +263,12 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) add_coin(MIN_CHANGE * 100); // trying to make 100.01 from these three coins - BOOST_CHECK(wallet.SelectCoinsMinConf(MIN_CHANGE * 10001 / 100, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 10001 / 100, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - BOOST_CHECK(wallet.SelectCoinsMinConf(MIN_CHANGE * 9990 / 100, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 9990 / 100, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 101 * MIN_CHANGE); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -277,7 +278,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) for (uint16_t j = 0; j < 676; j++) add_coin(amt); - BOOST_CHECK(wallet.SelectCoinsMinConf(2000, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(2000, 1, 1, 0, vCoins, setCoinsRet, nValueRet)); if (amt - 2000 < MIN_CHANGE) { // needs more than one input: uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); @@ -299,8 +300,8 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - BOOST_CHECK(wallet.SelectCoinsMinConf(50 * COIN, 1, 6, 0, vCoins, setCoinsRet , nValueRet)); - BOOST_CHECK(wallet.SelectCoinsMinConf(50 * COIN, 1, 6, 0, vCoins, setCoinsRet2, nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, 1, 6, 0, vCoins, setCoinsRet , nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, 1, 6, 0, vCoins, setCoinsRet2, nValueRet)); BOOST_CHECK(!equal_sets(setCoinsRet, setCoinsRet2)); int fails = 0; @@ -308,8 +309,8 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(wallet.SelectCoinsMinConf(COIN, 1, 6, 0, vCoins, setCoinsRet , nValueRet)); - BOOST_CHECK(wallet.SelectCoinsMinConf(COIN, 1, 6, 0, vCoins, setCoinsRet2, nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, 1, 6, 0, vCoins, setCoinsRet , nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, 1, 6, 0, vCoins, setCoinsRet2, nValueRet)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } @@ -329,8 +330,8 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(wallet.SelectCoinsMinConf(90*CENT, 1, 6, 0, vCoins, setCoinsRet , nValueRet)); - BOOST_CHECK(wallet.SelectCoinsMinConf(90*CENT, 1, 6, 0, vCoins, setCoinsRet2, nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, 1, 6, 0, vCoins, setCoinsRet , nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, 1, 6, 0, vCoins, setCoinsRet2, nValueRet)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } @@ -345,7 +346,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) CoinSet setCoinsRet; CAmount nValueRet; - LOCK(wallet.cs_wallet); + LOCK(testWallet.cs_wallet); empty_wallet(); @@ -354,7 +355,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) add_coin(1000 * COIN); add_coin(3 * COIN); - BOOST_CHECK(wallet.SelectCoinsMinConf(1003 * COIN, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(1003 * COIN, 1, 6, 0, vCoins, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1003 * COIN); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -366,6 +367,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) LOCK(cs_main); // Cap last block file size, and mine new block in a new block file. + CBlockIndex* const nullBlock = nullptr; CBlockIndex* oldTip = chainActive.Tip(); GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE; CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); @@ -377,7 +379,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CWallet wallet; LOCK(wallet.cs_wallet); wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 100 * COIN); } @@ -391,7 +393,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CWallet wallet; LOCK(wallet.cs_wallet); wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - BOOST_CHECK_EQUAL(newTip, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 50 * COIN); } @@ -415,7 +417,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CKey futureKey; futureKey.MakeNewKey(true); key.pushKV("scriptPubKey", HexStr(GetScriptForRawPubKey(futureKey.GetPubKey()))); - key.pushKV("timestamp", newTip->GetBlockTimeMax() + 7200); + key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1); key.pushKV("internal", UniValue(true)); keys.push_back(key); JSONRPCRequest request; @@ -423,9 +425,255 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) request.params.push_back(keys); UniValue response = importmulti(request); - BOOST_CHECK_EQUAL(response.write(), strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Failed to rescan before time %d, transactions may be missing.\"}},{\"success\":true}]", newTip->GetBlockTimeMax())); + BOOST_CHECK_EQUAL(response.write(), + strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Rescan failed for key with creation " + "timestamp %d. There was an error reading a block from time %d, which is after or within %d " + "seconds of key creation, and could contain transactions pertaining to the key. As a result, " + "transactions and coins using this key may not appear in the wallet. This error could be caused " + "by pruning or data corruption (see bitcoind log for details) and could be dealt with by " + "downloading and rescanning the relevant blocks (see -reindex and -rescan " + "options).\"}},{\"success\":true}]", + 0, oldTip->GetBlockTimeMax(), TIMESTAMP_WINDOW)); ::pwalletMain = backup; } } +// Verify importwallet RPC starts rescan at earliest block with timestamp +// greater or equal than key birthday. Previously there was a bug where +// importwallet RPC would start the scan at the latest block with timestamp less +// than or equal to key birthday. +BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) +{ + CWallet *pwalletMainBackup = ::pwalletMain; + LOCK(cs_main); + + // Create two blocks with same timestamp to verify that importwallet rescan + // will pick up both blocks, not just the first. + const int64_t BLOCK_TIME = chainActive.Tip()->GetBlockTimeMax() + 5; + SetMockTime(BLOCK_TIME); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + + // Set key birthday to block time increased by the timestamp window, so + // rescan will start at the block time. + const int64_t KEY_TIME = BLOCK_TIME + TIMESTAMP_WINDOW; + SetMockTime(KEY_TIME); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + + // Import key into wallet and call dumpwallet to create backup file. + { + CWallet wallet; + LOCK(wallet.cs_wallet); + wallet.mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; + wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + + JSONRPCRequest request; + request.params.setArray(); + request.params.push_back("wallet.backup"); + ::pwalletMain = &wallet; + ::dumpwallet(request); + } + + // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME + // were scanned, and no prior blocks were scanned. + { + CWallet wallet; + + JSONRPCRequest request; + request.params.setArray(); + request.params.push_back("wallet.backup"); + ::pwalletMain = &wallet; + ::importwallet(request); + + BOOST_CHECK_EQUAL(wallet.mapWallet.size(), 3); + BOOST_CHECK_EQUAL(coinbaseTxns.size(), 103); + for (size_t i = 0; i < coinbaseTxns.size(); ++i) { + bool found = wallet.GetWalletTx(coinbaseTxns[i].GetHash()); + bool expected = i >= 100; + BOOST_CHECK_EQUAL(found, expected); + } + } + + SetMockTime(0); + ::pwalletMain = pwalletMainBackup; +} + +// Check that GetImmatureCredit() returns a newly calculated value instead of +// the cached value after a MarkDirty() call. +// +// This is a regression test written to verify a bugfix for the immature credit +// function. Similar tests probably should be written for the other credit and +// debit functions. +BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) +{ + CWallet wallet; + CWalletTx wtx(&wallet, MakeTransactionRef(coinbaseTxns.back())); + LOCK2(cs_main, wallet.cs_wallet); + wtx.hashBlock = chainActive.Tip()->GetBlockHash(); + wtx.nIndex = 0; + + // Call GetImmatureCredit() once before adding the key to the wallet to + // cache the current immature credit amount, which is 0. + BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 0); + + // Invalidate the cached value, add the key, and make sure a new immature + // credit amount is calculated. + wtx.MarkDirty(); + wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 50*COIN); +} + +static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) +{ + CMutableTransaction tx; + tx.nLockTime = lockTime; + SetMockTime(mockTime); + CBlockIndex* block = nullptr; + if (blockTime > 0) { + auto inserted = mapBlockIndex.emplace(GetRandHash(), new CBlockIndex); + assert(inserted.second); + const uint256& hash = inserted.first->first; + block = inserted.first->second; + block->nTime = blockTime; + block->phashBlock = &hash; + } + + CWalletTx wtx(&wallet, MakeTransactionRef(tx)); + if (block) { + wtx.SetMerkleBranch(block, 0); + } + wallet.AddToWallet(wtx); + return wallet.mapWallet.at(wtx.GetHash()).nTimeSmart; +} + +// Simple test to verify assignment of CWalletTx::nSmartTime value. Could be +// expanded to cover more corner cases of smart time logic. +BOOST_AUTO_TEST_CASE(ComputeTimeSmart) +{ + CWallet wallet; + + // New transaction should use clock time if lower than block time. + BOOST_CHECK_EQUAL(AddTx(wallet, 1, 100, 120), 100); + + // Test that updating existing transaction does not change smart time. + BOOST_CHECK_EQUAL(AddTx(wallet, 1, 200, 220), 100); + + // New transaction should use clock time if there's no block time. + BOOST_CHECK_EQUAL(AddTx(wallet, 2, 300, 0), 300); + + // New transaction should use block time if lower than clock time. + BOOST_CHECK_EQUAL(AddTx(wallet, 3, 420, 400), 400); + + // New transaction should use latest entry time if higher than + // min(block time, clock time). + BOOST_CHECK_EQUAL(AddTx(wallet, 4, 500, 390), 400); + + // If there are future entries, new transaction should use time of the + // newest entry that is no more than 300 seconds ahead of the clock time. + BOOST_CHECK_EQUAL(AddTx(wallet, 5, 50, 600), 300); + + // Reset mock time for other tests. + SetMockTime(0); +} + +BOOST_AUTO_TEST_CASE(LoadReceiveRequests) +{ + CTxDestination dest = CKeyID(); + pwalletMain->AddDestData(dest, "misc", "val_misc"); + pwalletMain->AddDestData(dest, "rr0", "val_rr0"); + pwalletMain->AddDestData(dest, "rr1", "val_rr1"); + + auto values = pwalletMain->GetDestValues("rr"); + BOOST_CHECK_EQUAL(values.size(), 2); + BOOST_CHECK_EQUAL(values[0], "val_rr0"); + BOOST_CHECK_EQUAL(values[1], "val_rr1"); +} + +class ListCoinsTestingSetup : public TestChain100Setup +{ +public: + ListCoinsTestingSetup() + { + CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + ::bitdb.MakeMock(); + wallet.reset(new CWallet(std::unique_ptr<CWalletDBWrapper>(new CWalletDBWrapper(&bitdb, "wallet_test.dat")))); + bool firstRun; + wallet->LoadWallet(firstRun); + LOCK(wallet->cs_wallet); + wallet->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + wallet->ScanForWalletTransactions(chainActive.Genesis()); + } + + ~ListCoinsTestingSetup() + { + wallet.reset(); + ::bitdb.Flush(true); + ::bitdb.Reset(); + } + + CWalletTx& AddTx(CRecipient recipient) + { + CWalletTx wtx; + CReserveKey reservekey(wallet.get()); + CAmount fee; + int changePos = -1; + std::string error; + BOOST_CHECK(wallet->CreateTransaction({recipient}, wtx, reservekey, fee, changePos, error)); + CValidationState state; + BOOST_CHECK(wallet->CommitTransaction(wtx, reservekey, nullptr, state)); + auto it = wallet->mapWallet.find(wtx.GetHash()); + BOOST_CHECK(it != wallet->mapWallet.end()); + CreateAndProcessBlock({CMutableTransaction(*it->second.tx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + it->second.SetMerkleBranch(chainActive.Tip(), 1); + return it->second; + } + + std::unique_ptr<CWallet> wallet; +}; + +BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) +{ + std::string coinbaseAddress = coinbaseKey.GetPubKey().GetID().ToString(); + LOCK(wallet->cs_wallet); + + // Confirm ListCoins initially returns 1 coin grouped under coinbaseKey + // address. + auto list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 1); + + // Check initial balance from one mature coinbase transaction. + BOOST_CHECK_EQUAL(50 * COIN, wallet->GetAvailableBalance()); + + // Add a transaction creating a change address, and confirm ListCoins still + // returns the coin associated with the change address underneath the + // coinbaseKey pubkey, even though the change address has a different + // pubkey. + AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, false /* subtract fee */}); + list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 2); + + // Lock both coins. Confirm number of available coins drops to 0. + std::vector<COutput> available; + wallet->AvailableCoins(available); + BOOST_CHECK_EQUAL(available.size(), 2); + for (const auto& group : list) { + for (const auto& coin : group.second) { + wallet->LockCoin(COutPoint(coin.tx->GetHash(), coin.i)); + } + } + wallet->AvailableCoins(available); + BOOST_CHECK_EQUAL(available.size(), 0); + + // Confirm ListCoins still returns same result as before, despite coins + // being locked. + list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 2); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 63501b04be..3e000d2a9d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -11,15 +11,19 @@ #include "wallet/coincontrol.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "fs.h" #include "key.h" #include "keystore.h" #include "validation.h" #include "net.h" +#include "policy/fees.h" #include "policy/policy.h" +#include "policy/rbf.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "script/script.h" #include "script/sign.h" +#include "scheduler.h" #include "timedata.h" #include "txmempool.h" #include "util.h" @@ -29,17 +33,13 @@ #include <assert.h> #include <boost/algorithm/string/replace.hpp> -#include <boost/filesystem.hpp> #include <boost/thread.hpp> -using namespace std; - CWallet* pwalletMain = NULL; /** Transaction fee set by the user */ CFeeRate payTxFee(DEFAULT_TRANSACTION_FEE); unsigned int nTxConfirmTarget = DEFAULT_TX_CONFIRM_TARGET; bool bSpendZeroConfChange = DEFAULT_SPEND_ZEROCONF_CHANGE; -bool fSendFreeTransactions = DEFAULT_SEND_FREE_TRANSACTIONS; bool fWalletRbf = DEFAULT_WALLET_RBF; const char * DEFAULT_WALLET_DAT = "wallet.dat"; @@ -66,10 +66,10 @@ const uint256 CMerkleTx::ABANDON_HASH(uint256S("00000000000000000000000000000000 struct CompareValueOnly { - bool operator()(const pair<CAmount, pair<const CWalletTx*, unsigned int> >& t1, - const pair<CAmount, pair<const CWalletTx*, unsigned int> >& t2) const + bool operator()(const CInputCoin& t1, + const CInputCoin& t2) const { - return t1.first < t2.first; + return t1.txout.nValue < t2.txout.nValue; } }; @@ -87,7 +87,7 @@ const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const return &(it->second); } -CPubKey CWallet::GenerateNewKey() +CPubKey CWallet::GenerateNewKey(bool internal) { AssertLockHeld(cs_wallet); // mapKeyMetadata bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets @@ -100,7 +100,7 @@ CPubKey CWallet::GenerateNewKey() // use HD key derivation if HD was enabled during wallet creation if (IsHDEnabled()) { - DeriveNewChildKey(metadata, secret); + DeriveNewChildKey(metadata, secret, (CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); } else { secret.MakeNewKey(fCompressed); } @@ -120,13 +120,13 @@ CPubKey CWallet::GenerateNewKey() return pubkey; } -void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret) +void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool internal) { // for now we use a fixed keypath scheme of m/0'/0'/k CKey key; //master key seed (256bit) CExtKey masterKey; //hd master key CExtKey accountKey; //key at m/0' - CExtKey externalChainChildKey; //key at m/0'/0' + CExtKey chainChildKey; //key at m/0'/0' (external) or m/0'/1' (internal) CExtKey childKey; //key at m/0'/0'/<n>' // try to get the master key @@ -139,24 +139,30 @@ void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret) // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); - // derive m/0'/0' - accountKey.Derive(externalChainChildKey, BIP32_HARDENED_KEY_LIMIT); + // derive m/0'/0' (external chain) OR m/0'/1' (internal chain) + assert(internal ? CanSupportFeature(FEATURE_HD_SPLIT) : true); + accountKey.Derive(chainChildKey, BIP32_HARDENED_KEY_LIMIT+(internal ? 1 : 0)); // derive child key at next index, skip keys already known to the wallet do { // always derive hardened keys // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 - externalChainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; - metadata.hdMasterKeyID = hdChain.masterKeyID; - // increment childkey index - hdChain.nExternalChainCounter++; + if (internal) { + chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/1'/" + std::to_string(hdChain.nInternalChainCounter) + "'"; + hdChain.nInternalChainCounter++; + } + else { + chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; + hdChain.nExternalChainCounter++; + } } while (HaveKey(childKey.key.GetPubKey().GetID())); secret = childKey.key; - + metadata.hdMasterKeyID = hdChain.masterKeyID; // update the chain model in the database - if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) + if (!CWalletDB(*dbw).WriteHDChain(hdChain)) throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); } @@ -175,10 +181,8 @@ bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) if (HaveWatchOnly(script)) RemoveWatchOnly(script); - if (!fFileBacked) - return true; if (!IsCrypted()) { - return CWalletDB(strWalletFile).WriteKey(pubkey, + return CWalletDB(*dbw).WriteKey(pubkey, secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } @@ -186,12 +190,10 @@ bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) } bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, - const vector<unsigned char> &vchCryptedSecret) + const std::vector<unsigned char> &vchCryptedSecret) { if (!CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret)) return false; - if (!fFileBacked) - return true; { LOCK(cs_wallet); if (pwalletdbEncryption) @@ -199,7 +201,7 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); else - return CWalletDB(strWalletFile).WriteCryptedKey(vchPubKey, + return CWalletDB(*dbw).WriteCryptedKey(vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); } @@ -235,9 +237,7 @@ bool CWallet::AddCScript(const CScript& redeemScript) { if (!CCryptoKeyStore::AddCScript(redeemScript)) return false; - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteCScript(Hash160(redeemScript), redeemScript); + return CWalletDB(*dbw).WriteCScript(Hash160(redeemScript), redeemScript); } bool CWallet::LoadCScript(const CScript& redeemScript) @@ -263,9 +263,7 @@ bool CWallet::AddWatchOnly(const CScript& dest) const CKeyMetadata& meta = mapKeyMetadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteWatchOnly(dest, meta); + return CWalletDB(*dbw).WriteWatchOnly(dest, meta); } bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) @@ -281,9 +279,8 @@ bool CWallet::RemoveWatchOnly(const CScript &dest) return false; if (!HaveWatchOnly()) NotifyWatchonlyChanged(false); - if (fFileBacked) - if (!CWalletDB(strWalletFile).EraseWatchOnly(dest)) - return false; + if (!CWalletDB(*dbw).EraseWatchOnly(dest)) + return false; return true; } @@ -296,7 +293,7 @@ bool CWallet::LoadWatchOnly(const CScript &dest) bool CWallet::Unlock(const SecureString& strWalletPassphrase) { CCrypter crypter; - CKeyingMaterial vMasterKey; + CKeyingMaterial _vMasterKey; { LOCK(cs_wallet); @@ -304,9 +301,9 @@ bool CWallet::Unlock(const SecureString& strWalletPassphrase) { if(!crypter.SetKeyFromPassphrase(strWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod)) return false; - if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, vMasterKey)) + if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, _vMasterKey)) continue; // try another master key - if (CCryptoKeyStore::Unlock(vMasterKey)) + if (CCryptoKeyStore::Unlock(_vMasterKey)) return true; } } @@ -322,14 +319,14 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, Lock(); CCrypter crypter; - CKeyingMaterial vMasterKey; + CKeyingMaterial _vMasterKey; BOOST_FOREACH(MasterKeyMap::value_type& pMasterKey, mapMasterKeys) { if(!crypter.SetKeyFromPassphrase(strOldWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod)) return false; - if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, vMasterKey)) + if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, _vMasterKey)) return false; - if (CCryptoKeyStore::Unlock(vMasterKey)) + if (CCryptoKeyStore::Unlock(_vMasterKey)) { int64_t nStartTime = GetTimeMillis(); crypter.SetKeyFromPassphrase(strNewWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod); @@ -346,9 +343,9 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, if (!crypter.SetKeyFromPassphrase(strNewWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod)) return false; - if (!crypter.Encrypt(vMasterKey, pMasterKey.second.vchCryptedKey)) + if (!crypter.Encrypt(_vMasterKey, pMasterKey.second.vchCryptedKey)) return false; - CWalletDB(strWalletFile).WriteMasterKey(pMasterKey.first, pMasterKey.second); + CWalletDB(*dbw).WriteMasterKey(pMasterKey.first, pMasterKey.second); if (fWasLocked) Lock(); return true; @@ -361,7 +358,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, void CWallet::SetBestChain(const CBlockLocator& loc) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); walletdb.WriteBestBlock(loc); } @@ -380,9 +377,8 @@ bool CWallet::SetMinVersion(enum WalletFeature nVersion, CWalletDB* pwalletdbIn, if (nVersion > nWalletMaxVersion) nWalletMaxVersion = nVersion; - if (fFileBacked) { - CWalletDB* pwalletdb = pwalletdbIn ? pwalletdbIn : new CWalletDB(strWalletFile); + CWalletDB* pwalletdb = pwalletdbIn ? pwalletdbIn : new CWalletDB(*dbw); if (nWalletVersion > 40000) pwalletdb->WriteMinVersion(nWalletVersion); if (!pwalletdbIn) @@ -404,9 +400,9 @@ bool CWallet::SetMaxVersion(int nVersion) return true; } -set<uint256> CWallet::GetConflicts(const uint256& txid) const +std::set<uint256> CWallet::GetConflicts(const uint256& txid) const { - set<uint256> result; + std::set<uint256> result; AssertLockHeld(cs_wallet); std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(txid); @@ -436,7 +432,7 @@ bool CWallet::HasWalletSpend(const uint256& txid) const void CWallet::Flush(bool shutdown) { - bitdb.Flush(shutdown); + dbw->Flush(shutdown); } bool CWallet::Verify() @@ -444,61 +440,34 @@ bool CWallet::Verify() if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) return true; - LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); - std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - - LogPrintf("Using wallet %s\n", walletFile); uiInterface.InitMessage(_("Verifying wallet...")); + std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - // Wallet file must be a plain filename without a directory - if (walletFile != boost::filesystem::basename(walletFile) + boost::filesystem::extension(walletFile)) - return InitError(strprintf(_("Wallet %s resides outside data directory %s"), walletFile, GetDataDir().string())); + std::string strError; + if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError)) + return InitError(strError); - if (!bitdb.Open(GetDataDir())) - { - // try moving the database env out of the way - boost::filesystem::path pathDatabase = GetDataDir() / "database"; - boost::filesystem::path pathDatabaseBak = GetDataDir() / strprintf("database.%d.bak", GetTime()); - try { - boost::filesystem::rename(pathDatabase, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); - } catch (const boost::filesystem::filesystem_error&) { - // failure is ok (well, not really, but it's not worse than what we started with) - } - - // try again - if (!bitdb.Open(GetDataDir())) { - // if it still fails, it probably means we can't even create the database env - return InitError(strprintf(_("Error initializing wallet database environment %s!"), GetDataDir())); - } - } - if (GetBoolArg("-salvagewallet", false)) { // Recover readable keypairs: - if (!CWalletDB::Recover(bitdb, walletFile, true)) + CWallet dummyWallet; + if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, CWalletDB::RecoverKeysOnlyFilter)) return false; } - - if (boost::filesystem::exists(GetDataDir() / walletFile)) + + std::string strWarning; + bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), strWarning, strError); + if (!strWarning.empty()) + InitWarning(strWarning); + if (!dbV) { - CDBEnv::VerifyResult r = bitdb.Verify(walletFile, CWalletDB::Recover); - if (r == CDBEnv::RECOVER_OK) - { - InitWarning(strprintf(_("Warning: Wallet file corrupt, data salvaged!" - " Original %s saved as %s in %s; if" - " your balance or transactions are incorrect you should" - " restore from a backup."), - walletFile, "wallet.{timestamp}.bak", GetDataDir())); - } - if (r == CDBEnv::RECOVER_FAIL) - return InitError(strprintf(_("%s corrupt, salvage failed"), walletFile)); + InitError(strError); + return false; } - return true; } -void CWallet::SyncMetaData(pair<TxSpends::iterator, TxSpends::iterator> range) +void CWallet::SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator> range) { // We want all the wallet transactions in range to have the same metadata as // the oldest (smallest nOrderPos). @@ -542,7 +511,7 @@ void CWallet::SyncMetaData(pair<TxSpends::iterator, TxSpends::iterator> range) bool CWallet::IsSpent(const uint256& hash, unsigned int n) const { const COutPoint outpoint(hash, n); - pair<TxSpends::const_iterator, TxSpends::const_iterator> range; + std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range; range = mapTxSpends.equal_range(outpoint); for (TxSpends::const_iterator it = range.first; it != range.second; ++it) @@ -560,9 +529,9 @@ bool CWallet::IsSpent(const uint256& hash, unsigned int n) const void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid) { - mapTxSpends.insert(make_pair(outpoint, wtxid)); + mapTxSpends.insert(std::make_pair(outpoint, wtxid)); - pair<TxSpends::iterator, TxSpends::iterator> range; + std::pair<TxSpends::iterator, TxSpends::iterator> range; range = mapTxSpends.equal_range(outpoint); SyncMetaData(range); } @@ -584,10 +553,10 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) if (IsCrypted()) return false; - CKeyingMaterial vMasterKey; + CKeyingMaterial _vMasterKey; - vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); - GetStrongRandBytes(&vMasterKey[0], WALLET_CRYPTO_KEY_SIZE); + _vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); + GetStrongRandBytes(&_vMasterKey[0], WALLET_CRYPTO_KEY_SIZE); CMasterKey kMasterKey; @@ -610,30 +579,25 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) if (!crypter.SetKeyFromPassphrase(strWalletPassphrase, kMasterKey.vchSalt, kMasterKey.nDeriveIterations, kMasterKey.nDerivationMethod)) return false; - if (!crypter.Encrypt(vMasterKey, kMasterKey.vchCryptedKey)) + if (!crypter.Encrypt(_vMasterKey, kMasterKey.vchCryptedKey)) return false; { LOCK(cs_wallet); mapMasterKeys[++nMasterKeyMaxID] = kMasterKey; - if (fFileBacked) - { - assert(!pwalletdbEncryption); - pwalletdbEncryption = new CWalletDB(strWalletFile); - if (!pwalletdbEncryption->TxnBegin()) { - delete pwalletdbEncryption; - pwalletdbEncryption = NULL; - return false; - } - pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); + assert(!pwalletdbEncryption); + pwalletdbEncryption = new CWalletDB(*dbw); + if (!pwalletdbEncryption->TxnBegin()) { + delete pwalletdbEncryption; + pwalletdbEncryption = NULL; + return false; } + pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); - if (!EncryptKeys(vMasterKey)) + if (!EncryptKeys(_vMasterKey)) { - if (fFileBacked) { - pwalletdbEncryption->TxnAbort(); - delete pwalletdbEncryption; - } + pwalletdbEncryption->TxnAbort(); + delete pwalletdbEncryption; // We now probably have half of our keys encrypted in memory, and half not... // die and let the user reload the unencrypted wallet. assert(false); @@ -642,28 +606,24 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Encryption was introduced in version 0.4.0 SetMinVersion(FEATURE_WALLETCRYPT, pwalletdbEncryption, true); - if (fFileBacked) - { - if (!pwalletdbEncryption->TxnCommit()) { - delete pwalletdbEncryption; - // We now have keys encrypted in memory, but not on disk... - // die to avoid confusion and let the user reload the unencrypted wallet. - assert(false); - } - + if (!pwalletdbEncryption->TxnCommit()) { delete pwalletdbEncryption; - pwalletdbEncryption = NULL; + // We now have keys encrypted in memory, but not on disk... + // die to avoid confusion and let the user reload the unencrypted wallet. + assert(false); } + delete pwalletdbEncryption; + pwalletdbEncryption = NULL; + Lock(); Unlock(strWalletPassphrase); // if we are using HD, replace the HD master key (seed) with a new one if (IsHDEnabled()) { - CKey key; - CPubKey masterPubKey = GenerateNewHDMasterKey(); - if (!SetHDMasterKey(masterPubKey)) + if (!SetHDMasterKey(GenerateNewHDMasterKey())) { return false; + } } NewKeyPool(); @@ -671,7 +631,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Need to completely rewrite the wallet file; if we don't, bdb might keep // bits of the unencrypted private key in slack space in the database file. - CDB::Rewrite(strWalletFile); + dbw->Rewrite(); } NotifyStatusChanged(this); @@ -682,26 +642,26 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) DBErrors CWallet::ReorderTransactions() { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // Old wallets didn't have any defined order for transactions // Probably a bad idea to change the output of this // First: get all CWalletTx and CAccountingEntry into a sorted-by-time multimap. - typedef pair<CWalletTx*, CAccountingEntry*> TxPair; - typedef multimap<int64_t, TxPair > TxItems; + typedef std::pair<CWalletTx*, CAccountingEntry*> TxPair; + typedef std::multimap<int64_t, TxPair > TxItems; TxItems txByTime; - for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { CWalletTx* wtx = &((*it).second); - txByTime.insert(make_pair(wtx->nTimeReceived, TxPair(wtx, (CAccountingEntry*)0))); + txByTime.insert(std::make_pair(wtx->nTimeReceived, TxPair(wtx, (CAccountingEntry*)0))); } - list<CAccountingEntry> acentries; + std::list<CAccountingEntry> acentries; walletdb.ListAccountCreditDebit("", acentries); BOOST_FOREACH(CAccountingEntry& entry, acentries) { - txByTime.insert(make_pair(entry.nTime, TxPair((CWalletTx*)0, &entry))); + txByTime.insert(std::make_pair(entry.nTime, TxPair((CWalletTx*)0, &entry))); } nOrderPosNext = 0; @@ -763,14 +723,14 @@ int64_t CWallet::IncOrderPosNext(CWalletDB *pwalletdb) if (pwalletdb) { pwalletdb->WriteOrderPosNext(nOrderPosNext); } else { - CWalletDB(strWalletFile).WriteOrderPosNext(nOrderPosNext); + CWalletDB(*dbw).WriteOrderPosNext(nOrderPosNext); } return nRet; } bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmount, std::string strComment) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); if (!walletdb.TxnBegin()) return false; @@ -804,7 +764,7 @@ bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmoun bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); CAccount account; walletdb.ReadAccount(strAccount, account); @@ -815,7 +775,7 @@ bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bFo else { // Check if the current key has been used CScript scriptPubKey = GetScriptForDestination(account.vchPubKey.GetID()); - for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); + for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end() && account.vchPubKey.IsValid(); ++it) BOOST_FOREACH(const CTxOut& txout, (*it).second.tx->vout) @@ -828,7 +788,7 @@ bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bFo // Generate a new key if (bForceNew) { - if (!GetKeyFromPool(account.vchPubKey)) + if (!GetKeyFromPool(account.vchPubKey, false)) return false; SetAddressBook(account.vchPubKey.GetID(), strAccount, "receive"); @@ -865,7 +825,7 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) wtx.mapValue["replaced_by_txid"] = newHash.ToString(); - CWalletDB walletdb(strWalletFile, "r+"); + CWalletDB walletdb(*dbw, "r+"); bool success = true; if (!walletdb.WriteTx(wtx)) { @@ -882,12 +842,12 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile, "r+", fFlushOnClose); + CWalletDB walletdb(*dbw, "r+", fFlushOnClose); uint256 hash = wtxIn.GetHash(); // Inserts only if not already there, returns tx inserted or tx found - pair<map<uint256, CWalletTx>::iterator, bool> ret = mapWallet.insert(make_pair(hash, wtxIn)); + std::pair<std::map<uint256, CWalletTx>::iterator, bool> ret = mapWallet.insert(std::make_pair(hash, wtxIn)); CWalletTx& wtx = (*ret.first).second; wtx.BindWallet(this); bool fInsertedNew = ret.second; @@ -895,52 +855,8 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) { wtx.nTimeReceived = GetAdjustedTime(); wtx.nOrderPos = IncOrderPosNext(&walletdb); - wtxOrdered.insert(make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); - - wtx.nTimeSmart = wtx.nTimeReceived; - if (!wtxIn.hashUnset()) - { - if (mapBlockIndex.count(wtxIn.hashBlock)) - { - int64_t latestNow = wtx.nTimeReceived; - int64_t latestEntry = 0; - { - // Tolerate times up to the last timestamp in the wallet not more than 5 minutes into the future - int64_t latestTolerated = latestNow + 300; - const TxItems & txOrdered = wtxOrdered; - for (TxItems::const_reverse_iterator it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) - { - CWalletTx *const pwtx = (*it).second.first; - if (pwtx == &wtx) - continue; - CAccountingEntry *const pacentry = (*it).second.second; - int64_t nSmartTime; - if (pwtx) - { - nSmartTime = pwtx->nTimeSmart; - if (!nSmartTime) - nSmartTime = pwtx->nTimeReceived; - } - else - nSmartTime = pacentry->nTime; - if (nSmartTime <= latestTolerated) - { - latestEntry = nSmartTime; - if (nSmartTime > latestNow) - latestNow = nSmartTime; - break; - } - } - } - - int64_t blocktime = mapBlockIndex[wtxIn.hashBlock]->GetBlockTime(); - wtx.nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); - } - else - LogPrintf("AddToWallet(): found %s in block %s not in index\n", - wtxIn.GetHash().ToString(), - wtxIn.hashBlock.ToString()); - } + wtxOrdered.insert(std::make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); + wtx.nTimeSmart = ComputeTimeSmart(wtx); AddToSpends(hash); } @@ -1004,7 +920,7 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) mapWallet[hash] = wtxIn; CWalletTx& wtx = mapWallet[hash]; wtx.BindWallet(this); - wtxOrdered.insert(make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); + wtxOrdered.insert(std::make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); AddToSpends(hash); BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapWallet.count(txin.prevout.hash)) { @@ -1021,9 +937,9 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) /** * Add a transaction to the wallet, or update it. pIndex and posInBlock should * be set when the transaction was known to be included in a block. When - * posInBlock = SYNC_TRANSACTION_NOT_IN_BLOCK (-1) , then wallet state is not - * updated in AddToWallet, but notifications happen and cached balances are - * marked dirty. + * pIndex == NULL, then wallet state is not updated in AddToWallet, but + * notifications happen and cached balances are marked dirty. + * * If fUpdate is true, existing transactions will be updated. * TODO: One exception to this is that the abandoned state is cleared under the * assumption that any further notification of a transaction that was considered @@ -1031,12 +947,13 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) * Abandoned state should probably be more carefully tracked via different * posInBlock signals or by checking mempool presence when necessary. */ -bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate) { + const CTransaction& tx = *ptx; { AssertLockHeld(cs_wallet); - if (posInBlock != -1) { + if (pIndex != NULL) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(txin.prevout); while (range.first != range.second) { @@ -1053,10 +970,10 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex if (fExisted && !fUpdate) return false; if (fExisted || IsMine(tx) || IsFromMe(tx)) { - CWalletTx wtx(this, MakeTransactionRef(tx)); + CWalletTx wtx(this, ptx); // Get merkle branch if transaction was found in a block - if (posInBlock != -1) + if (pIndex != NULL) wtx.SetMerkleBranch(pIndex, posInBlock); return AddToWallet(wtx, false); @@ -1065,11 +982,18 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex return false; } +bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const +{ + LOCK2(cs_main, cs_wallet); + const CWalletTx* wtx = GetWalletTx(hashTx); + return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() <= 0 && !wtx->InMempool(); +} + bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK2(cs_main, cs_wallet); - CWalletDB walletdb(strWalletFile, "r+"); + CWalletDB walletdb(*dbw, "r+"); std::set<uint256> todo; std::set<uint256> done; @@ -1141,7 +1065,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) return; // Do not flush the wallet here for performance reasons - CWalletDB walletdb(strWalletFile, "r+", false); + CWalletDB walletdb(*dbw, "r+", false); std::set<uint256> todo; std::set<uint256> done; @@ -1181,11 +1105,10 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } } -void CWallet::SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock) -{ - LOCK2(cs_main, cs_wallet); +void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pindex, int posInBlock) { + const CTransaction& tx = *ptx; - if (!AddToWalletIfInvolvingMe(tx, pindex, posInBlock, true)) + if (!AddToWalletIfInvolvingMe(ptx, pindex, posInBlock, true)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1198,12 +1121,44 @@ void CWallet::SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, } } +void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { + LOCK2(cs_main, cs_wallet); + SyncTransaction(ptx); +} + +void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { + LOCK2(cs_main, cs_wallet); + // TODO: Temporarily ensure that mempool removals are notified before + // connected transactions. This shouldn't matter, but the abandoned + // state of transactions in our wallet is currently cleared when we + // receive another notification and there is a race condition where + // notification of a connected conflict might cause an outside process + // to abandon a transaction and then have it inadvertently cleared by + // the notification that the conflicted transaction was evicted. + + for (const CTransactionRef& ptx : vtxConflicted) { + SyncTransaction(ptx); + } + for (size_t i = 0; i < pblock->vtx.size(); i++) { + SyncTransaction(pblock->vtx[i], pindex, i); + } +} + +void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { + LOCK2(cs_main, cs_wallet); + + for (const CTransactionRef& ptx : pblock->vtx) { + SyncTransaction(ptx); + } +} + + isminetype CWallet::IsMine(const CTxIn &txin) const { { LOCK(cs_wallet); - map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); + std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); if (mi != mapWallet.end()) { const CWalletTx& prev = (*mi).second; @@ -1220,7 +1175,7 @@ CAmount CWallet::GetDebit(const CTxIn &txin, const isminefilter& filter) const { { LOCK(cs_wallet); - map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); + std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); if (mi != mapWallet.end()) { const CWalletTx& prev = (*mi).second; @@ -1376,14 +1331,11 @@ CPubKey CWallet::GenerateNewHDMasterKey() bool CWallet::SetHDMasterKey(const CPubKey& pubkey) { LOCK(cs_wallet); - - // ensure this wallet.dat can only be opened by clients supporting HD - SetMinVersion(FEATURE_HD); - // store the keyid (hash160) together with // the child index counter in the database // as a hdchain object CHDChain newHdChain; + newHdChain.nVersion = CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; newHdChain.masterKeyID = pubkey.GetID(); SetHDChain(newHdChain, false); @@ -1393,14 +1345,14 @@ bool CWallet::SetHDMasterKey(const CPubKey& pubkey) bool CWallet::SetHDChain(const CHDChain& chain, bool memonly) { LOCK(cs_wallet); - if (!memonly && !CWalletDB(strWalletFile).WriteHDChain(chain)) - throw runtime_error(std::string(__func__) + ": writing chain failed"); + if (!memonly && !CWalletDB(*dbw).WriteHDChain(chain)) + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); hdChain = chain; return true; } -bool CWallet::IsHDEnabled() +bool CWallet::IsHDEnabled() const { return !hdChain.masterKeyID.IsNull(); } @@ -1422,7 +1374,7 @@ int CWalletTx::GetRequestCount() const // Generated block if (!hashUnset()) { - map<uint256, int>::const_iterator mi = pwallet->mapRequestCount.find(hashBlock); + std::map<uint256, int>::const_iterator mi = pwallet->mapRequestCount.find(hashBlock); if (mi != pwallet->mapRequestCount.end()) nRequests = (*mi).second; } @@ -1430,7 +1382,7 @@ int CWalletTx::GetRequestCount() const else { // Did anyone request this transaction? - map<uint256, int>::const_iterator mi = pwallet->mapRequestCount.find(GetHash()); + std::map<uint256, int>::const_iterator mi = pwallet->mapRequestCount.find(GetHash()); if (mi != pwallet->mapRequestCount.end()) { nRequests = (*mi).second; @@ -1438,7 +1390,7 @@ int CWalletTx::GetRequestCount() const // How about the block it's in? if (nRequests == 0 && !hashUnset()) { - map<uint256, int>::const_iterator _mi = pwallet->mapRequestCount.find(hashBlock); + std::map<uint256, int>::const_iterator _mi = pwallet->mapRequestCount.find(hashBlock); if (_mi != pwallet->mapRequestCount.end()) nRequests = (*_mi).second; else @@ -1450,8 +1402,8 @@ int CWalletTx::GetRequestCount() const return nRequests; } -void CWalletTx::GetAmounts(list<COutputEntry>& listReceived, - list<COutputEntry>& listSent, CAmount& nFee, string& strSentAccount, const isminefilter& filter) const +void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, + std::list<COutputEntry>& listSent, CAmount& nFee, std::string& strSentAccount, const isminefilter& filter) const { nFee = 0; listReceived.clear(); @@ -1506,91 +1458,60 @@ void CWalletTx::GetAmounts(list<COutputEntry>& listReceived, } -void CWalletTx::GetAccountAmounts(const string& strAccount, CAmount& nReceived, - CAmount& nSent, CAmount& nFee, const isminefilter& filter) const -{ - nReceived = nSent = nFee = 0; - - CAmount allFee; - string strSentAccount; - list<COutputEntry> listReceived; - list<COutputEntry> listSent; - GetAmounts(listReceived, listSent, allFee, strSentAccount, filter); - - if (strAccount == strSentAccount) - { - BOOST_FOREACH(const COutputEntry& s, listSent) - nSent += s.amount; - nFee = allFee; - } - { - LOCK(pwallet->cs_wallet); - BOOST_FOREACH(const COutputEntry& r, listReceived) - { - if (pwallet->mapAddressBook.count(r.destination)) - { - map<CTxDestination, CAddressBookData>::const_iterator mi = pwallet->mapAddressBook.find(r.destination); - if (mi != pwallet->mapAddressBook.end() && (*mi).second.name == strAccount) - nReceived += r.amount; - } - else if (strAccount.empty()) - { - nReceived += r.amount; - } - } - } -} - /** * Scan the block chain (starting in pindexStart) for transactions * from or to us. If fUpdate is true, found transactions that already * exist in the wallet will be updated. * - * Returns pointer to the first block in the last contiguous range that was - * successfully scanned. - * + * Returns null if scan was successful. Otherwise, if a complete rescan was not + * possible (due to pruning or corruption), returns pointer to the most recent + * block that could not be scanned. */ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) { - CBlockIndex* ret = nullptr; int64_t nNow = GetTime(); const CChainParams& chainParams = Params(); CBlockIndex* pindex = pindexStart; + CBlockIndex* ret = nullptr; { LOCK2(cs_main, cs_wallet); + fAbortRescan = false; + fScanningWallet = true; // no need to read and scan block, if block was created before // our wallet birthday (as adjusted for block time variability) - while (pindex && nTimeFirstKey && (pindex->GetBlockTime() < (nTimeFirstKey - 7200))) + while (pindex && nTimeFirstKey && (pindex->GetBlockTime() < (nTimeFirstKey - TIMESTAMP_WINDOW))) pindex = chainActive.Next(pindex); ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex); double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()); - while (pindex) + while (pindex && !fAbortRescan) { if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); + if (GetTime() >= nNow + 60) { + nNow = GetTime(); + LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); + } CBlock block; if (ReadBlockFromDisk(block, pindex, Params().GetConsensus())) { for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - AddToWalletIfInvolvingMe(*block.vtx[posInBlock], pindex, posInBlock, fUpdate); - } - if (!ret) { - ret = pindex; + AddToWalletIfInvolvingMe(block.vtx[posInBlock], pindex, posInBlock, fUpdate); } } else { - ret = nullptr; + ret = pindex; } pindex = chainActive.Next(pindex); - if (GetTime() >= nNow + 60) { - nNow = GetTime(); - LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); - } + } + if (pindex && fAbortRescan) { + LogPrintf("Rescan aborted at block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); } ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI + + fScanningWallet = false; } return ret; } @@ -1650,9 +1571,9 @@ bool CWalletTx::RelayWalletTransaction(CConnman* connman) return false; } -set<uint256> CWalletTx::GetConflicts() const +std::set<uint256> CWalletTx::GetConflicts() const { - set<uint256> result; + std::set<uint256> result; if (pwallet != NULL) { uint256 myHash = GetHash(); @@ -1825,10 +1746,7 @@ CAmount CWalletTx::GetChange() const bool CWalletTx::InMempool() const { LOCK(mempool.cs); - if (mempool.exists(GetHash())) { - return true; - } - return false; + return mempool.exists(GetHash()); } bool CWalletTx::IsTrusted() const @@ -1866,8 +1784,8 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { CMutableTransaction tx1 = *this->tx; CMutableTransaction tx2 = *_tx.tx; - for (unsigned int i = 0; i < tx1.vin.size(); i++) tx1.vin[i].scriptSig = CScript(); - for (unsigned int i = 0; i < tx2.vin.size(); i++) tx2.vin[i].scriptSig = CScript(); + for (auto& txin : tx1.vin) txin.scriptSig = CScript(); + for (auto& txin : tx2.vin) txin.scriptSig = CScript(); return CTransaction(tx1) == CTransaction(tx2); } @@ -1877,14 +1795,14 @@ std::vector<uint256> CWallet::ResendWalletTransactionsBefore(int64_t nTime, CCon LOCK(cs_wallet); // Sort them in chronological order - multimap<unsigned int, CWalletTx*> mapSorted; + std::multimap<unsigned int, CWalletTx*> mapSorted; BOOST_FOREACH(PAIRTYPE(const uint256, CWalletTx)& item, mapWallet) { CWalletTx& wtx = item.second; // Don't rebroadcast if newer than nTime: if (wtx.nTimeReceived > nTime) continue; - mapSorted.insert(make_pair(wtx.nTimeReceived, &wtx)); + mapSorted.insert(std::make_pair(wtx.nTimeReceived, &wtx)); } BOOST_FOREACH(PAIRTYPE(const unsigned int, CWalletTx*)& item, mapSorted) { @@ -1934,7 +1852,7 @@ CAmount CWallet::GetBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const CWalletTx* pcoin = &(*it).second; if (pcoin->IsTrusted()) @@ -1950,7 +1868,7 @@ CAmount CWallet::GetUnconfirmedBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const CWalletTx* pcoin = &(*it).second; if (!pcoin->IsTrusted() && pcoin->GetDepthInMainChain() == 0 && pcoin->InMempool()) @@ -1965,7 +1883,7 @@ CAmount CWallet::GetImmatureBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const CWalletTx* pcoin = &(*it).second; nTotal += pcoin->GetImmatureCredit(); @@ -1979,7 +1897,7 @@ CAmount CWallet::GetWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const CWalletTx* pcoin = &(*it).second; if (pcoin->IsTrusted()) @@ -1995,7 +1913,7 @@ CAmount CWallet::GetUnconfirmedWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const CWalletTx* pcoin = &(*it).second; if (!pcoin->IsTrusted() && pcoin->GetDepthInMainChain() == 0 && pcoin->InMempool()) @@ -2010,7 +1928,7 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const CWalletTx* pcoin = &(*it).second; nTotal += pcoin->GetImmatureWatchOnlyCredit(); @@ -2019,13 +1937,74 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const return nTotal; } -void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const CCoinControl *coinControl, bool fIncludeZeroValue) const +// Calculate total balance in a different way from GetBalance. The biggest +// difference is that GetBalance sums up all unspent TxOuts paying to the +// wallet, while this sums up both spent and unspent TxOuts paying to the +// wallet, and then subtracts the values of TxIns spending from the wallet. This +// also has fewer restrictions on which unconfirmed transactions are considered +// trusted. +CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const +{ + LOCK2(cs_main, cs_wallet); + + CAmount balance = 0; + for (const auto& entry : mapWallet) { + const CWalletTx& wtx = entry.second; + const int depth = wtx.GetDepthInMainChain(); + if (depth < 0 || !CheckFinalTx(*wtx.tx) || wtx.GetBlocksToMaturity() > 0) { + continue; + } + + // Loop through tx outputs and add incoming payments. For outgoing txs, + // treat change outputs specially, as part of the amount debited. + CAmount debit = wtx.GetDebit(filter); + const bool outgoing = debit > 0; + for (const CTxOut& out : wtx.tx->vout) { + if (outgoing && IsChange(out)) { + debit -= out.nValue; + } else if (IsMine(out) & filter && depth >= minDepth && (!account || *account == GetAccountName(out.scriptPubKey))) { + balance += out.nValue; + } + } + + // For outgoing txs, subtract amount debited. + if (outgoing && (!account || *account == wtx.strFromAccount)) { + balance -= debit; + } + } + + if (account) { + balance += CWalletDB(*dbw).GetAccountCreditDebit(*account); + } + + return balance; +} + +CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const +{ + LOCK2(cs_main, cs_wallet); + + CAmount balance = 0; + std::vector<COutput> vCoins; + AvailableCoins(vCoins, true, coinControl); + for (const COutput& out : vCoins) { + if (out.fSpendable) { + balance += out.tx->tx->vout[out.i].nValue; + } + } + return balance; +} + +void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t &nMaximumCount, const int &nMinDepth, const int &nMaxDepth) const { vCoins.clear(); { LOCK2(cs_main, cs_wallet); - for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + + CAmount nTotal = 0; + + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const uint256& wtxid = it->first; const CWalletTx* pcoin = &(*it).second; @@ -2033,9 +2012,6 @@ void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const if (!CheckFinalTx(*pcoin)) continue; - if (fOnlyConfirmed && !pcoin->IsTrusted()) - continue; - if (pcoin->IsCoinBase() && pcoin->GetBlocksToMaturity() > 0) continue; @@ -2048,6 +2024,8 @@ void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const if (nDepth == 0 && !pcoin->InMempool()) continue; + bool safeTx = pcoin->IsTrusted(); + // We should not consider coins from transactions that are replacing // other transactions. // @@ -2063,8 +2041,8 @@ void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const // be a 1-block reorg away from the chain where transactions A and C // were accepted to another chain where B, B', and C were all // accepted. - if (nDepth == 0 && fOnlyConfirmed && pcoin->mapValue.count("replaces_txid")) { - continue; + if (nDepth == 0 && pcoin->mapValue.count("replaces_txid")) { + safeTx = false; } // Similarly, we should not consider coins from transactions that @@ -2075,28 +2053,126 @@ void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const // intending to replace A', but potentially resulting in a scenario // where A, A', and D could all be accepted (instead of just B and // D, or just A and A' like the user would want). - if (nDepth == 0 && fOnlyConfirmed && pcoin->mapValue.count("replaced_by_txid")) { + if (nDepth == 0 && pcoin->mapValue.count("replaced_by_txid")) { + safeTx = false; + } + + if (fOnlySafe && !safeTx) { continue; } + if (nDepth < nMinDepth || nDepth > nMaxDepth) + continue; + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { + if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) + continue; + + if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint((*it).first, i))) + continue; + + if (IsLockedCoin((*it).first, i)) + continue; + + if (IsSpent(wtxid, i)) + continue; + isminetype mine = IsMine(pcoin->tx->vout[i]); - if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && - !IsLockedCoin((*it).first, i) && (pcoin->tx->vout[i].nValue > 0 || fIncludeZeroValue) && - (!coinControl || !coinControl->HasSelected() || coinControl->fAllowOtherInputs || coinControl->IsSelected(COutPoint((*it).first, i)))) - vCoins.push_back(COutput(pcoin, i, nDepth, - ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || - (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO), - (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO)); + + if (mine == ISMINE_NO) { + continue; + } + + bool fSpendableIn = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO); + bool fSolvableIn = (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO; + + vCoins.push_back(COutput(pcoin, i, nDepth, fSpendableIn, fSolvableIn, safeTx)); + + // Checks the sum amount of all UTXO's. + if (nMinimumSumAmount != MAX_MONEY) { + nTotal += pcoin->tx->vout[i].nValue; + + if (nTotal >= nMinimumSumAmount) { + return; + } + } + + // Checks the maximum number of UTXO's. + if (nMaximumCount > 0 && vCoins.size() >= nMaximumCount) { + return; + } + } + } + } +} + +std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const +{ + // TODO: Add AssertLockHeld(cs_wallet) here. + // + // Because the return value from this function contains pointers to + // CWalletTx objects, callers to this function really should acquire the + // cs_wallet lock before calling it. However, the current caller doesn't + // acquire this lock yet. There was an attempt to add the missing lock in + // https://github.com/bitcoin/bitcoin/pull/10340, but that change has been + // postponed until after https://github.com/bitcoin/bitcoin/pull/10244 to + // avoid adding some extra complexity to the Qt code. + + std::map<CTxDestination, std::vector<COutput>> result; + + std::vector<COutput> availableCoins; + AvailableCoins(availableCoins); + + LOCK2(cs_main, cs_wallet); + for (auto& coin : availableCoins) { + CTxDestination address; + if (coin.fSpendable && + ExtractDestination(FindNonChangeParentOutput(*coin.tx->tx, coin.i).scriptPubKey, address)) { + result[address].emplace_back(std::move(coin)); + } + } + + std::vector<COutPoint> lockedCoins; + ListLockedCoins(lockedCoins); + for (const auto& output : lockedCoins) { + auto it = mapWallet.find(output.hash); + if (it != mapWallet.end()) { + int depth = it->second.GetDepthInMainChain(); + if (depth >= 0 && output.n < it->second.tx->vout.size() && + IsMine(it->second.tx->vout[output.n]) == ISMINE_SPENDABLE) { + CTxDestination address; + if (ExtractDestination(FindNonChangeParentOutput(*it->second.tx, output.n).scriptPubKey, address)) { + result[address].emplace_back( + &it->second, output.n, depth, true /* spendable */, true /* solvable */, false /* safe */); + } } } } + + return result; +} + +const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int output) const +{ + const CTransaction* ptx = &tx; + int n = output; + while (IsChange(ptx->vout[n]) && ptx->vin.size() > 0) { + const COutPoint& prevout = ptx->vin[0].prevout; + auto it = mapWallet.find(prevout.hash); + if (it == mapWallet.end() || it->second.tx->vout.size() <= prevout.n || + !IsMine(it->second.tx->vout[prevout.n])) { + break; + } + ptx = it->second.tx.get(); + n = prevout.n; + } + return ptx->vout[n]; } -static void ApproximateBestSubset(vector<pair<CAmount, pair<const CWalletTx*,unsigned int> > >vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, - vector<char>& vfBest, CAmount& nBest, int iterations = 1000) +static void ApproximateBestSubset(const std::vector<CInputCoin>& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, + std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { - vector<char> vfIncluded; + std::vector<char> vfIncluded; vfBest.assign(vValue.size(), true); nBest = nTotalLower; @@ -2118,9 +2194,9 @@ static void ApproximateBestSubset(vector<pair<CAmount, pair<const CWalletTx*,uns //that the rng is fast. We do not use a constant random sequence, //because there may be some privacy improvement by making //the selection random. - if (nPass == 0 ? insecure_rand.rand32()&1 : !vfIncluded[i]) + if (nPass == 0 ? insecure_rand.randbool() : !vfIncluded[i]) { - nTotal += vValue[i].first; + nTotal += vValue[i].txout.nValue; vfIncluded[i] = true; if (nTotal >= nTargetValue) { @@ -2130,7 +2206,7 @@ static void ApproximateBestSubset(vector<pair<CAmount, pair<const CWalletTx*,uns nBest = nTotal; vfBest = vfIncluded; } - nTotal -= vValue[i].first; + nTotal -= vValue[i].txout.nValue; vfIncluded[i] = false; } } @@ -2139,17 +2215,15 @@ static void ApproximateBestSubset(vector<pair<CAmount, pair<const CWalletTx*,uns } } -bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMine, const int nConfTheirs, const uint64_t nMaxAncestors, vector<COutput> vCoins, - set<pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const +bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMine, const int nConfTheirs, const uint64_t nMaxAncestors, std::vector<COutput> vCoins, + std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet) const { setCoinsRet.clear(); nValueRet = 0; // List of values less than target - pair<CAmount, pair<const CWalletTx*,unsigned int> > coinLowestLarger; - coinLowestLarger.first = std::numeric_limits<CAmount>::max(); - coinLowestLarger.second.first = NULL; - vector<pair<CAmount, pair<const CWalletTx*,unsigned int> > > vValue; + boost::optional<CInputCoin> coinLowestLarger; + std::vector<CInputCoin> vValue; CAmount nTotalLower = 0; random_shuffle(vCoins.begin(), vCoins.end(), GetRandInt); @@ -2168,22 +2242,21 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin continue; int i = output.i; - CAmount n = pcoin->tx->vout[i].nValue; - pair<CAmount,pair<const CWalletTx*,unsigned int> > coin = make_pair(n,make_pair(pcoin, i)); + CInputCoin coin = CInputCoin(pcoin, i); - if (n == nTargetValue) + if (coin.txout.nValue == nTargetValue) { - setCoinsRet.insert(coin.second); - nValueRet += coin.first; + setCoinsRet.insert(coin); + nValueRet += coin.txout.nValue; return true; } - else if (n < nTargetValue + MIN_CHANGE) + else if (coin.txout.nValue < nTargetValue + MIN_CHANGE) { vValue.push_back(coin); - nTotalLower += n; + nTotalLower += coin.txout.nValue; } - else if (n < coinLowestLarger.first) + else if (!coinLowestLarger || coin.txout.nValue < coinLowestLarger->txout.nValue) { coinLowestLarger = coin; } @@ -2191,27 +2264,27 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin if (nTotalLower == nTargetValue) { - for (unsigned int i = 0; i < vValue.size(); ++i) + for (const auto& input : vValue) { - setCoinsRet.insert(vValue[i].second); - nValueRet += vValue[i].first; + setCoinsRet.insert(input); + nValueRet += input.txout.nValue; } return true; } if (nTotalLower < nTargetValue) { - if (coinLowestLarger.second.first == NULL) + if (!coinLowestLarger) return false; - setCoinsRet.insert(coinLowestLarger.second); - nValueRet += coinLowestLarger.first; + setCoinsRet.insert(coinLowestLarger.get()); + nValueRet += coinLowestLarger->txout.nValue; return true; } // Solve subset sum by stochastic approximation std::sort(vValue.begin(), vValue.end(), CompareValueOnly()); std::reverse(vValue.begin(), vValue.end()); - vector<char> vfBest; + std::vector<char> vfBest; CAmount nBest; ApproximateBestSubset(vValue, nTotalLower, nTargetValue, vfBest, nBest); @@ -2220,33 +2293,37 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin // If we have a bigger coin and (either the stochastic approximation didn't find a good solution, // or the next bigger coin is closer), return the bigger coin - if (coinLowestLarger.second.first && - ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || coinLowestLarger.first <= nBest)) + if (coinLowestLarger && + ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || coinLowestLarger->txout.nValue <= nBest)) { - setCoinsRet.insert(coinLowestLarger.second); - nValueRet += coinLowestLarger.first; + setCoinsRet.insert(coinLowestLarger.get()); + nValueRet += coinLowestLarger->txout.nValue; } else { for (unsigned int i = 0; i < vValue.size(); i++) if (vfBest[i]) { - setCoinsRet.insert(vValue[i].second); - nValueRet += vValue[i].first; + setCoinsRet.insert(vValue[i]); + nValueRet += vValue[i].txout.nValue; } - LogPrint("selectcoins", "SelectCoins() best subset: "); - for (unsigned int i = 0; i < vValue.size(); i++) - if (vfBest[i]) - LogPrint("selectcoins", "%s ", FormatMoney(vValue[i].first)); - LogPrint("selectcoins", "total %s\n", FormatMoney(nBest)); + if (LogAcceptCategory(BCLog::SELECTCOINS)) { + LogPrint(BCLog::SELECTCOINS, "SelectCoins() best subset: "); + for (unsigned int i = 0; i < vValue.size(); i++) { + if (vfBest[i]) { + LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(vValue[i].txout.nValue)); + } + } + LogPrint(BCLog::SELECTCOINS, "total %s\n", FormatMoney(nBest)); + } } return true; } -bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, set<pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const +bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const { - vector<COutput> vCoins(vAvailableCoins); + std::vector<COutput> vCoins(vAvailableCoins); // coin control -> return all selected outputs (we want all selected to go into the transaction for sure) if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs) @@ -2256,13 +2333,13 @@ bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& if (!out.fSpendable) continue; nValueRet += out.tx->tx->vout[out.i].nValue; - setCoinsRet.insert(make_pair(out.tx, out.i)); + setCoinsRet.insert(CInputCoin(out.tx, out.i)); } return (nValueRet >= nTargetValue); } // calculate value from preset inputs and store them - set<pair<const CWalletTx*, uint32_t> > setPresetCoins; + std::set<CInputCoin> setPresetCoins; CAmount nValueFromPresetInputs = 0; std::vector<COutPoint> vPresetInputs; @@ -2270,7 +2347,7 @@ bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& coinControl->ListSelected(vPresetInputs); BOOST_FOREACH(const COutPoint& outpoint, vPresetInputs) { - map<uint256, CWalletTx>::const_iterator it = mapWallet.find(outpoint.hash); + std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(outpoint.hash); if (it != mapWallet.end()) { const CWalletTx* pcoin = &it->second; @@ -2278,15 +2355,15 @@ bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& if (pcoin->tx->vout.size() <= outpoint.n) return false; nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; - setPresetCoins.insert(make_pair(pcoin, outpoint.n)); + setPresetCoins.insert(CInputCoin(pcoin, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs } // remove preset inputs from vCoins - for (vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coinControl && coinControl->HasSelected();) + for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coinControl && coinControl->HasSelected();) { - if (setPresetCoins.count(make_pair(it->tx, it->i))) + if (setPresetCoins.count(CInputCoin(it->tx, it->i))) it = vCoins.erase(it); else ++it; @@ -2313,9 +2390,33 @@ bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& return res; } +bool CWallet::SignTransaction(CMutableTransaction &tx) +{ + AssertLockHeld(cs_wallet); // mapWallet + + // sign the new tx + CTransaction txNewConst(tx); + int nIn = 0; + for (const auto& input : tx.vin) { + std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(input.prevout.hash); + if(mi == mapWallet.end() || input.prevout.n >= mi->second.tx->vout.size()) { + return false; + } + const CScript& scriptPubKey = mi->second.tx->vout[input.prevout.n].scriptPubKey; + const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; + SignatureData sigdata; + if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { + return false; + } + UpdateTransaction(tx, nIn, sigdata); + nIn++; + } + return true; +} + bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool overrideEstimatedFeeRate, const CFeeRate& specificFeeRate, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, bool keepReserveKey, const CTxDestination& destChange) { - vector<CRecipient> vecSend; + std::vector<CRecipient> vecSend; // Turn the txout set into a CRecipient vector for (size_t idx = 0; idx < tx.vout.size(); idx++) @@ -2369,7 +2470,7 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool ov return true; } -bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wtxNew, CReserveKey& reservekey, CAmount& nFeeRet, +bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletTx& wtxNew, CReserveKey& reservekey, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, const CCoinControl* coinControl, bool sign) { CAmount nValue = 0; @@ -2430,7 +2531,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt assert(txNew.nLockTime < LOCKTIME_THRESHOLD); { - set<pair<const CWalletTx*,unsigned int> > setCoins; + std::set<CInputCoin> setCoins; LOCK2(cs_main, cs_wallet); { std::vector<COutput> vAvailableCoins; @@ -2449,7 +2550,6 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt CAmount nValueToSelect = nValue; if (nSubtractFeeFromAmount == 0) nValueToSelect += nFeeRet; - double dPriority = 0; // vouts to the payees for (const auto& recipient : vecSend) { @@ -2466,7 +2566,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt } } - if (txout.IsDust(dustRelayFee)) + if (IsDust(txout, ::dustRelayFee)) { if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { @@ -2490,19 +2590,6 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt strFailReason = _("Insufficient funds"); return false; } - for (const auto& pcoin : setCoins) - { - CAmount nCredit = pcoin.first->tx->vout[pcoin.second].nValue; - //The coin age after the next block (depth+1) is used instead of the current, - //reflecting an assumption the user would accept a bit more delay for - //a chance at a free transaction. - //But mempool inputs might still be in the mempool, so their age stays 0 - int age = pcoin.first->GetDepthInMainChain(); - assert(age >= 0); - if (age != 0) - age += 1; - dPriority += (double)nCredit * age; - } const CAmount nChange = nValueIn - nValueToSelect; if (nChange > 0) @@ -2529,7 +2616,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt // Reserve a new key pair from key pool CPubKey vchPubKey; bool ret; - ret = reservekey.GetReservedKey(vchPubKey); + ret = reservekey.GetReservedKey(vchPubKey, true); if (!ret) { strFailReason = _("Keypool ran out, please call keypoolrefill first"); @@ -2544,16 +2631,16 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt // We do not move dust-change to fees, because the sender would end up paying more than requested. // This would be against the purpose of the all-inclusive feature. // So instead we raise the change and deduct from the recipient. - if (nSubtractFeeFromAmount > 0 && newTxOut.IsDust(dustRelayFee)) + if (nSubtractFeeFromAmount > 0 && IsDust(newTxOut, ::dustRelayFee)) { - CAmount nDust = newTxOut.GetDustThreshold(dustRelayFee) - newTxOut.nValue; + CAmount nDust = GetDustThreshold(newTxOut, ::dustRelayFee) - newTxOut.nValue; newTxOut.nValue += nDust; // raise change until no more dust for (unsigned int i = 0; i < vecSend.size(); i++) // subtract from first recipient { if (vecSend[i].fSubtractFeeFromAmount) { txNew.vout[i].nValue -= nDust; - if (txNew.vout[i].IsDust(dustRelayFee)) + if (IsDust(txNew.vout[i], ::dustRelayFee)) { strFailReason = _("The transaction amount is too small to send after the fee has been deducted"); return false; @@ -2565,7 +2652,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt // Never create dust outputs; if we would, just // add the dust to the fee. - if (newTxOut.IsDust(dustRelayFee)) + if (IsDust(newTxOut, ::dustRelayFee)) { nChangePosInOut = -1; nFeeRet += nChange; @@ -2584,12 +2671,13 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt return false; } - vector<CTxOut>::iterator position = txNew.vout.begin()+nChangePosInOut; + std::vector<CTxOut>::iterator position = txNew.vout.begin()+nChangePosInOut; txNew.vout.insert(position, newTxOut); } - } - else + } else { reservekey.ReturnKey(); + nChangePosInOut = -1; + } // Fill vin // @@ -2601,9 +2689,10 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt // to avoid conflicting with other possible uses of nSequence, // and in the spirit of "smallest possible change from prior // behavior." + bool rbf = coinControl ? coinControl->signalRbf : fWalletRbf; for (const auto& coin : setCoins) - txNew.vin.push_back(CTxIn(coin.first->GetHash(),coin.second,CScript(), - std::numeric_limits<unsigned int>::max() - (fWalletRbf ? 2 : 1))); + txNew.vin.push_back(CTxIn(coin.outpoint,CScript(), + std::numeric_limits<unsigned int>::max() - (rbf ? 2 : 1))); // Fill in dummy signatures for fee calculation. if (!DummySignTx(txNew, setCoins)) { @@ -2614,7 +2703,6 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt unsigned int nBytes = GetVirtualTransactionSize(txNew); CTransaction txNewConst(txNew); - dPriority = txNewConst.ComputePriority(dPriority, nBytes); // Remove scriptSigs to eliminate the fee calculation dummy signatures for (auto& vin : txNew.vin) { @@ -2627,20 +2715,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt if (coinControl && coinControl->nConfirmTarget > 0) currentConfirmationTarget = coinControl->nConfirmTarget; - // Can we complete this as a free transaction? - if (fSendFreeTransactions && nBytes <= MAX_FREE_TRANSACTION_CREATE_SIZE) - { - // Not enough fee: enough priority? - double dPriorityNeeded = mempool.estimateSmartPriority(currentConfirmationTarget); - // Require at least hard-coded AllowFree. - if (dPriority >= dPriorityNeeded && AllowFree(dPriority)) - break; - } - - CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, mempool); - if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { - nFeeNeeded = coinControl->nMinimumTotalFee; - } + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator); if (coinControl && coinControl->fOverrideFeeRate) nFeeNeeded = coinControl->nFeeRate.GetFee(nBytes); @@ -2665,7 +2740,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt // to be addressed so we avoid creating too small an output. if (nFeeRet > nFeeNeeded && nChangePosInOut != -1 && nSubtractFeeFromAmount == 0) { CAmount extraFeePaid = nFeeRet - nFeeNeeded; - vector<CTxOut>::iterator change_position = txNew.vout.begin()+nChangePosInOut; + std::vector<CTxOut>::iterator change_position = txNew.vout.begin()+nChangePosInOut; change_position->nValue += extraFeePaid; nFeeRet -= extraFeePaid; } @@ -2675,7 +2750,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt // Try to reduce change to include necessary fee if (nChangePosInOut != -1 && nSubtractFeeFromAmount == 0) { CAmount additionalFeeNeeded = nFeeNeeded - nFeeRet; - vector<CTxOut>::iterator change_position = txNew.vout.begin()+nChangePosInOut; + std::vector<CTxOut>::iterator change_position = txNew.vout.begin()+nChangePosInOut; // Only reduce change if remaining amount is still a large enough output. if (change_position->nValue >= MIN_FINAL_CHANGE + additionalFeeNeeded) { change_position->nValue -= additionalFeeNeeded; @@ -2696,10 +2771,10 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt int nIn = 0; for (const auto& coin : setCoins) { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; - if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.first->tx->vout[coin.second].nValue, SIGHASH_ALL), scriptPubKey, sigdata)) + if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) { strFailReason = _("Signing transaction failed"); return false; @@ -2725,7 +2800,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt if (GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits LockPoints lp; - CTxMemPoolEntry entry(wtxNew.tx, 0, 0, 0, 0, 0, false, 0, lp); + CTxMemPoolEntry entry(wtxNew.tx, 0, 0, 0, false, 0, lp); CTxMemPool::setEntries setAncestors; size_t nLimitAncestors = GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); size_t nLimitAncestorSize = GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000; @@ -2783,13 +2858,13 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon } void CWallet::ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); return walletdb.ListAccountCreditDebit(strAccount, entries); } bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); return AddAccountingEntry(acentry, &walletdb); } @@ -2801,7 +2876,7 @@ bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry, CWalletDB *pwa laccentries.push_back(acentry); CAccountingEntry & entry = laccentries.back(); - wtxOrdered.insert(make_pair(entry.nOrderPos, TxPair((CWalletTx*)0, &entry))); + wtxOrdered.insert(std::make_pair(entry.nOrderPos, TxPair((CWalletTx*)0, &entry))); return true; } @@ -2811,19 +2886,14 @@ CAmount CWallet::GetRequiredFee(unsigned int nTxBytes) return std::max(minTxFee.GetFee(nTxBytes), ::minRelayTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreGlobalPayTxFee) { // payTxFee is the user-set global for desired feerate - return GetMinimumFee(nTxBytes, nConfirmTarget, pool, payTxFee.GetFee(nTxBytes)); -} - -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee) -{ - CAmount nFeeNeeded = targetFee; + CAmount nFeeNeeded = payTxFee.GetFee(nTxBytes); // User didn't set: use -txconfirmtarget to estimate... - if (nFeeNeeded == 0) { + if (nFeeNeeded == 0 || ignoreGlobalPayTxFee) { int estimateFoundTarget = nConfirmTarget; - nFeeNeeded = pool.estimateSmartFee(nConfirmTarget, &estimateFoundTarget).GetFee(nTxBytes); + nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, &estimateFoundTarget, pool).GetFee(nTxBytes); // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee if (nFeeNeeded == 0) nFeeNeeded = fallbackFee.GetFee(nTxBytes); @@ -2841,13 +2911,11 @@ CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarge DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { - if (!fFileBacked) - return DB_LOAD_OK; fFirstRunRet = false; - DBErrors nLoadWalletRet = CWalletDB(strWalletFile,"cr+").LoadWallet(this); + DBErrors nLoadWalletRet = CWalletDB(*dbw,"cr+").LoadWallet(this); if (nLoadWalletRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { LOCK(cs_wallet); setKeyPool.clear(); @@ -2866,16 +2934,18 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) return DB_LOAD_OK; } -DBErrors CWallet::ZapSelectTx(vector<uint256>& vHashIn, vector<uint256>& vHashOut) +DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) { - if (!fFileBacked) - return DB_LOAD_OK; - DBErrors nZapSelectTxRet = CWalletDB(strWalletFile,"cr+").ZapSelectTx(this, vHashIn, vHashOut); + AssertLockHeld(cs_wallet); // mapWallet + vchDefaultKey = CPubKey(); + DBErrors nZapSelectTxRet = CWalletDB(*dbw,"cr+").ZapSelectTx(vHashIn, vHashOut); + for (uint256 hash : vHashOut) + mapWallet.erase(hash); + if (nZapSelectTxRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { - LOCK(cs_wallet); setKeyPool.clear(); // Note: can't top-up keypool here, because wallet is locked. // User will be prompted to unlock wallet the next operation @@ -2894,12 +2964,11 @@ DBErrors CWallet::ZapSelectTx(vector<uint256>& vHashIn, vector<uint256>& vHashOu DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { - if (!fFileBacked) - return DB_LOAD_OK; - DBErrors nZapWalletTxRet = CWalletDB(strWalletFile,"cr+").ZapWalletTx(this, vWtx); + vchDefaultKey = CPubKey(); + DBErrors nZapWalletTxRet = CWalletDB(*dbw,"cr+").ZapWalletTx(vWtx); if (nZapWalletTxRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { LOCK(cs_wallet); setKeyPool.clear(); @@ -2916,7 +2985,7 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) } -bool CWallet::SetAddressBook(const CTxDestination& address, const string& strName, const string& strPurpose) +bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& strPurpose) { bool fUpdated = false; { @@ -2929,11 +2998,9 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const string& strNam } NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); - if (!fFileBacked) - return false; - if (!strPurpose.empty() && !CWalletDB(strWalletFile).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) + if (!strPurpose.empty() && !CWalletDB(*dbw).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) return false; - return CWalletDB(strWalletFile).WriteName(CBitcoinAddress(address).ToString(), strName); + return CWalletDB(*dbw).WriteName(CBitcoinAddress(address).ToString(), strName); } bool CWallet::DelAddressBook(const CTxDestination& address) @@ -2941,65 +3008,88 @@ bool CWallet::DelAddressBook(const CTxDestination& address) { LOCK(cs_wallet); // mapAddressBook - if(fFileBacked) + // Delete destdata tuples associated with address + std::string strAddress = CBitcoinAddress(address).ToString(); + BOOST_FOREACH(const PAIRTYPE(std::string, std::string) &item, mapAddressBook[address].destdata) { - // Delete destdata tuples associated with address - std::string strAddress = CBitcoinAddress(address).ToString(); - BOOST_FOREACH(const PAIRTYPE(string, string) &item, mapAddressBook[address].destdata) - { - CWalletDB(strWalletFile).EraseDestData(strAddress, item.first); - } + CWalletDB(*dbw).EraseDestData(strAddress, item.first); } mapAddressBook.erase(address); } NotifyAddressBookChanged(this, address, "", ::IsMine(*this, address) != ISMINE_NO, "", CT_DELETED); - if (!fFileBacked) - return false; - CWalletDB(strWalletFile).ErasePurpose(CBitcoinAddress(address).ToString()); - return CWalletDB(strWalletFile).EraseName(CBitcoinAddress(address).ToString()); + CWalletDB(*dbw).ErasePurpose(CBitcoinAddress(address).ToString()); + return CWalletDB(*dbw).EraseName(CBitcoinAddress(address).ToString()); } -bool CWallet::SetDefaultKey(const CPubKey &vchPubKey) +const std::string& CWallet::GetAccountName(const CScript& scriptPubKey) const { - if (fFileBacked) - { - if (!CWalletDB(strWalletFile).WriteDefaultKey(vchPubKey)) - return false; + CTxDestination address; + if (ExtractDestination(scriptPubKey, address) && !scriptPubKey.IsUnspendable()) { + auto mi = mapAddressBook.find(address); + if (mi != mapAddressBook.end()) { + return mi->second.name; + } } + // A scriptPubKey that doesn't have an entry in the address book is + // associated with the default account (""). + const static std::string DEFAULT_ACCOUNT_NAME; + return DEFAULT_ACCOUNT_NAME; +} + +bool CWallet::SetDefaultKey(const CPubKey &vchPubKey) +{ + if (!CWalletDB(*dbw).WriteDefaultKey(vchPubKey)) + return false; vchDefaultKey = vchPubKey; return true; } /** * Mark old keypool keys as used, - * and generate all new keys + * and generate all new keys */ bool CWallet::NewKeyPool() { { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); BOOST_FOREACH(int64_t nIndex, setKeyPool) walletdb.ErasePool(nIndex); setKeyPool.clear(); - if (IsLocked()) + if (!TopUpKeyPool()) { return false; - - int64_t nKeys = max(GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t)0); - for (int i = 0; i < nKeys; i++) - { - int64_t nIndex = i+1; - walletdb.WritePool(nIndex, CKeyPool(GenerateNewKey())); - setKeyPool.insert(nIndex); } - LogPrintf("CWallet::NewKeyPool wrote %d new keys\n", nKeys); + LogPrintf("CWallet::NewKeyPool rewrote keypool\n"); } return true; } +size_t CWallet::KeypoolCountExternalKeys() +{ + AssertLockHeld(cs_wallet); // setKeyPool + + // immediately return setKeyPool's size if HD or HD_SPLIT is disabled or not supported + if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) + return setKeyPool.size(); + + CWalletDB walletdb(*dbw); + + // count amount of external keys + size_t amountE = 0; + for(const int64_t& id : setKeyPool) + { + CKeyPool tmpKeypool; + if (!walletdb.ReadPool(id, tmpKeypool)) + throw std::runtime_error(std::string(__func__) + ": read failed"); + amountE += !tmpKeypool.fInternal; + } + + return amountE; +} + bool CWallet::TopUpKeyPool(unsigned int kpSize) { { @@ -3008,30 +3098,44 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) if (IsLocked()) return false; - CWalletDB walletdb(strWalletFile); - // Top up key pool unsigned int nTargetSize; if (kpSize > 0) nTargetSize = kpSize; else - nTargetSize = max(GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); + nTargetSize = std::max(GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); + + // count amount of available keys (internal, external) + // make sure the keypool of external and internal keys fits the user selected target (-keypool) + int64_t amountExternal = KeypoolCountExternalKeys(); + int64_t amountInternal = setKeyPool.size() - amountExternal; + int64_t missingExternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - amountExternal, (int64_t) 0); + int64_t missingInternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - amountInternal, (int64_t) 0); - while (setKeyPool.size() < (nTargetSize + 1)) + if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) + { + // don't create extra internal keys + missingInternal = 0; + } + bool internal = false; + CWalletDB walletdb(*dbw); + for (int64_t i = missingInternal + missingExternal; i--;) { int64_t nEnd = 1; + if (i < missingInternal) + internal = true; if (!setKeyPool.empty()) nEnd = *(--setKeyPool.end()) + 1; - if (!walletdb.WritePool(nEnd, CKeyPool(GenerateNewKey()))) - throw runtime_error(std::string(__func__) + ": writing generated key failed"); + if (!walletdb.WritePool(nEnd, CKeyPool(GenerateNewKey(internal), internal))) + throw std::runtime_error(std::string(__func__) + ": writing generated key failed"); setKeyPool.insert(nEnd); - LogPrintf("keypool added key %d, size=%u\n", nEnd, setKeyPool.size()); + LogPrintf("keypool added key %d, size=%u, internal=%d\n", nEnd, setKeyPool.size(), internal); } } return true; } -void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool) +void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool internal) { nIndex = -1; keypool.vchPubKey = CPubKey(); @@ -3045,27 +3149,34 @@ void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool) if(setKeyPool.empty()) return; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); - nIndex = *(setKeyPool.begin()); - setKeyPool.erase(setKeyPool.begin()); - if (!walletdb.ReadPool(nIndex, keypool)) - throw runtime_error(std::string(__func__) + ": read failed"); - if (!HaveKey(keypool.vchPubKey.GetID())) - throw runtime_error(std::string(__func__) + ": unknown key in key pool"); - assert(keypool.vchPubKey.IsValid()); - LogPrintf("keypool reserve %d\n", nIndex); + // try to find a key that matches the internal/external filter + for(const int64_t& id : setKeyPool) + { + CKeyPool tmpKeypool; + if (!walletdb.ReadPool(id, tmpKeypool)) + throw std::runtime_error(std::string(__func__) + ": read failed"); + if (!HaveKey(tmpKeypool.vchPubKey.GetID())) + throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); + if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT) || tmpKeypool.fInternal == internal) + { + nIndex = id; + keypool = tmpKeypool; + setKeyPool.erase(id); + assert(keypool.vchPubKey.IsValid()); + LogPrintf("keypool reserve %d\n", nIndex); + return; + } + } } } void CWallet::KeepKey(int64_t nIndex) { // Remove from key pool - if (fFileBacked) - { - CWalletDB walletdb(strWalletFile); - walletdb.ErasePool(nIndex); - } + CWalletDB walletdb(*dbw); + walletdb.ErasePool(nIndex); LogPrintf("keypool keep %d\n", nIndex); } @@ -3079,17 +3190,17 @@ void CWallet::ReturnKey(int64_t nIndex) LogPrintf("keypool return %d\n", nIndex); } -bool CWallet::GetKeyFromPool(CPubKey& result) +bool CWallet::GetKeyFromPool(CPubKey& result, bool internal) { int64_t nIndex = 0; CKeyPool keypool; { LOCK(cs_wallet); - ReserveKeyFromKeyPool(nIndex, keypool); + ReserveKeyFromKeyPool(nIndex, keypool, internal); if (nIndex == -1) { if (IsLocked()) return false; - result = GenerateNewKey(); + result = GenerateNewKey(internal); return true; } KeepKey(nIndex); @@ -3106,25 +3217,49 @@ int64_t CWallet::GetOldestKeyPoolTime() if (setKeyPool.empty()) return GetTime(); - // load oldest key from keypool, get time and return CKeyPool keypool; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); + + if (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) + { + // if HD & HD Chain Split is enabled, response max(oldest-internal-key, oldest-external-key) + int64_t now = GetTime(); + int64_t oldest_external = now, oldest_internal = now; + + for(const int64_t& id : setKeyPool) + { + if (!walletdb.ReadPool(id, keypool)) { + throw std::runtime_error(std::string(__func__) + ": read failed"); + } + if (keypool.fInternal && keypool.nTime < oldest_internal) { + oldest_internal = keypool.nTime; + } + else if (!keypool.fInternal && keypool.nTime < oldest_external) { + oldest_external = keypool.nTime; + } + if (oldest_internal != now && oldest_external != now) { + break; + } + } + return std::max(oldest_internal, oldest_external); + } + // load oldest key from keypool, get time and return int64_t nIndex = *(setKeyPool.begin()); if (!walletdb.ReadPool(nIndex, keypool)) - throw runtime_error(std::string(__func__) + ": read oldest key in keypool failed"); + throw std::runtime_error(std::string(__func__) + ": read oldest key in keypool failed"); assert(keypool.vchPubKey.IsValid()); return keypool.nTime; } std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() { - map<CTxDestination, CAmount> balances; + std::map<CTxDestination, CAmount> balances; { LOCK(cs_wallet); - BOOST_FOREACH(PAIRTYPE(uint256, CWalletTx) walletEntry, mapWallet) + for (const auto& walletEntry : mapWallet) { - CWalletTx *pcoin = &walletEntry.second; + const CWalletTx *pcoin = &walletEntry.second; if (!pcoin->IsTrusted()) continue; @@ -3156,15 +3291,15 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() return balances; } -set< set<CTxDestination> > CWallet::GetAddressGroupings() +std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() { AssertLockHeld(cs_wallet); // mapWallet - set< set<CTxDestination> > groupings; - set<CTxDestination> grouping; + std::set< std::set<CTxDestination> > groupings; + std::set<CTxDestination> grouping; - BOOST_FOREACH(PAIRTYPE(uint256, CWalletTx) walletEntry, mapWallet) + for (const auto& walletEntry : mapWallet) { - CWalletTx *pcoin = &walletEntry.second; + const CWalletTx *pcoin = &walletEntry.second; if (pcoin->tx->vin.size() > 0) { @@ -3201,11 +3336,11 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) - if (IsMine(pcoin->tx->vout[i])) + for (const auto& txout : pcoin->tx->vout) + if (IsMine(txout)) { CTxDestination address; - if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, address)) + if(!ExtractDestination(txout.scriptPubKey, address)) continue; grouping.insert(address); groupings.insert(grouping); @@ -3213,20 +3348,20 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() } } - set< set<CTxDestination>* > uniqueGroupings; // a set of pointers to groups of addresses - map< CTxDestination, set<CTxDestination>* > setmap; // map addresses to the unique group containing it - BOOST_FOREACH(set<CTxDestination> _grouping, groupings) + std::set< std::set<CTxDestination>* > uniqueGroupings; // a set of pointers to groups of addresses + std::map< CTxDestination, std::set<CTxDestination>* > setmap; // map addresses to the unique group containing it + BOOST_FOREACH(std::set<CTxDestination> _grouping, groupings) { // make a set of all the groups hit by this new group - set< set<CTxDestination>* > hits; - map< CTxDestination, set<CTxDestination>* >::iterator it; + std::set< std::set<CTxDestination>* > hits; + std::map< CTxDestination, std::set<CTxDestination>* >::iterator it; BOOST_FOREACH(CTxDestination address, _grouping) if ((it = setmap.find(address)) != setmap.end()) hits.insert((*it).second); // merge all hit groups into a new single group and delete old groups - set<CTxDestination>* merged = new set<CTxDestination>(_grouping); - BOOST_FOREACH(set<CTxDestination>* hit, hits) + std::set<CTxDestination>* merged = new std::set<CTxDestination>(_grouping); + BOOST_FOREACH(std::set<CTxDestination>* hit, hits) { merged->insert(hit->begin(), hit->end()); uniqueGroupings.erase(hit); @@ -3239,8 +3374,8 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() setmap[element] = merged; } - set< set<CTxDestination> > ret; - BOOST_FOREACH(set<CTxDestination>* uniqueGrouping, uniqueGroupings) + std::set< std::set<CTxDestination> > ret; + BOOST_FOREACH(std::set<CTxDestination>* uniqueGrouping, uniqueGroupings) { ret.insert(*uniqueGrouping); delete uniqueGrouping; @@ -3249,57 +3384,26 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() return ret; } -CAmount CWallet::GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CWalletDB walletdb(strWalletFile); - return GetAccountBalance(walletdb, strAccount, nMinDepth, filter); -} - -CAmount CWallet::GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CAmount nBalance = 0; - - // Tally wallet transactions - for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount nReceived, nSent, nFee; - wtx.GetAccountAmounts(strAccount, nReceived, nSent, nFee, filter); - - if (nReceived != 0 && wtx.GetDepthInMainChain() >= nMinDepth) - nBalance += nReceived; - nBalance -= nSent + nFee; - } - - // Tally internal accounting entries - nBalance += walletdb.GetAccountCreditDebit(strAccount); - - return nBalance; -} - std::set<CTxDestination> CWallet::GetAccountAddresses(const std::string& strAccount) const { LOCK(cs_wallet); - set<CTxDestination> result; + std::set<CTxDestination> result; BOOST_FOREACH(const PAIRTYPE(CTxDestination, CAddressBookData)& item, mapAddressBook) { const CTxDestination& address = item.first; - const string& strName = item.second.name; + const std::string& strName = item.second.name; if (strName == strAccount) result.insert(address); } return result; } -bool CReserveKey::GetReservedKey(CPubKey& pubkey) +bool CReserveKey::GetReservedKey(CPubKey& pubkey, bool internal) { if (nIndex == -1) { CKeyPool keypool; - pwallet->ReserveKeyFromKeyPool(nIndex, keypool); + pwallet->ReserveKeyFromKeyPool(nIndex, keypool, internal); if (nIndex != -1) vchPubKey = keypool.vchPubKey; else { @@ -3327,40 +3431,29 @@ void CReserveKey::ReturnKey() vchPubKey = CPubKey(); } -void CWallet::GetAllReserveKeys(set<CKeyID>& setAddress) const +void CWallet::GetAllReserveKeys(std::set<CKeyID>& setAddress) const { setAddress.clear(); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); LOCK2(cs_main, cs_wallet); BOOST_FOREACH(const int64_t& id, setKeyPool) { CKeyPool keypool; if (!walletdb.ReadPool(id, keypool)) - throw runtime_error(std::string(__func__) + ": read failed"); + throw std::runtime_error(std::string(__func__) + ": read failed"); assert(keypool.vchPubKey.IsValid()); CKeyID keyID = keypool.vchPubKey.GetID(); if (!HaveKey(keyID)) - throw runtime_error(std::string(__func__) + ": unknown key in key pool"); + throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); setAddress.insert(keyID); } } -void CWallet::UpdatedTransaction(const uint256 &hashTx) -{ - { - LOCK(cs_wallet); - // Only notify UI if this transaction is in this wallet - map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(hashTx); - if (mi != mapWallet.end()) - NotifyTransactionChanged(this, hashTx, CT_UPDATED); - } -} - -void CWallet::GetScriptForMining(boost::shared_ptr<CReserveScript> &script) +void CWallet::GetScriptForMining(std::shared_ptr<CReserveScript> &script) { - boost::shared_ptr<CReserveKey> rKey(new CReserveKey(this)); + std::shared_ptr<CReserveKey> rKey = std::make_shared<CReserveKey>(this); CPubKey pubkey; if (!rKey->GetReservedKey(pubkey)) return; @@ -3395,7 +3488,7 @@ bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const return (setLockedCoins.count(outpt) > 0); } -void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) +void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const { AssertLockHeld(cs_wallet); // setLockedCoins for (std::set<COutPoint>::iterator it = setLockedCoins.begin(); @@ -3490,7 +3583,72 @@ void CWallet::GetKeyBirthTimes(std::map<CTxDestination, int64_t> &mapKeyBirth) c // Extract block timestamps for those keys for (std::map<CKeyID, CBlockIndex*>::const_iterator it = mapKeyFirstBlock.begin(); it != mapKeyFirstBlock.end(); it++) - mapKeyBirth[it->first] = it->second->GetBlockTime() - 7200; // block times can be 2h off + mapKeyBirth[it->first] = it->second->GetBlockTime() - TIMESTAMP_WINDOW; // block times can be 2h off +} + +/** + * Compute smart timestamp for a transaction being added to the wallet. + * + * Logic: + * - If sending a transaction, assign its timestamp to the current time. + * - If receiving a transaction outside a block, assign its timestamp to the + * current time. + * - If receiving a block with a future timestamp, assign all its (not already + * known) transactions' timestamps to the current time. + * - If receiving a block with a past timestamp, before the most recent known + * transaction (that we care about), assign all its (not already known) + * transactions' timestamps to the same timestamp as that most-recent-known + * transaction. + * - If receiving a block with a past timestamp, but after the most recent known + * transaction, assign all its (not already known) transactions' timestamps to + * the block time. + * + * For more information see CWalletTx::nTimeSmart, + * https://bitcointalk.org/?topic=54527, or + * https://github.com/bitcoin/bitcoin/pull/1393. + */ +unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const +{ + unsigned int nTimeSmart = wtx.nTimeReceived; + if (!wtx.hashUnset()) { + if (mapBlockIndex.count(wtx.hashBlock)) { + int64_t latestNow = wtx.nTimeReceived; + int64_t latestEntry = 0; + + // Tolerate times up to the last timestamp in the wallet not more than 5 minutes into the future + int64_t latestTolerated = latestNow + 300; + const TxItems& txOrdered = wtxOrdered; + for (auto it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { + CWalletTx* const pwtx = it->second.first; + if (pwtx == &wtx) { + continue; + } + CAccountingEntry* const pacentry = it->second.second; + int64_t nSmartTime; + if (pwtx) { + nSmartTime = pwtx->nTimeSmart; + if (!nSmartTime) { + nSmartTime = pwtx->nTimeReceived; + } + } else { + nSmartTime = pacentry->nTime; + } + if (nSmartTime <= latestTolerated) { + latestEntry = nSmartTime; + if (nSmartTime > latestNow) { + latestNow = nSmartTime; + } + break; + } + } + + int64_t blocktime = mapBlockIndex[wtx.hashBlock]->GetBlockTime(); + nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); + } else { + LogPrintf("%s: found %s in block %s not in index\n", __func__, wtx.GetHash().ToString(), wtx.hashBlock.ToString()); + } + } + return nTimeSmart; } bool CWallet::AddDestData(const CTxDestination &dest, const std::string &key, const std::string &value) @@ -3499,18 +3657,14 @@ bool CWallet::AddDestData(const CTxDestination &dest, const std::string &key, co return false; mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteDestData(CBitcoinAddress(dest).ToString(), key, value); + return CWalletDB(*dbw).WriteDestData(CBitcoinAddress(dest).ToString(), key, value); } bool CWallet::EraseDestData(const CTxDestination &dest, const std::string &key) { if (!mapAddressBook[dest].destdata.erase(key)) return false; - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).EraseDestData(CBitcoinAddress(dest).ToString(), key); + return CWalletDB(*dbw).EraseDestData(CBitcoinAddress(dest).ToString(), key); } bool CWallet::LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value) @@ -3535,6 +3689,20 @@ bool CWallet::GetDestData(const CTxDestination &dest, const std::string &key, st return false; } +std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const +{ + LOCK(cs_wallet); + std::vector<std::string> values; + for (const auto& address : mapAddressBook) { + for (const auto& data : address.second.destdata) { + if (!data.first.compare(0, prefix.size(), prefix)) { + values.emplace_back(data.second); + } + } + } + return values; +} + std::string CWallet::GetWalletHelpString(bool showDebug) { std::string strUsage = HelpMessageGroup(_("Wallet options:")); @@ -3548,8 +3716,6 @@ std::string CWallet::GetWalletHelpString(bool showDebug) CURRENCY_UNIT, FormatMoney(payTxFee.GetFeePerK()))); strUsage += HelpMessageOpt("-rescan", _("Rescan the block chain for missing wallet transactions on startup")); strUsage += HelpMessageOpt("-salvagewallet", _("Attempt to recover private keys from a corrupt wallet on startup")); - if (showDebug) - strUsage += HelpMessageOpt("-sendfreetransactions", strprintf(_("Send transactions as zero-fee transactions if possible (default: %u)"), DEFAULT_SEND_FREE_TRANSACTIONS)); strUsage += HelpMessageOpt("-spendzeroconfchange", strprintf(_("Spend unconfirmed change when sending transactions (default: %u)"), DEFAULT_SPEND_ZEROCONF_CHANGE)); strUsage += HelpMessageOpt("-txconfirmtarget=<n>", strprintf(_("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)"), DEFAULT_TX_CONFIRM_TARGET)); strUsage += HelpMessageOpt("-usehd", _("Use hierarchical deterministic key generation (HD) after BIP32. Only has effect during wallet creation/first start") + " " + strprintf(_("(default: %u)"), DEFAULT_USE_HD_WALLET)); @@ -3582,7 +3748,8 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) if (GetBoolArg("-zapwallettxes", false)) { uiInterface.InitMessage(_("Zapping all transactions from wallet...")); - CWallet *tempWallet = new CWallet(walletFile); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); + CWallet *tempWallet = new CWallet(std::move(dbw)); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DB_LOAD_OK) { InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); @@ -3597,7 +3764,8 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) int64_t nStart = GetTimeMillis(); bool fFirstRun = true; - CWallet *walletInstance = new CWallet(walletFile); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); + CWallet *walletInstance = new CWallet(std::move(dbw)); DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DB_LOAD_OK) { @@ -3649,13 +3817,17 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) { // Create new keyUser and set as default key if (GetBoolArg("-usehd", DEFAULT_USE_HD_WALLET) && !walletInstance->IsHDEnabled()) { + + // ensure this wallet.dat can only be opened by clients supporting HD with chain split + walletInstance->SetMinVersion(FEATURE_HD_SPLIT); + // generate a new master key CPubKey masterPubKey = walletInstance->GenerateNewHDMasterKey(); if (!walletInstance->SetHDMasterKey(masterPubKey)) throw std::runtime_error(std::string(__func__) + ": Storing master key failed"); } CPubKey newDefaultKey; - if (walletInstance->GetKeyFromPool(newDefaultKey)) { + if (walletInstance->GetKeyFromPool(newDefaultKey, false)) { walletInstance->SetDefaultKey(newDefaultKey); if (!walletInstance->SetAddressBook(walletInstance->vchDefaultKey.GetID(), "", "receive")) { InitError(_("Cannot write default address") += "\n"); @@ -3681,17 +3853,13 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) RegisterValidationInterface(walletInstance); - CBlockIndex *pindexRescan = chainActive.Tip(); - if (GetBoolArg("-rescan", false)) - pindexRescan = chainActive.Genesis(); - else + CBlockIndex *pindexRescan = chainActive.Genesis(); + if (!GetBoolArg("-rescan", false)) { - CWalletDB walletdb(walletFile); + CWalletDB walletdb(*walletInstance->dbw); CBlockLocator locator; if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); - else - pindexRescan = chainActive.Genesis(); } if (chainActive.Tip() && chainActive.Tip() != pindexRescan) { @@ -3721,7 +3889,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) // Restore wallet transaction metadata after -zapwallettxes=1 if (GetBoolArg("-zapwallettxes", false) && GetArg("-zapwallettxes", "1") != "2") { - CWalletDB walletdb(walletFile); + CWalletDB walletdb(*walletInstance->dbw); BOOST_FOREACH(const CWalletTx& wtxOld, vWtx) { @@ -3765,6 +3933,12 @@ bool CWallet::InitLoadWallet() std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); + if (boost::filesystem::path(walletFile).filename() != walletFile) { + return InitError(_("-wallet parameter must only specify a filename (not a path)")); + } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { + return InitError(_("Invalid characters in -wallet filename")); + } + CWallet * const pwallet = CreateWalletFromFile(walletFile); if (!pwallet) { return false; @@ -3774,17 +3948,17 @@ bool CWallet::InitLoadWallet() return true; } -std::atomic<bool> CWallet::fFlushThreadRunning(false); +std::atomic<bool> CWallet::fFlushScheduled(false); -void CWallet::postInitProcess(boost::thread_group& threadGroup) +void CWallet::postInitProcess(CScheduler& scheduler) { // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded ReacceptWalletTransactions(); // Run a thread to flush wallet periodically - if (!CWallet::fFlushThreadRunning.exchange(true)) { - threadGroup.create_thread(ThreadFlushWalletDB); + if (!CWallet::fFlushScheduled.exchange(true)) { + scheduler.scheduleEvery(MaybeCompactWalletDB, 500); } } @@ -3868,60 +4042,27 @@ bool CWallet::ParameterInteraction() } nTxConfirmTarget = GetArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); bSpendZeroConfChange = GetBoolArg("-spendzeroconfchange", DEFAULT_SPEND_ZEROCONF_CHANGE); - fSendFreeTransactions = GetBoolArg("-sendfreetransactions", DEFAULT_SEND_FREE_TRANSACTIONS); fWalletRbf = GetBoolArg("-walletrbf", DEFAULT_WALLET_RBF); - if (fSendFreeTransactions && GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) <= 0) - return InitError("Creation of free transactions with their relay disabled is not supported."); - return true; } bool CWallet::BackupWallet(const std::string& strDest) { - if (!fFileBacked) - return false; - while (true) - { - { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(strWalletFile) || bitdb.mapFileUseCount[strWalletFile] == 0) - { - // Flush log data to the dat file - bitdb.CloseDb(strWalletFile); - bitdb.CheckpointLSN(strWalletFile); - bitdb.mapFileUseCount.erase(strWalletFile); - - // Copy wallet file - boost::filesystem::path pathSrc = GetDataDir() / strWalletFile; - boost::filesystem::path pathDest(strDest); - if (boost::filesystem::is_directory(pathDest)) - pathDest /= strWalletFile; - - try { - boost::filesystem::copy_file(pathSrc, pathDest, boost::filesystem::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strWalletFile, pathDest.string()); - return true; - } catch (const boost::filesystem::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", strWalletFile, pathDest.string(), e.what()); - return false; - } - } - } - MilliSleep(100); - } - return false; + return dbw->Backup(strDest); } CKeyPool::CKeyPool() { nTime = GetTime(); + fInternal = false; } -CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn) +CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn) { nTime = GetTime(); vchPubKey = vchPubKeyIn; + fInternal = internalIn; } CWalletKey::CWalletKey(int64_t nExpires) @@ -3962,7 +4103,7 @@ int CMerkleTx::GetBlocksToMaturity() const { if (!IsCoinBase()) return 0; - return max(0, (COINBASE_MATURITY+1) - GetDepthInMainChain()); + return std::max(0, (COINBASE_MATURITY+1) - GetDepthInMainChain()); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 98e4fb87b9..a3974bf00b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -7,6 +7,7 @@ #define BITCOIN_WALLET_WALLET_H #include "amount.h" +#include "policy/feerate.h" #include "streams.h" #include "tinyformat.h" #include "ui_interface.h" @@ -28,9 +29,6 @@ #include <utility> #include <vector> -#include <boost/shared_ptr.hpp> -#include <boost/thread.hpp> - extern CWallet* pwalletMain; /** @@ -39,7 +37,6 @@ extern CWallet* pwalletMain; extern CFeeRate payTxFee; extern unsigned int nTxConfirmTarget; extern bool bSpendZeroConfChange; -extern bool fSendFreeTransactions; extern bool fWalletRbf; static const unsigned int DEFAULT_KEYPOOL_SIZE = 100; @@ -57,16 +54,12 @@ static const CAmount MIN_CHANGE = CENT; static const CAmount MIN_FINAL_CHANGE = MIN_CHANGE/2; //! Default for -spendzeroconfchange static const bool DEFAULT_SPEND_ZEROCONF_CHANGE = true; -//! Default for -sendfreetransactions -static const bool DEFAULT_SEND_FREE_TRANSACTIONS = false; //! Default for -walletrejectlongchains static const bool DEFAULT_WALLET_REJECT_LONG_CHAINS = false; //! -txconfirmtarget default static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 6; //! -walletrbf default static const bool DEFAULT_WALLET_RBF = false; -//! Largest (in bytes) free transaction we're willing to create -static const unsigned int MAX_FREE_TRANSACTION_CREATE_SIZE = 1000; static const bool DEFAULT_WALLETBROADCAST = true; static const bool DEFAULT_DISABLE_WALLET = false; //! if set, all keys will be derived by using BIP32 @@ -79,7 +72,9 @@ class CCoinControl; class COutput; class CReserveKey; class CScript; +class CScheduler; class CTxMemPool; +class CBlockPolicyEstimator; class CWalletTx; /** (client) version numbers for particular wallet features */ @@ -91,6 +86,9 @@ enum WalletFeature FEATURE_COMPRPUBKEY = 60000, // compressed public keys FEATURE_HD = 130000, // Hierarchical key derivation after BIP32 (HD Wallet) + + FEATURE_HD_SPLIT = 139900, // Wallet with HD chain split (change outputs will use m/0'/1'/k) + FEATURE_LATEST = FEATURE_COMPRPUBKEY // HD is optional, use FEATURE_COMPRPUBKEY as latest version }; @@ -101,9 +99,10 @@ class CKeyPool public: int64_t nTime; CPubKey vchPubKey; + bool fInternal; // for change outputs CKeyPool(); - CKeyPool(const CPubKey& vchPubKeyIn); + CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); ADD_SERIALIZE_METHODS; @@ -114,6 +113,19 @@ public: READWRITE(nVersion); READWRITE(nTime); READWRITE(vchPubKey); + if (ser_action.ForRead()) { + try { + READWRITE(fInternal); + } + catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + fInternal = false; + } + } + else { + READWRITE(fInternal); + } } }; @@ -256,10 +268,44 @@ private: const CWallet* pwallet; public: + /** + * Key/value map with information about the transaction. + * + * The following keys can be read and written through the map and are + * serialized in the wallet database: + * + * "comment", "to" - comment strings provided to sendtoaddress, + * sendfrom, sendmany wallet RPCs + * "replaces_txid" - txid (as HexStr) of transaction replaced by + * bumpfee on transaction created by bumpfee + * "replaced_by_txid" - txid (as HexStr) of transaction created by + * bumpfee on transaction replaced by bumpfee + * "from", "message" - obsolete fields that could be set in UI prior to + * 2011 (removed in commit 4d9b223) + * + * The following keys are serialized in the wallet database, but shouldn't + * be read or written through the map (they will be temporarily added and + * removed from the map during serialization): + * + * "fromaccount" - serialized strFromAccount value + * "n" - serialized nOrderPos value + * "timesmart" - serialized nTimeSmart value + * "spent" - serialized vfSpent value that existed prior to + * 2014 (removed in commit 93a18a3) + */ mapValue_t mapValue; std::vector<std::pair<std::string, std::string> > vOrderForm; unsigned int fTimeReceivedIsTxTime; unsigned int nTimeReceived; //!< time received by this node + /** + * Stable timestamp that never changes, and reflects the order a transaction + * was added to the wallet. Timestamp is based on the block time for a + * transaction added as part of a block, or else the time when the + * transaction was received if it wasn't part of a block, with the timestamp + * adjusted in both cases so timestamp order matches the order transactions + * were added to the wallet. More details can be found in + * CWallet::ComputeTimeSmart(). + */ unsigned int nTimeSmart; /** * From me flag is set to 1 for transactions that were created by the wallet @@ -369,7 +415,6 @@ public: } mapValue.erase("fromaccount"); - mapValue.erase("version"); mapValue.erase("spent"); mapValue.erase("n"); mapValue.erase("timesmart"); @@ -407,9 +452,6 @@ public: void GetAmounts(std::list<COutputEntry>& listReceived, std::list<COutputEntry>& listSent, CAmount& nFee, std::string& strSentAccount, const isminefilter& filter) const; - void GetAccountAmounts(const std::string& strAccount, CAmount& nReceived, - CAmount& nSent, CAmount& nFee, const isminefilter& filter) const; - bool IsFromMe(const isminefilter& filter) const { return (GetDebit(filter) > 0); @@ -430,7 +472,34 @@ public: }; +class CInputCoin { +public: + CInputCoin(const CWalletTx* walletTx, unsigned int i) + { + if (!walletTx) + throw std::invalid_argument("walletTx should not be null"); + if (i >= walletTx->tx->vout.size()) + throw std::out_of_range("The output index is out of range"); + outpoint = COutPoint(walletTx->GetHash(), i); + txout = walletTx->tx->vout[i]; + } + + COutPoint outpoint; + CTxOut txout; + + bool operator<(const CInputCoin& rhs) const { + return outpoint < rhs.outpoint; + } + + bool operator!=(const CInputCoin& rhs) const { + return outpoint != rhs.outpoint; + } + + bool operator==(const CInputCoin& rhs) const { + return outpoint == rhs.outpoint; + } +}; class COutput { @@ -438,12 +507,23 @@ public: const CWalletTx *tx; int i; int nDepth; + + /** Whether we have the private keys to spend this output */ bool fSpendable; + + /** Whether we know how to spend this output, ignoring the lack of keys */ bool fSolvable; - COutput(const CWalletTx *txIn, int iIn, int nDepthIn, bool fSpendableIn, bool fSolvableIn) + /** + * Whether this output is considered safe to spend. Unconfirmed transactions + * from outside keys and unconfirmed replacement transactions are considered + * unsafe and will not be used to fund new spending transactions. + */ + bool fSafe; + + COutput(const CWalletTx *txIn, int iIn, int nDepthIn, bool fSpendableIn, bool fSolvableIn, bool fSafeIn) { - tx = txIn; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; fSolvable = fSolvableIn; + tx = txIn; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; fSolvable = fSolvableIn; fSafe = fSafeIn; } std::string ToString() const; @@ -569,14 +649,16 @@ private: class CWallet : public CCryptoKeyStore, public CValidationInterface { private: - static std::atomic<bool> fFlushThreadRunning; + static std::atomic<bool> fFlushScheduled; + std::atomic<bool> fAbortRescan; + std::atomic<bool> fScanningWallet; /** * Select a set of coins such that nValueRet >= nTargetValue and at least * all coins from coinControl are selected; Never select unconfirmed coins * if they are not ours */ - bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet, const CCoinControl *coinControl = NULL) const; + bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl *coinControl = NULL) const; CWalletDB *pwalletdbEncryption; @@ -605,10 +687,15 @@ private: void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>); + /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected. + * Should be called with pindexBlock and posInBlock if this is for a transaction that is included in a block. */ + void SyncTransaction(const CTransactionRef& tx, const CBlockIndex *pindex = NULL, int posInBlock = 0); + /* the HD chain data model (external chain counters) */ CHDChain hdChain; - bool fFileBacked; + /* HD derive new child key (on internal or external chain) */ + void DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool internal = false); std::set<int64_t> setKeyPool; @@ -625,17 +712,33 @@ private: */ bool AddWatchOnly(const CScript& dest) override; + std::unique_ptr<CWalletDBWrapper> dbw; + public: /* * Main wallet lock. - * This lock protects all the fields added by CWallet - * except for: - * fFileBacked (immutable after instantiation) - * strWalletFile (immutable after instantiation) + * This lock protects all the fields added by CWallet. */ mutable CCriticalSection cs_wallet; - const std::string strWalletFile; + /** Get database handle used by this wallet. Ideally this function would + * not be necessary. + */ + CWalletDBWrapper& GetDBHandle() + { + return *dbw; + } + + /** Get a name for this wallet for logging/debugging purposes. + */ + std::string GetName() const + { + if (dbw) { + return dbw->GetName(); + } else { + return "dummy"; + } + } void LoadKeyPool(int nIndex, const CKeyPool &keypool) { @@ -657,15 +760,16 @@ public: MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID; - CWallet() + // Create wallet with dummy database handle + CWallet(): dbw(new CWalletDBWrapper()) { SetNull(); } - CWallet(const std::string& strWalletFileIn) : strWalletFile(strWalletFileIn) + // Create wallet with passed-in database handle + CWallet(std::unique_ptr<CWalletDBWrapper> dbw_in) : dbw(std::move(dbw_in)) { SetNull(); - fFileBacked = true; } ~CWallet() @@ -678,7 +782,6 @@ public: { nWalletVersion = FEATURE_BASE; nWalletMaxVersion = FEATURE_BASE; - fFileBacked = false; nMasterKeyMaxID = 0; pwalletdbEncryption = NULL; nOrderPosNext = 0; @@ -686,6 +789,9 @@ public: nLastResend = 0; nTimeFirstKey = 0; fBroadcastTransactions = false; + nRelockTime = 0; + fAbortRescan = false; + fScanningWallet = false; } std::map<uint256, CWalletTx> mapWallet; @@ -712,7 +818,17 @@ public: /** * populate vCoins with vector of available COutputs. */ - void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlyConfirmed=true, const CCoinControl *coinControl = NULL, bool fIncludeZeroValue=false) const; + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = NULL, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t& nMaximumCount = 0, const int& nMinDepth = 0, const int& nMaxDepth = 9999999) const; + + /** + * Return list of available coins and locked coins grouped by non-change output address. + */ + std::map<CTxDestination, std::vector<COutput>> ListCoins() const; + + /** + * Find non-change parent output. + */ + const CTxOut& FindNonChangeParentOutput(const CTransaction& tx, int output) const; /** * Shuffle and select coins until nTargetValue is reached while avoiding @@ -720,7 +836,7 @@ public: * completion the coin set and corresponding actual target value is * assembled */ - bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, uint64_t nMaxAncestors, std::vector<COutput> vCoins, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const; + bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, uint64_t nMaxAncestors, std::vector<COutput> vCoins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet) const; bool IsSpent(const uint256& hash, unsigned int n) const; @@ -728,14 +844,20 @@ public: void LockCoin(const COutPoint& output); void UnlockCoin(const COutPoint& output); void UnlockAllCoins(); - void ListLockedCoins(std::vector<COutPoint>& vOutpts); + void ListLockedCoins(std::vector<COutPoint>& vOutpts) const; + + /* + * Rescan abort properties + */ + void AbortRescan() { fAbortRescan = true; } + bool IsAbortingRescan() { return fAbortRescan; } + bool IsScanning() { return fScanningWallet; } /** * keystore implementation * Generate a new key */ - CPubKey GenerateNewKey(); - void DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret); + CPubKey GenerateNewKey(bool internal = false); //! Adds a key to the store, and saves it to disk. bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; //! Adds a key to the store, without saving it to disk (used by LoadWallet) @@ -761,6 +883,8 @@ public: bool LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value); //! Look up a destination data tuple in the store, return true if found false otherwise bool GetDestData(const CTxDestination &dest, const std::string &key, std::string *value) const; + //! Get all destination values matching a prefix. + std::vector<std::string> GetDestValues(const std::string& prefix) const; //! Adds a watch-only address to the store, and saves it to disk. bool AddWatchOnly(const CScript& dest, int64_t nCreateTime); @@ -768,11 +892,15 @@ public: //! Adds a watch-only address to the store, without saving it to disk (used by LoadWallet) bool LoadWatchOnly(const CScript &dest); + //! Holds a timestamp at which point the wallet is scheduled (externally) to be relocked. Caller must arrange for actual relocking to occur via Lock(). + int64_t nRelockTime; + bool Unlock(const SecureString& strWalletPassphrase); bool ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, const SecureString& strNewWalletPassphrase); bool EncryptWallet(const SecureString& strWalletPassphrase); void GetKeyBirthTimes(std::map<CTxDestination, int64_t> &mapKeyBirth) const; + unsigned int ComputeTimeSmart(const CWalletTx& wtx) const; /** * Increment the next transaction order id @@ -786,8 +914,10 @@ public: void MarkDirty(); bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); bool LoadToWallet(const CWalletTx& wtxIn); - void SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock) override; - bool AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); + void TransactionAddedToMempool(const CTransactionRef& tx) override; + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false); void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override; @@ -798,12 +928,15 @@ public: CAmount GetWatchOnlyBalance() const; CAmount GetUnconfirmedWatchOnlyBalance() const; CAmount GetImmatureWatchOnlyBalance() const; + CAmount GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const; + CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; /** * Insert additional inputs into the transaction by * calling CreateTransaction(); */ bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool overrideEstimatedFeeRate, const CFeeRate& specificFeeRate, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, bool keepReserveKey = true, const CTxDestination& destChange = CNoDestination()); + bool SignTransaction(CMutableTransaction& tx); /** * Create a new transaction paying the recipients with a set of coins @@ -818,7 +951,7 @@ public: bool AddAccountingEntry(const CAccountingEntry&); bool AddAccountingEntry(const CAccountingEntry&, CWalletDB *pwalletdb); template <typename ContainerType> - bool DummySignTx(CMutableTransaction &txNew, const ContainerType &coins); + bool DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) const; static CFeeRate minTxFee; static CFeeRate fallbackFee; @@ -826,12 +959,7 @@ public: * Estimate the minimum fee considering user set parameters * and the required fee */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool); - /** - * Estimate the minimum fee considering required fee and targetFee or if 0 - * then fee estimation for nConfirmTarget - */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreGlobalPayTxFee = false); /** * Return the minimum required fee taking into account the * floating relay fee and user set minimum transaction fee @@ -839,19 +967,18 @@ public: static CAmount GetRequiredFee(unsigned int nTxBytes); bool NewKeyPool(); + size_t KeypoolCountExternalKeys(); bool TopUpKeyPool(unsigned int kpSize = 0); - void ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool); + void ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool internal); void KeepKey(int64_t nIndex); void ReturnKey(int64_t nIndex); - bool GetKeyFromPool(CPubKey &key); + bool GetKeyFromPool(CPubKey &key, bool internal = false); int64_t GetOldestKeyPoolTime(); void GetAllReserveKeys(std::set<CKeyID>& setAddress) const; std::set< std::set<CTxDestination> > GetAddressGroupings(); std::map<CTxDestination, CAmount> GetAddressBalances(); - CAmount GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter); - CAmount GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter); std::set<CTxDestination> GetAccountAddresses(const std::string& strAccount) const; isminetype IsMine(const CTxIn& txin) const; @@ -882,7 +1009,7 @@ public: bool DelAddressBook(const CTxDestination& address); - void UpdatedTransaction(const uint256 &hashTx) override; + const std::string& GetAccountName(const CScript& scriptPubKey) const; void Inventory(const uint256 &hash) override { @@ -894,12 +1021,7 @@ public: } } - void GetScriptForMining(boost::shared_ptr<CReserveScript> &script) override; - void ResetRequestCount(const uint256 &hash) override - { - LOCK(cs_wallet); - mapRequestCount[hash] = 0; - }; + void GetScriptForMining(std::shared_ptr<CReserveScript> &script) override; unsigned int GetKeyPoolSize() { @@ -957,6 +1079,9 @@ public: /** Set whether this wallet broadcasts transactions. */ void SetBroadcastTransactions(bool broadcast) { fBroadcastTransactions = broadcast; } + /** Return whether transaction can be abandoned */ + bool TransactionCanBeAbandoned(const uint256& hashTx) const; + /* Mark a transaction (and it in-wallet descendants) as abandoned so its inputs may be respent. */ bool AbandonTransaction(const uint256& hashTx); @@ -974,7 +1099,7 @@ public: * Wallet post-init setup * Gives the wallet a chance to register repetitive tasks and complete post-init tasks */ - void postInitProcess(boost::thread_group& threadGroup); + void postInitProcess(CScheduler& scheduler); /* Wallets parameter interaction */ static bool ParameterInteraction(); @@ -983,15 +1108,18 @@ public: /* Set the HD chain model (chain child index counters) */ bool SetHDChain(const CHDChain& chain, bool memonly); - const CHDChain& GetHDChain() { return hdChain; } + const CHDChain& GetHDChain() const { return hdChain; } /* Returns true if HD is enabled */ - bool IsHDEnabled(); + bool IsHDEnabled() const; /* Generates a new HD master key (will not be activated) */ CPubKey GenerateNewHDMasterKey(); - /* Set the current HD master key (will reset the chain child index counters) */ + /* Set the current HD master key (will reset the chain child index counters) + Sets the master key's version based on the current wallet version (so the + caller must ensure the current wallet version is correct before calling + this function). */ bool SetHDMasterKey(const CPubKey& key); }; @@ -1009,13 +1137,17 @@ public: pwallet = pwalletIn; } + CReserveKey() = default; + CReserveKey(const CReserveKey&) = delete; + CReserveKey& operator=(const CReserveKey&) = delete; + ~CReserveKey() { ReturnKey(); } void ReturnKey(); - bool GetReservedKey(CPubKey &pubkey); + bool GetReservedKey(CPubKey &pubkey, bool internal = false); void KeepKey(); void KeepScript() { KeepKey(); } }; @@ -1055,13 +1187,13 @@ public: // ContainerType is meant to hold pair<CWalletTx *, int>, and be iterable // so that each entry corresponds to each vIn, in order. template <typename ContainerType> -bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) +bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) const { // Fill in dummy signatures for fee calculation. int nIn = 0; for (const auto& coin : coins) { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; if (!ProduceSignature(DummySignatureCreator(this), scriptPubKey, sigdata)) diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 81fdde401e..342c797dd3 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -6,8 +6,9 @@ #include "wallet/walletdb.h" #include "base58.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" -#include "validation.h" // For CheckTransaction +#include "fs.h" #include "protocol.h" #include "serialize.h" #include "sync.h" @@ -18,12 +19,9 @@ #include <atomic> #include <boost/version.hpp> -#include <boost/filesystem.hpp> #include <boost/foreach.hpp> #include <boost/thread.hpp> -using namespace std; - static uint64_t nAccountingEntryNumber = 0; static std::atomic<unsigned int> nWalletDBUpdateCounter; @@ -32,49 +30,49 @@ static std::atomic<unsigned int> nWalletDBUpdateCounter; // CWalletDB // -bool CWalletDB::WriteName(const string& strAddress, const string& strName) +bool CWalletDB::WriteName(const std::string& strAddress, const std::string& strName) { nWalletDBUpdateCounter++; - return Write(make_pair(string("name"), strAddress), strName); + return batch.Write(std::make_pair(std::string("name"), strAddress), strName); } -bool CWalletDB::EraseName(const string& strAddress) +bool CWalletDB::EraseName(const std::string& strAddress) { // This should only be used for sending addresses, never for receiving addresses, // receiving addresses must always have an address book entry if they're not change return. nWalletDBUpdateCounter++; - return Erase(make_pair(string("name"), strAddress)); + return batch.Erase(std::make_pair(std::string("name"), strAddress)); } -bool CWalletDB::WritePurpose(const string& strAddress, const string& strPurpose) +bool CWalletDB::WritePurpose(const std::string& strAddress, const std::string& strPurpose) { nWalletDBUpdateCounter++; - return Write(make_pair(string("purpose"), strAddress), strPurpose); + return batch.Write(std::make_pair(std::string("purpose"), strAddress), strPurpose); } -bool CWalletDB::ErasePurpose(const string& strPurpose) +bool CWalletDB::ErasePurpose(const std::string& strPurpose) { nWalletDBUpdateCounter++; - return Erase(make_pair(string("purpose"), strPurpose)); + return batch.Erase(std::make_pair(std::string("purpose"), strPurpose)); } bool CWalletDB::WriteTx(const CWalletTx& wtx) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); + return batch.Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); } bool CWalletDB::EraseTx(uint256 hash) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("tx"), hash)); + return batch.Erase(std::make_pair(std::string("tx"), hash)); } bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata& keyMeta) { nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), + if (!batch.Write(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta, false)) return false; @@ -84,7 +82,7 @@ bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, c vchKey.insert(vchKey.end(), vchPubKey.begin(), vchPubKey.end()); vchKey.insert(vchKey.end(), vchPrivKey.begin(), vchPrivKey.end()); - return Write(std::make_pair(std::string("key"), vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); + return batch.Write(std::make_pair(std::string("key"), vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); } bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, @@ -94,16 +92,16 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, const bool fEraseUnencryptedKey = true; nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), + if (!batch.Write(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta)) return false; - if (!Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) + if (!batch.Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) return false; if (fEraseUnencryptedKey) { - Erase(std::make_pair(std::string("key"), vchPubKey)); - Erase(std::make_pair(std::string("wkey"), vchPubKey)); + batch.Erase(std::make_pair(std::string("key"), vchPubKey)); + batch.Erase(std::make_pair(std::string("wkey"), vchPubKey)); } return true; } @@ -111,92 +109,92 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, bool CWalletDB::WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); + return batch.Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); } bool CWalletDB::WriteCScript(const uint160& hash, const CScript& redeemScript) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); + return batch.Write(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); } bool CWalletDB::WriteWatchOnly(const CScript &dest, const CKeyMetadata& keyMeta) { nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) + if (!batch.Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) return false; - return Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); + return batch.Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); } bool CWalletDB::EraseWatchOnly(const CScript &dest) { nWalletDBUpdateCounter++; - if (!Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) + if (!batch.Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) return false; - return Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); + return batch.Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); } bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) { nWalletDBUpdateCounter++; - Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan - return Write(std::string("bestblock_nomerkle"), locator); + batch.Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan + return batch.Write(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::ReadBestBlock(CBlockLocator& locator) { - if (Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; - return Read(std::string("bestblock_nomerkle"), locator); + if (batch.Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; + return batch.Read(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::WriteOrderPosNext(int64_t nOrderPosNext) { nWalletDBUpdateCounter++; - return Write(std::string("orderposnext"), nOrderPosNext); + return batch.Write(std::string("orderposnext"), nOrderPosNext); } bool CWalletDB::WriteDefaultKey(const CPubKey& vchPubKey) { nWalletDBUpdateCounter++; - return Write(std::string("defaultkey"), vchPubKey); + return batch.Write(std::string("defaultkey"), vchPubKey); } bool CWalletDB::ReadPool(int64_t nPool, CKeyPool& keypool) { - return Read(std::make_pair(std::string("pool"), nPool), keypool); + return batch.Read(std::make_pair(std::string("pool"), nPool), keypool); } bool CWalletDB::WritePool(int64_t nPool, const CKeyPool& keypool) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("pool"), nPool), keypool); + return batch.Write(std::make_pair(std::string("pool"), nPool), keypool); } bool CWalletDB::ErasePool(int64_t nPool) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("pool"), nPool)); + return batch.Erase(std::make_pair(std::string("pool"), nPool)); } bool CWalletDB::WriteMinVersion(int nVersion) { - return Write(std::string("minversion"), nVersion); + return batch.Write(std::string("minversion"), nVersion); } -bool CWalletDB::ReadAccount(const string& strAccount, CAccount& account) +bool CWalletDB::ReadAccount(const std::string& strAccount, CAccount& account) { account.SetNull(); - return Read(make_pair(string("acc"), strAccount), account); + return batch.Read(std::make_pair(std::string("acc"), strAccount), account); } -bool CWalletDB::WriteAccount(const string& strAccount, const CAccount& account) +bool CWalletDB::WriteAccount(const std::string& strAccount, const CAccount& account) { - return Write(make_pair(string("acc"), strAccount), account); + return batch.Write(std::make_pair(std::string("acc"), strAccount), account); } bool CWalletDB::WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry) { - return Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); + return batch.Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); } bool CWalletDB::WriteAccountingEntry_Backend(const CAccountingEntry& acentry) @@ -204,9 +202,9 @@ bool CWalletDB::WriteAccountingEntry_Backend(const CAccountingEntry& acentry) return WriteAccountingEntry(++nAccountingEntryNumber, acentry); } -CAmount CWalletDB::GetAccountCreditDebit(const string& strAccount) +CAmount CWalletDB::GetAccountCreditDebit(const std::string& strAccount) { - list<CAccountingEntry> entries; + std::list<CAccountingEntry> entries; ListAccountCreditDebit(strAccount, entries); CAmount nCreditDebit = 0; @@ -216,33 +214,33 @@ CAmount CWalletDB::GetAccountCreditDebit(const string& strAccount) return nCreditDebit; } -void CWalletDB::ListAccountCreditDebit(const string& strAccount, list<CAccountingEntry>& entries) +void CWalletDB::ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries) { bool fAllAccounts = (strAccount == "*"); - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) - throw runtime_error(std::string(__func__) + ": cannot create DB cursor"); + throw std::runtime_error(std::string(__func__) + ": cannot create DB cursor"); bool setRange = true; while (true) { // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); if (setRange) - ssKey << std::make_pair(std::string("acentry"), std::make_pair((fAllAccounts ? string("") : strAccount), uint64_t(0))); + ssKey << std::make_pair(std::string("acentry"), std::make_pair((fAllAccounts ? std::string("") : strAccount), uint64_t(0))); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue, setRange); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue, setRange); setRange = false; if (ret == DB_NOTFOUND) break; else if (ret != 0) { pcursor->close(); - throw runtime_error(std::string(__func__) + ": error scanning DB"); + throw std::runtime_error(std::string(__func__) + ": error scanning DB"); } // Unserialize - string strType; + std::string strType; ssKey >> strType; if (strType != "acentry") break; @@ -268,7 +266,7 @@ public: bool fIsEncrypted; bool fAnyUnordered; int nFileVersion; - vector<uint256> vWalletUpgrade; + std::vector<uint256> vWalletUpgrade; CWalletScanState() { nKeys = nCKeys = nWatchKeys = nKeyMeta = 0; @@ -280,7 +278,7 @@ public: bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, - CWalletScanState &wss, string& strType, string& strErr) + CWalletScanState &wss, std::string& strType, std::string& strErr) { try { // Unserialize @@ -289,13 +287,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssKey >> strType; if (strType == "name") { - string strAddress; + std::string strAddress; ssKey >> strAddress; ssValue >> pwallet->mapAddressBook[CBitcoinAddress(strAddress).Get()].name; } else if (strType == "purpose") { - string strAddress; + std::string strAddress; ssKey >> strAddress; ssValue >> pwallet->mapAddressBook[CBitcoinAddress(strAddress).Get()].purpose; } @@ -336,7 +334,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == "acentry") { - string strAccount; + std::string strAccount; ssKey >> strAccount; uint64_t nNumber; ssKey >> nNumber; @@ -449,7 +447,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, strErr = "Error reading wallet database: CPubKey corrupt"; return false; } - vector<unsigned char> vchPrivKey; + std::vector<unsigned char> vchPrivKey; ssValue >> vchPrivKey; wss.nCKeys++; @@ -546,7 +544,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, return true; } -static bool IsKeyType(string strType) +bool CWalletDB::IsKeyType(const std::string& strType) { return (strType== "key" || strType == "wkey" || strType == "mkey" || strType == "ckey"); @@ -562,7 +560,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) LOCK(pwallet->cs_wallet); try { int nMinVersion = 0; - if (Read((string)"minversion", nMinVersion)) + if (batch.Read((std::string)"minversion", nMinVersion)) { if (nMinVersion > CLIENT_VERSION) return DB_TOO_NEW; @@ -570,7 +568,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) } // Get cursor - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); @@ -582,7 +580,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) break; else if (ret != 0) @@ -592,7 +590,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) } // Try to be tolerant of single corrupt records: - string strType, strErr; + std::string strType, strErr; if (!ReadKeyValue(pwallet, ssKey, ssValue, wss, strType, strErr)) { // losing keys is considered a catastrophic error, anything else @@ -659,24 +657,21 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) return result; } -DBErrors CWalletDB::FindWalletTx(CWallet* pwallet, vector<uint256>& vTxHash, vector<CWalletTx>& vWtx) +DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx) { - pwallet->vchDefaultKey = CPubKey(); bool fNoncriticalErrors = false; DBErrors result = DB_LOAD_OK; try { - LOCK(pwallet->cs_wallet); int nMinVersion = 0; - if (Read((string)"minversion", nMinVersion)) + if (batch.Read((std::string)"minversion", nMinVersion)) { if (nMinVersion > CLIENT_VERSION) return DB_TOO_NEW; - pwallet->LoadMinVersion(nMinVersion); } // Get cursor - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); @@ -688,7 +683,7 @@ DBErrors CWalletDB::FindWalletTx(CWallet* pwallet, vector<uint256>& vTxHash, vec // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) break; else if (ret != 0) @@ -697,7 +692,7 @@ DBErrors CWalletDB::FindWalletTx(CWallet* pwallet, vector<uint256>& vTxHash, vec return DB_CORRUPT; } - string strType; + std::string strType; ssKey >> strType; if (strType == "tx") { uint256 hash; @@ -725,12 +720,12 @@ DBErrors CWalletDB::FindWalletTx(CWallet* pwallet, vector<uint256>& vTxHash, vec return result; } -DBErrors CWalletDB::ZapSelectTx(CWallet* pwallet, vector<uint256>& vTxHashIn, vector<uint256>& vTxHashOut) +DBErrors CWalletDB::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<uint256>& vTxHashOut) { // build list of wallet TXs and hashes - vector<uint256> vTxHash; - vector<CWalletTx> vWtx; - DBErrors err = FindWalletTx(pwallet, vTxHash, vWtx); + std::vector<uint256> vTxHash; + std::vector<CWalletTx> vWtx; + DBErrors err = FindWalletTx(vTxHash, vWtx); if (err != DB_LOAD_OK) { return err; } @@ -740,7 +735,7 @@ DBErrors CWalletDB::ZapSelectTx(CWallet* pwallet, vector<uint256>& vTxHashIn, ve // erase each matching wallet TX bool delerror = false; - vector<uint256>::iterator it = vTxHashIn.begin(); + std::vector<uint256>::iterator it = vTxHashIn.begin(); BOOST_FOREACH (uint256 hash, vTxHash) { while (it < vTxHashIn.end() && (*it) < hash) { it++; @@ -749,9 +744,8 @@ DBErrors CWalletDB::ZapSelectTx(CWallet* pwallet, vector<uint256>& vTxHashIn, ve break; } else if ((*it) == hash) { - pwallet->mapWallet.erase(hash); if(!EraseTx(hash)) { - LogPrint("db", "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); + LogPrint(BCLog::DB, "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); delerror = true; } vTxHashOut.push_back(hash); @@ -764,11 +758,11 @@ DBErrors CWalletDB::ZapSelectTx(CWallet* pwallet, vector<uint256>& vTxHashIn, ve return DB_LOAD_OK; } -DBErrors CWalletDB::ZapWalletTx(CWallet* pwallet, vector<CWalletTx>& vWtx) +DBErrors CWalletDB::ZapWalletTx(std::vector<CWalletTx>& vWtx) { // build list of wallet TXs - vector<uint256> vTxHash; - DBErrors err = FindWalletTx(pwallet, vTxHash, vWtx); + std::vector<uint256> vTxHash; + DBErrors err = FindWalletTx(vTxHash, vWtx); if (err != DB_LOAD_OK) return err; @@ -781,175 +775,100 @@ DBErrors CWalletDB::ZapWalletTx(CWallet* pwallet, vector<CWalletTx>& vWtx) return DB_LOAD_OK; } -void ThreadFlushWalletDB() +void MaybeCompactWalletDB() { - // Make this thread recognisable as the wallet flushing thread - RenameThread("bitcoin-wallet"); - - static bool fOneThread; - if (fOneThread) + static std::atomic<bool> fOneThread; + if (fOneThread.exchange(true)) { return; - fOneThread = true; - if (!GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) + } + if (!GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { return; + } - unsigned int nLastSeen = CWalletDB::GetUpdateCounter(); - unsigned int nLastFlushed = CWalletDB::GetUpdateCounter(); - int64_t nLastWalletUpdate = GetTime(); - while (true) - { - MilliSleep(500); - - if (nLastSeen != CWalletDB::GetUpdateCounter()) - { - nLastSeen = CWalletDB::GetUpdateCounter(); - nLastWalletUpdate = GetTime(); - } + static unsigned int nLastSeen = CWalletDB::GetUpdateCounter(); + static unsigned int nLastFlushed = CWalletDB::GetUpdateCounter(); + static int64_t nLastWalletUpdate = GetTime(); - if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) - { - TRY_LOCK(bitdb.cs_db,lockDb); - if (lockDb) - { - // Don't do this if any databases are in use - int nRefCount = 0; - map<string, int>::iterator mi = bitdb.mapFileUseCount.begin(); - while (mi != bitdb.mapFileUseCount.end()) - { - nRefCount += (*mi).second; - mi++; - } + if (nLastSeen != CWalletDB::GetUpdateCounter()) + { + nLastSeen = CWalletDB::GetUpdateCounter(); + nLastWalletUpdate = GetTime(); + } - if (nRefCount == 0) - { - boost::this_thread::interruption_point(); - const std::string& strFile = pwalletMain->strWalletFile; - map<string, int>::iterator _mi = bitdb.mapFileUseCount.find(strFile); - if (_mi != bitdb.mapFileUseCount.end()) - { - LogPrint("db", "Flushing %s\n", strFile); - nLastFlushed = CWalletDB::GetUpdateCounter(); - int64_t nStart = GetTimeMillis(); - - // Flush wallet file so it's self contained - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); - - bitdb.mapFileUseCount.erase(_mi++); - LogPrint("db", "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); - } - } - } + if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) + { + if (CDB::PeriodicFlush(pwalletMain->GetDBHandle())) { + nLastFlushed = CWalletDB::GetUpdateCounter(); } } + fOneThread = false; } // // Try to (very carefully!) recover wallet file if there is a problem. // -bool CWalletDB::Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys) -{ - // Recovery procedure: - // move wallet file to wallet.timestamp.bak - // Call Salvage with fAggressive=true to - // get as much data as possible. - // Rewrite salvaged data to fresh wallet file - // Set -rescan so any missing transactions will be - // found. - int64_t now = GetTime(); - std::string newFilename = strprintf("wallet.%d.bak", now); - - int result = dbenv.dbenv->dbrename(NULL, filename.c_str(), NULL, - newFilename.c_str(), DB_AUTO_COMMIT); - if (result == 0) - LogPrintf("Renamed %s to %s\n", filename, newFilename); - else - { - LogPrintf("Failed to rename %s to %s\n", filename, newFilename); - return false; - } +bool CWalletDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)) +{ + return CDB::Recover(filename, callbackDataIn, recoverKVcallback); +} - std::vector<CDBEnv::KeyValPair> salvagedData; - bool fSuccess = dbenv.Salvage(newFilename, true, salvagedData); - if (salvagedData.empty()) +bool CWalletDB::Recover(const std::string& filename) +{ + // recover without a key filter callback + // results in recovering all record types + return CWalletDB::Recover(filename, NULL, NULL); +} + +bool CWalletDB::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue) +{ + CWallet *dummyWallet = reinterpret_cast<CWallet*>(callbackData); + CWalletScanState dummyWss; + std::string strType, strErr; + bool fReadOK; { - LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); - return false; + // Required in LoadKeyMetadata(): + LOCK(dummyWallet->cs_wallet); + fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, + dummyWss, strType, strErr); } - LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); - - std::unique_ptr<Db> pdbCopy(new Db(dbenv.dbenv, 0)); - int ret = pdbCopy->open(NULL, // Txn pointer - filename.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) + if (!IsKeyType(strType) && strType != "hdchain") + return false; + if (!fReadOK) { - LogPrintf("Cannot create database file %s\n", filename); + LogPrintf("WARNING: CWalletDB::Recover skipping %s: %s\n", strType, strErr); return false; } - CWallet dummyWallet; - CWalletScanState wss; - DbTxn* ptxn = dbenv.TxnBegin(); - BOOST_FOREACH(CDBEnv::KeyValPair& row, salvagedData) - { - if (fOnlyKeys) - { - CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); - CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); - string strType, strErr; - bool fReadOK; - { - // Required in LoadKeyMetadata(): - LOCK(dummyWallet.cs_wallet); - fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, - wss, strType, strErr); - } - if (!IsKeyType(strType) && strType != "hdchain") - continue; - if (!fReadOK) - { - LogPrintf("WARNING: CWalletDB::Recover skipping %s: %s\n", strType, strErr); - continue; - } - } - Dbt datKey(&row.first[0], row.first.size()); - Dbt datValue(&row.second[0], row.second.size()); - int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); - if (ret2 > 0) - fSuccess = false; - } - ptxn->commit(0); - pdbCopy->close(0); + return true; +} - return fSuccess; +bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +{ + return CDB::VerifyEnvironment(walletFile, dataDir, errorStr); } -bool CWalletDB::Recover(CDBEnv& dbenv, const std::string& filename) +bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr) { - return CWalletDB::Recover(dbenv, filename, false); + return CDB::VerifyDatabaseFile(walletFile, dataDir, errorStr, warningStr, CWalletDB::Recover); } bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("destdata"), std::make_pair(address, key)), value); + return batch.Write(std::make_pair(std::string("destdata"), std::make_pair(address, key)), value); } bool CWalletDB::EraseDestData(const std::string &address, const std::string &key) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); + return batch.Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); } bool CWalletDB::WriteHDChain(const CHDChain& chain) { nWalletDBUpdateCounter++; - return Write(std::string("hdchain"), chain); + return batch.Write(std::string("hdchain"), chain); } void CWalletDB::IncrementUpdateCounter() @@ -961,3 +880,28 @@ unsigned int CWalletDB::GetUpdateCounter() { return nWalletDBUpdateCounter; } + +bool CWalletDB::TxnBegin() +{ + return batch.TxnBegin(); +} + +bool CWalletDB::TxnCommit() +{ + return batch.TxnCommit(); +} + +bool CWalletDB::TxnAbort() +{ + return batch.TxnAbort(); +} + +bool CWalletDB::ReadVersion(int& nVersion) +{ + return batch.ReadVersion(nVersion); +} + +bool CWalletDB::WriteVersion(int nVersion) +{ + return batch.WriteVersion(nVersion); +} diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index c7c65465df..cd9fe279c5 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -17,6 +17,21 @@ #include <utility> #include <vector> +/** + * Overview of wallet database classes: + * + * - CDBEnv is an environment in which the database exists (has no analog in dbwrapper.h) + * - CWalletDBWrapper represents a wallet database (similar to CDBWrapper in dbwrapper.h) + * - CDB is a low-level database transaction (similar to CDBBatch in dbwrapper.h) + * - CWalletDB is a modifier object for the wallet, and encapsulates a database + * transaction as well as methods to act on the database (no analog in + * dbwrapper.h) + * + * The latter two are named confusingly, in contrast to what the names CDB + * and CWalletDB suggest they are transient transaction objects and don't + * represent the database itself. + */ + static const bool DEFAULT_FLUSHWALLET = true; class CAccount; @@ -46,9 +61,12 @@ class CHDChain { public: uint32_t nExternalChainCounter; + uint32_t nInternalChainCounter; CKeyID masterKeyID; //!< master key hash160 - static const int CURRENT_VERSION = 1; + static const int VERSION_HD_BASE = 1; + static const int VERSION_HD_CHAIN_SPLIT = 2; + static const int CURRENT_VERSION = VERSION_HD_CHAIN_SPLIT; int nVersion; CHDChain() { SetNull(); } @@ -59,12 +77,15 @@ public: READWRITE(this->nVersion); READWRITE(nExternalChainCounter); READWRITE(masterKeyID); + if (this->nVersion >= VERSION_HD_CHAIN_SPLIT) + READWRITE(nInternalChainCounter); } void SetNull() { nVersion = CHDChain::CURRENT_VERSION; nExternalChainCounter = 0; + nInternalChainCounter = 0; masterKeyID.SetNull(); } }; @@ -112,11 +133,16 @@ public: } }; -/** Access to the wallet database */ -class CWalletDB : public CDB +/** Access to the wallet database. + * This should really be named CWalletDBBatch, as it represents a single transaction at the + * database. It will be committed when the object goes out of scope. + * Optionally (on by default) it will flush to disk as well. + */ +class CWalletDB { public: - CWalletDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnClose = true) : CDB(strFilename, pszMode, fFlushOnClose) + CWalletDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool _fFlushOnClose = true) : + batch(dbw, pszMode, _fFlushOnClose) { } @@ -167,22 +193,46 @@ public: void ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& acentries); DBErrors LoadWallet(CWallet* pwallet); - DBErrors FindWalletTx(CWallet* pwallet, std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); - DBErrors ZapWalletTx(CWallet* pwallet, std::vector<CWalletTx>& vWtx); - DBErrors ZapSelectTx(CWallet* pwallet, std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut); - static bool Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys); - static bool Recover(CDBEnv& dbenv, const std::string& filename); + DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); + DBErrors ZapWalletTx(std::vector<CWalletTx>& vWtx); + DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut); + /* Try to (very carefully!) recover wallet database (with a possible key type filter) */ + static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)); + /* Recover convenience-function to bypass the key filter callback, called when verify fails, recovers everything */ + static bool Recover(const std::string& filename); + /* Recover filter (used as callback), will only let keys (cryptographical keys) as KV/key-type pass through */ + static bool RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue); + /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ + static bool IsKeyType(const std::string& strType); + /* verifies the database environment */ + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + /* verifies the database file */ + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); static void IncrementUpdateCounter(); static unsigned int GetUpdateCounter(); + + //! Begin a new transaction + bool TxnBegin(); + //! Commit current transaction + bool TxnCommit(); + //! Abort current transaction + bool TxnAbort(); + //! Read wallet version + bool ReadVersion(int& nVersion); + //! Write wallet version + bool WriteVersion(int nVersion); private: + CDB batch; + CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); }; -void ThreadFlushWalletDB(); +//! Compacts BDB state so that wallet.dat is self-contained (if there are changes) +void MaybeCompactWalletDB(); #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 431d8c9ac9..c063898056 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -12,7 +12,7 @@ void zmqError(const char *str) { - LogPrint("zmq", "zmq: Error: %s, errno=%s\n", str, zmq_strerror(errno)); + LogPrint(BCLog::ZMQ, "zmq: Error: %s, errno=%s\n", str, zmq_strerror(errno)); } CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(NULL) @@ -72,7 +72,7 @@ CZMQNotificationInterface* CZMQNotificationInterface::Create() // Called at startup to conditionally set up ZMQ socket(s) bool CZMQNotificationInterface::Initialize() { - LogPrint("zmq", "zmq: Initialize notification interface\n"); + LogPrint(BCLog::ZMQ, "zmq: Initialize notification interface\n"); assert(!pcontext); pcontext = zmq_init(1); @@ -89,11 +89,11 @@ bool CZMQNotificationInterface::Initialize() CZMQAbstractNotifier *notifier = *i; if (notifier->Initialize(pcontext)) { - LogPrint("zmq", " Notifier %s ready (address = %s)\n", notifier->GetType(), notifier->GetAddress()); + LogPrint(BCLog::ZMQ, " Notifier %s ready (address = %s)\n", notifier->GetType(), notifier->GetAddress()); } else { - LogPrint("zmq", " Notifier %s failed (address = %s)\n", notifier->GetType(), notifier->GetAddress()); + LogPrint(BCLog::ZMQ, " Notifier %s failed (address = %s)\n", notifier->GetType(), notifier->GetAddress()); break; } } @@ -109,13 +109,13 @@ bool CZMQNotificationInterface::Initialize() // Called during shutdown sequence void CZMQNotificationInterface::Shutdown() { - LogPrint("zmq", "zmq: Shutdown notification interface\n"); + LogPrint(BCLog::ZMQ, "zmq: Shutdown notification interface\n"); if (pcontext) { for (std::list<CZMQAbstractNotifier*>::iterator i=notifiers.begin(); i!=notifiers.end(); ++i) { CZMQAbstractNotifier *notifier = *i; - LogPrint("zmq", " Shutdown notifier %s at %s\n", notifier->GetType(), notifier->GetAddress()); + LogPrint(BCLog::ZMQ, " Shutdown notifier %s at %s\n", notifier->GetType(), notifier->GetAddress()); notifier->Shutdown(); } zmq_ctx_destroy(pcontext); @@ -144,8 +144,12 @@ void CZMQNotificationInterface::UpdatedBlockTip(const CBlockIndex *pindexNew, co } } -void CZMQNotificationInterface::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int posInBlock) +void CZMQNotificationInterface::TransactionAddedToMempool(const CTransactionRef& ptx) { + // Used by BlockConnected and BlockDisconnected as well, because they're + // all the same external callback. + const CTransaction& tx = *ptx; + for (std::list<CZMQAbstractNotifier*>::iterator i = notifiers.begin(); i!=notifiers.end(); ) { CZMQAbstractNotifier *notifier = *i; @@ -160,3 +164,19 @@ void CZMQNotificationInterface::SyncTransaction(const CTransaction& tx, const CB } } } + +void CZMQNotificationInterface::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) +{ + for (const CTransactionRef& ptx : pblock->vtx) { + // Do a normal notify for each transaction added in the block + TransactionAddedToMempool(ptx); + } +} + +void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) +{ + for (const CTransactionRef& ptx : pblock->vtx) { + // Do a normal notify for each transaction removed in block disconnection + TransactionAddedToMempool(ptx); + } +} diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index beabb78da6..eec6f7bc64 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -8,6 +8,7 @@ #include "validationinterface.h" #include <string> #include <map> +#include <list> class CBlockIndex; class CZMQAbstractNotifier; @@ -24,8 +25,10 @@ protected: void Shutdown(); // CValidationInterface - void SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock); - void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); + void TransactionAddedToMempool(const CTransactionRef& tx) override; + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; private: CZMQNotificationInterface(); diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index caca1248a1..700c39f66e 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "chain.h" #include "chainparams.h" #include "streams.h" #include "zmqpublishnotifier.h" @@ -30,6 +31,7 @@ static int zmq_send_multipart(void *sock, const void* data, size_t size, ...) if (rc != 0) { zmqError("Unable to initialize ZMQ msg"); + va_end(args); return -1; } @@ -43,6 +45,7 @@ static int zmq_send_multipart(void *sock, const void* data, size_t size, ...) { zmqError("Unable to send ZMQ msg"); zmq_msg_close(&msg); + va_end(args); return -1; } @@ -53,6 +56,7 @@ static int zmq_send_multipart(void *sock, const void* data, size_t size, ...) size = va_arg(args, size_t); } + va_end(args); return 0; } @@ -86,7 +90,7 @@ bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) } else { - LogPrint("zmq", "zmq: Reusing socket for address %s\n", address); + LogPrint(BCLog::ZMQ, "zmq: Reusing socket for address %s\n", address); psocket = i->second->psocket; mapPublishNotifiers.insert(std::make_pair(address, this)); @@ -116,7 +120,7 @@ void CZMQAbstractPublishNotifier::Shutdown() if (count == 1) { - LogPrint("zmq", "Close socket at address %s\n", address); + LogPrint(BCLog::ZMQ, "Close socket at address %s\n", address); int linger = 0; zmq_setsockopt(psocket, ZMQ_LINGER, &linger, sizeof(linger)); zmq_close(psocket); @@ -145,7 +149,7 @@ bool CZMQAbstractPublishNotifier::SendMessage(const char *command, const void* d bool CZMQPublishHashBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { uint256 hash = pindex->GetBlockHash(); - LogPrint("zmq", "zmq: Publish hashblock %s\n", hash.GetHex()); + LogPrint(BCLog::ZMQ, "zmq: Publish hashblock %s\n", hash.GetHex()); char data[32]; for (unsigned int i = 0; i < 32; i++) data[31 - i] = hash.begin()[i]; @@ -155,7 +159,7 @@ bool CZMQPublishHashBlockNotifier::NotifyBlock(const CBlockIndex *pindex) bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &transaction) { uint256 hash = transaction.GetHash(); - LogPrint("zmq", "zmq: Publish hashtx %s\n", hash.GetHex()); + LogPrint(BCLog::ZMQ, "zmq: Publish hashtx %s\n", hash.GetHex()); char data[32]; for (unsigned int i = 0; i < 32; i++) data[31 - i] = hash.begin()[i]; @@ -164,7 +168,7 @@ bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &t bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { - LogPrint("zmq", "zmq: Publish rawblock %s\n", pindex->GetBlockHash().GetHex()); + LogPrint(BCLog::ZMQ, "zmq: Publish rawblock %s\n", pindex->GetBlockHash().GetHex()); const Consensus::Params& consensusParams = Params().GetConsensus(); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); @@ -186,7 +190,7 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &transaction) { uint256 hash = transaction.GetHash(); - LogPrint("zmq", "zmq: Publish rawtx %s\n", hash.GetHex()); + LogPrint(BCLog::ZMQ, "zmq: Publish rawtx %s\n", hash.GetHex()); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ss << transaction; return SendMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000000..4dd512638d --- /dev/null +++ b/test/README.md @@ -0,0 +1,99 @@ +This directory contains integration tests that test bitcoind and its +utilities in their entirety. It does not contain unit tests, which +can be found in [/src/test](/src/test), [/src/wallet/test](/src/wallet/test), +etc. + +There are currently two sets of tests in this directory: + +- [functional](/test/functional) which test the functionality of +bitcoind and bitcoin-qt by interacting with them through the RPC and P2P +interfaces. +- [util](test/util) which tests the bitcoin utilities, currently only +bitcoin-tx. + +The util tests are run as part of `make check` target. The functional +tests are run by the travis continuous build process whenever a pull +request is opened. Both sets of tests can also be run locally. + +Functional Test dependencies +============================ +The ZMQ functional test requires a python ZMQ library. To install it: + +- on Unix, run `sudo apt-get install python3-zmq` +- on mac OS, run `pip3 install pyzmq` + +Running tests locally +===================== + +Build for your system first. Be sure to enable wallet, utils and daemon when you configure. Tests will not run otherwise. + +Functional tests +---------------- + +You can run any single test by calling + + test/functional/test_runner.py <testname> + +Or you can run any combination (incl. duplicates) of tests by calling + + test/functional/test_runner.py <testname1> <testname2> <testname3> ... + +Run the regression test suite with + + test/functional/test_runner.py + +Run all possible tests with + + test/functional/test_runner.py --extended + +By default, tests will be run in parallel. To specify how many jobs to run, +append `--jobs=n` (default n=4). + +If you want to create a basic coverage report for the RPC test suite, append `--coverage`. + +Possible options, which apply to each individual test run: + +``` + -h, --help show this help message and exit + --nocleanup Leave bitcoinds and test.* datadir on exit or error + --noshutdown Don't stop bitcoinds after the test execution + --srcdir=SRCDIR Source directory containing bitcoind/bitcoin-cli + (default: ../../src) + --tmpdir=TMPDIR Root directory for datadirs + --tracerpc Print out all RPC calls as they are made + --coveragedir=COVERAGEDIR + Write tested RPC commands into this directory +``` + +If you set the environment variable `PYTHON_DEBUG=1` you will get some debug +output (example: `PYTHON_DEBUG=1 test/functional/test_runner.py wallet`). + +A 200-block -regtest blockchain and wallets for four nodes +is created the first time a regression test is run and +is stored in the cache/ directory. Each node has 25 mature +blocks (25*50=1250 BTC) in its wallet. + +After the first run, the cache/ blockchain and wallets are +copied into a temporary directory and used as the initial +test state. + +If you get into a bad state, you should be able +to recover with: + +```bash +rm -rf cache +killall bitcoind +``` + +Util tests +---------- + +Util tests can be run locally by running `test/util/bitcoin-util-test.py`. +Use the `-v` option for verbose output. + +Writing functional tests +======================== + +You are encouraged to write functional tests for new or existing features. +Further information about the functional test framework and individual +tests is found in [test/functional](/test/functional). diff --git a/qa/rpc-tests/.gitignore b/test/functional/.gitignore index cb41d94423..cb41d94423 100644 --- a/qa/rpc-tests/.gitignore +++ b/test/functional/.gitignore diff --git a/qa/rpc-tests/README.md b/test/functional/README.md index 651b01f18a..e6c4849702 100644 --- a/qa/rpc-tests/README.md +++ b/test/functional/README.md @@ -59,7 +59,7 @@ thread.) * RPC calls are available in p2p tests. * Can be used to write free-form tests, where specific p2p-protocol behavior -is tested. Examples: ```p2p-accept-block.py```, ```maxblocksinflight.py```. +is tested. Examples: ```p2p-accept-block.py```, ```p2p-compactblocks.py```. ## Comptool diff --git a/qa/rpc-tests/abandonconflict.py b/test/functional/abandonconflict.py index b32d4e2ce0..c87c02492d 100755 --- a/qa/rpc-tests/abandonconflict.py +++ b/test/functional/abandonconflict.py @@ -10,22 +10,15 @@ which are not included in a block and are not currently in the mempool. It has no effect on transactions which are already conflicted or abandoned. """ - from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -import urllib.parse class AbandonConflictTest(BitcoinTestFramework): def __init__(self): super().__init__() self.num_nodes = 2 self.setup_clean_chain = False - - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-logtimemicros","-minrelaytxfee=0.00001"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-logtimemicros"])) - connect_nodes(self.nodes[0], 1) + self.extra_args = [["-minrelaytxfee=0.00001"], []] def run_test(self): self.nodes[1].generate(100) @@ -42,8 +35,8 @@ class AbandonConflictTest(BitcoinTestFramework): assert(balance - newbalance < Decimal("0.001")) #no more than fees lost balance = newbalance - url = urllib.parse.urlparse(self.nodes[1].url) - self.nodes[0].disconnectnode(url.hostname+":"+str(p2p_port(1))) + # Disconnect nodes so node0's transactions don't get into node1's mempool + disconnect_nodes(self.nodes[0], 1) # Identify the 10btc outputs nA = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txA, 1)["vout"]) if vout["value"] == Decimal("10")) @@ -80,12 +73,12 @@ class AbandonConflictTest(BitcoinTestFramework): # Restart the node with a higher min relay fee so the parent tx is no longer in mempool # TODO: redo with eviction - # Note had to make sure tx did not have AllowFree priority - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-logtimemicros","-minrelaytxfee=0.0001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) - # Verify txs no longer in mempool + # Verify txs no longer in either node's mempool assert_equal(len(self.nodes[0].getrawmempool()), 0) + assert_equal(len(self.nodes[1].getrawmempool()), 0) # Not in mempool txs from self should only reduce balance # inputs are still spent, but change not received @@ -107,8 +100,8 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-logtimemicros","-minrelaytxfee=0.00001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.00001"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(self.nodes[0].getbalance(), balance) @@ -127,8 +120,8 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Remove using high relay fee again - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-logtimemicros","-minrelaytxfee=0.0001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) newbalance = self.nodes[0].getbalance() assert_equal(newbalance, balance - Decimal("24.9996")) @@ -159,9 +152,9 @@ class AbandonConflictTest(BitcoinTestFramework): self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) newbalance = self.nodes[0].getbalance() #assert_equal(newbalance, balance - Decimal("10")) - print("If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer") - print("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") - print(str(balance) + " -> " + str(newbalance) + " ?") + self.log.info("If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer") + self.log.info("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") + self.log.info(str(balance) + " -> " + str(newbalance) + " ?") if __name__ == '__main__': AbandonConflictTest().main() diff --git a/qa/rpc-tests/assumevalid.py b/test/functional/assumevalid.py index b8dafff161..9d17faac51 100755 --- a/qa/rpc-tests/assumevalid.py +++ b/test/functional/assumevalid.py @@ -15,7 +15,7 @@ transactions: 2-101: bury that block with 100 blocks so the coinbase transaction output can be spent 102: a block containing a transaction spending the coinbase - transaction output. The transaction has an invalid signature. + transaction output. The transaction has an invalid signature. 103-2202: bury the bad block with just over two weeks' worth of blocks (2100 blocks) @@ -29,40 +29,31 @@ Start three nodes: block 200. node2 will reject block 102 since it's assumed valid, but it isn't buried by at least two weeks' work. """ +import time -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * -from test_framework.blocktools import create_block, create_coinbase +from test_framework.blocktools import (create_block, create_coinbase) from test_framework.key import CECKey -from test_framework.script import * - -class BaseNode(SingleNodeConnCB): - def __init__(self): - SingleNodeConnCB.__init__(self) - self.last_inv = None - self.last_headers = None - self.last_block = None - self.last_getdata = None - self.block_announced = False - self.last_getheaders = None - self.disconnected = False - self.last_blockhash_announced = None - - def on_close(self, conn): - self.disconnected = True - - def wait_for_disconnect(self, timeout=60): - test_function = lambda: self.disconnected - assert(wait_until(test_function, timeout=timeout)) - return +from test_framework.mininode import (CBlockHeader, + COutPoint, + CTransaction, + CTxIn, + CTxOut, + NetworkThread, + NodeConn, + NodeConnCB, + msg_block, + msg_headers) +from test_framework.script import (CScript, OP_TRUE) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import (p2p_port, assert_equal) +class BaseNode(NodeConnCB): def send_header_for_blocks(self, new_blocks): headers_message = msg_headers() - headers_message.headers = [ CBlockHeader(b) for b in new_blocks ] + headers_message.headers = [CBlockHeader(b) for b in new_blocks] self.send_message(headers_message) -class SendHeadersTest(BitcoinTestFramework): +class AssumeValidTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True @@ -72,8 +63,34 @@ class SendHeadersTest(BitcoinTestFramework): # Start node0. We don't start the other nodes yet since # we need to pre-mine a block with an invalid transaction # signature so we can pass in the block hash as assumevalid. - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes = [self.start_node(0, self.options.tmpdir)] + + def send_blocks_until_disconnected(self, node): + """Keep sending blocks to the node until we're disconnected.""" + for i in range(len(self.blocks)): + try: + node.send_message(msg_block(self.blocks[i])) + except IOError as e: + assert str(e) == 'Not connected, no pushbuf' + break + + def assert_blockchain_height(self, node, height): + """Wait until the blockchain is no longer advancing and verify it's reached the expected height.""" + last_height = node.getblock(node.getbestblockhash())['height'] + timeout = 10 + while True: + time.sleep(0.25) + current_height = node.getblock(node.getbestblockhash())['height'] + if current_height != last_height: + last_height = current_height + if timeout < 0: + assert False, "blockchain too short after timeout: %d" % current_height + timeout - 0.25 + continue + elif current_height > height: + assert False, "blockchain too long: %d" % current_height + elif current_height == height: + break def run_test(self): @@ -83,7 +100,7 @@ class SendHeadersTest(BitcoinTestFramework): connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) node0.add_connection(connections[0]) - NetworkThread().start() # Start up network handling in another thread + NetworkThread().start() # Start up network handling in another thread node0.wait_for_verack() # Build the blockchain @@ -120,7 +137,7 @@ class SendHeadersTest(BitcoinTestFramework): # Create a transaction spending the coinbase output with an invalid (null) signature tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.block1.vtx[0].sha256, 0), scriptSig=b"")) - tx.vout.append(CTxOut(49*100000000, CScript([OP_TRUE]))) + tx.vout.append(CTxOut(49 * 100000000, CScript([OP_TRUE]))) tx.calc_sha256() block102 = create_block(self.tip, create_coinbase(height), self.block_time) @@ -145,15 +162,15 @@ class SendHeadersTest(BitcoinTestFramework): height += 1 # Start node1 and node2 with assumevalid so they accept a block with a bad signature. - self.nodes.append(start_node(1, self.options.tmpdir, - ["-debug", "-assumevalid=" + hex(block102.sha256)])) + self.nodes.append(self.start_node(1, self.options.tmpdir, + ["-assumevalid=" + hex(block102.sha256)])) node1 = BaseNode() # connects to node1 connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], node1)) node1.add_connection(connections[1]) node1.wait_for_verack() - self.nodes.append(start_node(2, self.options.tmpdir, - ["-debug", "-assumevalid=" + hex(block102.sha256)])) + self.nodes.append(self.start_node(2, self.options.tmpdir, + ["-assumevalid=" + hex(block102.sha256)])) node2 = BaseNode() # connects to node2 connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2)) node2.add_connection(connections[2]) @@ -166,25 +183,20 @@ class SendHeadersTest(BitcoinTestFramework): node1.send_header_for_blocks(self.blocks[2000:]) node2.send_header_for_blocks(self.blocks[0:200]) - # Send 102 blocks to node0. Block 102 will be rejected. - for i in range(101): - node0.send_message(msg_block(self.blocks[i])) - node0.sync_with_ping() # make sure the most recent block is synced - node0.send_message(msg_block(self.blocks[101])) - assert_equal(self.nodes[0].getblock(self.nodes[0].getbestblockhash())['height'], 101) + # Send blocks to node0. Block 102 will be rejected. + self.send_blocks_until_disconnected(node0) + self.assert_blockchain_height(self.nodes[0], 101) - # Send 3102 blocks to node1. All blocks will be accepted. + # Send all blocks to node1. All blocks will be accepted. for i in range(2202): node1.send_message(msg_block(self.blocks[i])) - node1.sync_with_ping() # make sure the most recent block is synced + # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. + node1.sync_with_ping(120) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) - # Send 102 blocks to node2. Block 102 will be rejected. - for i in range(101): - node2.send_message(msg_block(self.blocks[i])) - node2.sync_with_ping() # make sure the most recent block is synced - node2.send_message(msg_block(self.blocks[101])) - assert_equal(self.nodes[2].getblock(self.nodes[2].getbestblockhash())['height'], 101) + # Send blocks to node2. Block 102 will be rejected. + self.send_blocks_until_disconnected(node2) + self.assert_blockchain_height(self.nodes[2], 101) if __name__ == '__main__': - SendHeadersTest().main() + AssumeValidTest().main() diff --git a/qa/rpc-tests/bip65-cltv-p2p.py b/test/functional/bip65-cltv-p2p.py index ea9c3d73ab..bb83042f35 100755 --- a/qa/rpc-tests/bip65-cltv-p2p.py +++ b/test/functional/bip65-cltv-p2p.py @@ -39,12 +39,7 @@ class BIP65Test(ComparisonTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - - def setup_network(self): - # Must set the blockversion for this test - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1', '-blockversion=3']], - binary=[self.options.testbinary]) + self.extra_args = [['-whitelist=127.0.0.1', '-blockversion=3']] def run_test(self): test = TestManager(self, self.options.tmpdir) diff --git a/qa/rpc-tests/bip65-cltv.py b/test/functional/bip65-cltv.py index c9d02a98f3..ddf932c746 100755 --- a/qa/rpc-tests/bip65-cltv.py +++ b/test/functional/bip65-cltv.py @@ -12,15 +12,12 @@ class BIP65Test(BitcoinTestFramework): super().__init__() self.num_nodes = 3 self.setup_clean_chain = False + self.extra_args = [[], ["-blockversion=3"], ["-blockversion=4"]] def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, [])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-blockversion=3"])) - self.nodes.append(start_node(2, self.options.tmpdir, ["-blockversion=4"])) + self.setup_nodes() connect_nodes(self.nodes[1], 0) connect_nodes(self.nodes[2], 0) - self.is_network_split = False self.sync_all() def run_test(self): @@ -69,12 +66,8 @@ class BIP65Test(BitcoinTestFramework): if (self.nodes[0].getblockcount() != cnt + 1051): raise AssertionError("Failed to mine a version=4 block") - # Mine 1 old-version blocks - try: - self.nodes[1].generate(1) - raise AssertionError("Succeeded to mine a version=3 block after 950 version=4 blocks") - except JSONRPCException: - pass + # Mine 1 old-version blocks. This should fail + assert_raises_jsonrpc(-1,"CreateNewBlock: TestBlockValidity failed: bad-version(0x00000003)", self.nodes[1].generate, 1) self.sync_all() if (self.nodes[0].getblockcount() != cnt + 1051): raise AssertionError("Accepted a version=3 block after 950 version=4 blocks") diff --git a/qa/rpc-tests/bip68-112-113-p2p.py b/test/functional/bip68-112-113-p2p.py index 836267ea7f..5a322e8c0e 100755 --- a/qa/rpc-tests/bip68-112-113-p2p.py +++ b/test/functional/bip68-112-113-p2p.py @@ -95,12 +95,7 @@ class BIP68_112_113Test(ComparisonTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - - def setup_network(self): - # Must set the blockversion for this test - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1', '-blockversion=4']], - binary=[self.options.testbinary]) + self.extra_args = [['-whitelist=127.0.0.1', '-blockversion=4']] def run_test(self): test = TestManager(self, self.options.tmpdir) diff --git a/qa/rpc-tests/bip68-sequence.py b/test/functional/bip68-sequence.py index 74ac393fe9..87a50692f6 100755 --- a/qa/rpc-tests/bip68-sequence.py +++ b/test/functional/bip68-sequence.py @@ -21,41 +21,34 @@ class BIP68Test(BitcoinTestFramework): super().__init__() self.num_nodes = 2 self.setup_clean_chain = False + self.extra_args = [[], ["-acceptnonstdtxn=0"]] - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-blockprioritysize=0"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-blockprioritysize=0", "-acceptnonstdtxn=0"])) - self.is_network_split = False + def run_test(self): self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - connect_nodes(self.nodes[0], 1) - def run_test(self): # Generate some coins self.nodes[0].generate(110) - print("Running test disable flag") + self.log.info("Running test disable flag") self.test_disable_flag() - print("Running test sequence-lock-confirmed-inputs") + self.log.info("Running test sequence-lock-confirmed-inputs") self.test_sequence_lock_confirmed_inputs() - print("Running test sequence-lock-unconfirmed-inputs") + self.log.info("Running test sequence-lock-unconfirmed-inputs") self.test_sequence_lock_unconfirmed_inputs() - print("Running test BIP68 not consensus before versionbits activation") + self.log.info("Running test BIP68 not consensus before versionbits activation") self.test_bip68_not_consensus() - print("Verifying nVersion=2 transactions aren't standard") - self.test_version2_relay(before_activation=True) - - print("Activating BIP68 (and 112/113)") + self.log.info("Activating BIP68 (and 112/113)") self.activateCSV() - print("Verifying nVersion=2 transactions are now standard") - self.test_version2_relay(before_activation=False) + self.log.info("Verifying nVersion=2 transactions are standard.") + self.log.info("Note that nVersion=2 transactions are always standard (independent of BIP68 activation status).") + self.test_version2_relay() - print("Passed\n") + self.log.info("Passed") # Test that BIP68 is not in effect if tx version is 1, or if # the first sequence bit is set. @@ -92,12 +85,7 @@ class BIP68Test(BitcoinTestFramework): tx2.vout = [CTxOut(int(value-self.relayfee*COIN), CScript([b'a']))] tx2.rehash() - try: - self.nodes[0].sendrawtransaction(ToHex(tx2)) - except JSONRPCException as exp: - assert_equal(exp.error["message"], NOT_FINAL_ERROR) - else: - assert(False) + assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx2)) # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. @@ -192,14 +180,12 @@ class BIP68Test(BitcoinTestFramework): tx.vout.append(CTxOut(int(value-self.relayfee*tx_size*COIN/1000), CScript([b'a']))) rawtx = self.nodes[0].signrawtransaction(ToHex(tx))["hex"] - try: - self.nodes[0].sendrawtransaction(rawtx) - except JSONRPCException as exp: - assert(not should_pass and using_sequence_locks) - assert_equal(exp.error["message"], NOT_FINAL_ERROR) + if (using_sequence_locks and not should_pass): + # This transaction should be rejected + assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, rawtx) else: - assert(should_pass or not using_sequence_locks) - # Recalculate utxos if we successfully sent the transaction + # This raw transaction should be accepted + self.nodes[0].sendrawtransaction(rawtx) utxos = self.nodes[0].listunspent() # Test that sequence locks on unconfirmed inputs must have nSequence @@ -241,14 +227,13 @@ class BIP68Test(BitcoinTestFramework): tx.vout = [CTxOut(int(orig_tx.vout[0].nValue - relayfee*COIN), CScript([b'a']))] tx.rehash() - try: - node.sendrawtransaction(ToHex(tx)) - except JSONRPCException as exp: - assert_equal(exp.error["message"], NOT_FINAL_ERROR) - assert(orig_tx.hash in node.getrawmempool()) + if (orig_tx.hash in node.getrawmempool()): + # sendrawtransaction should fail if the tx is in the mempool + assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, node.sendrawtransaction, ToHex(tx)) else: - # orig_tx must not be in mempool - assert(orig_tx.hash not in node.getrawmempool()) + # sendrawtransaction should succeed if the tx is not in the mempool + node.sendrawtransaction(ToHex(tx)) + return tx test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=True) @@ -256,7 +241,7 @@ class BIP68Test(BitcoinTestFramework): # Now mine some blocks, but make sure tx2 doesn't get mined. # Use prioritisetransaction to lower the effective feerate to 0 - self.nodes[0].prioritisetransaction(tx2.hash, -1e15, int(-self.relayfee*COIN)) + self.nodes[0].prioritisetransaction(txid=tx2.hash, fee_delta=int(-self.relayfee*COIN)) cur_time = int(time.time()) for i in range(10): self.nodes[0].setmocktime(cur_time + 600) @@ -269,7 +254,7 @@ class BIP68Test(BitcoinTestFramework): test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) # Mine tx2, and then try again - self.nodes[0].prioritisetransaction(tx2.hash, 1e15, int(self.relayfee*COIN)) + self.nodes[0].prioritisetransaction(txid=tx2.hash, fee_delta=int(self.relayfee*COIN)) # Advance the time on the node so that we can test timelocks self.nodes[0].setmocktime(cur_time+600) @@ -297,12 +282,7 @@ class BIP68Test(BitcoinTestFramework): tx5.vout[0].nValue += int(utxos[0]["amount"]*COIN) raw_tx5 = self.nodes[0].signrawtransaction(ToHex(tx5))["hex"] - try: - self.nodes[0].sendrawtransaction(raw_tx5) - except JSONRPCException as exp: - assert_equal(exp.error["message"], NOT_FINAL_ERROR) - else: - assert(False) + assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, raw_tx5) # Test mempool-BIP68 consistency after reorg # @@ -375,12 +355,7 @@ class BIP68Test(BitcoinTestFramework): tx3.vout = [CTxOut(int(tx2.vout[0].nValue - self.relayfee*COIN), CScript([b'a']))] tx3.rehash() - try: - self.nodes[0].sendrawtransaction(ToHex(tx3)) - except JSONRPCException as exp: - assert_equal(exp.error["message"], NOT_FINAL_ERROR) - else: - assert(False) + assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx3)) # make a block that violates bip68; ensure that the tip updates tip = int(self.nodes[0].getbestblockhash(), 16) @@ -398,13 +373,13 @@ class BIP68Test(BitcoinTestFramework): # activation should happen at block height 432 (3 periods) min_activation_height = 432 height = self.nodes[0].getblockcount() - assert(height < 432) - self.nodes[0].generate(432-height) + assert(height < min_activation_height) + self.nodes[0].generate(min_activation_height-height) assert(get_bip9_status(self.nodes[0], 'csv')['status'] == 'active') sync_blocks(self.nodes) - # Use self.nodes[1] to test standardness relay policy - def test_version2_relay(self, before_activation): + # Use self.nodes[1] to test that version 2 transactions are standard. + def test_version2_relay(self): inputs = [ ] outputs = { self.nodes[1].getnewaddress() : 1.0 } rawtx = self.nodes[1].createrawtransaction(inputs, outputs) @@ -412,12 +387,7 @@ class BIP68Test(BitcoinTestFramework): tx = FromHex(CTransaction(), rawtxfund) tx.nVersion = 2 tx_signed = self.nodes[1].signrawtransaction(ToHex(tx))["hex"] - try: - tx_id = self.nodes[1].sendrawtransaction(tx_signed) - assert(before_activation == False) - except: - assert(before_activation) - + tx_id = self.nodes[1].sendrawtransaction(tx_signed) if __name__ == '__main__': BIP68Test().main() diff --git a/qa/rpc-tests/bip9-softforks.py b/test/functional/bip9-softforks.py index 70d4be3f69..b90b0ca628 100755 --- a/qa/rpc-tests/bip9-softforks.py +++ b/test/functional/bip9-softforks.py @@ -16,7 +16,6 @@ test that enforcement has not triggered (which triggers ACTIVE) test that enforcement has triggered """ -from test_framework.blockstore import BlockStore from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * from test_framework.mininode import CTransaction, NetworkThread @@ -32,11 +31,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1']], - binary=[self.options.testbinary]) + self.extra_args = [['-whitelist=127.0.0.1']] def run_test(self): self.test = TestManager(self, self.options.tmpdir) @@ -104,12 +99,43 @@ class BIP9SoftForksTest(ComparisonTestFramework): assert_equal(self.get_bip9_status(bipName)['status'], 'started') assert_equal(self.get_bip9_status(bipName)['since'], 144) + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) assert_equal(tmpl['vbavailable'][bipName], bitno) assert_equal(tmpl['vbrequired'], 0) assert(tmpl['version'] & activated_version) + # Test 1-A + # check stats after max number of "signalling not" blocks such that LOCKED_IN still possible this period + test_blocks = self.generate_blocks(36, 4, test_blocks) # 0x00000004 (signalling not) + test_blocks = self.generate_blocks(10, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 46) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 10) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + + # Test 1-B + # check stats after one additional "signalling not" block -- LOCKED_IN no longer possible this period + test_blocks = self.generate_blocks(1, 4, test_blocks) # 0x00000004 (signalling not) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 47) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 10) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], False) + + # Test 1-C + # finish period with "ready" blocks, but soft fork will still fail to advance to LOCKED_IN + test_blocks = self.generate_blocks(97, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + assert_equal(self.get_bip9_status(bipName)['status'], 'started') + # Test 2 # Fail to achieve LOCKED_IN 100 out of 144 signal bit 1 # using a variety of bits to simulate multiple parallel softforks @@ -121,6 +147,8 @@ class BIP9SoftForksTest(ComparisonTestFramework): assert_equal(self.get_bip9_status(bipName)['status'], 'started') assert_equal(self.get_bip9_status(bipName)['since'], 144) + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) assert_equal(tmpl['vbavailable'][bipName], bitno) @@ -130,14 +158,24 @@ class BIP9SoftForksTest(ComparisonTestFramework): # Test 3 # 108 out of 144 signal bit 1 to achieve LOCKED_IN # using a variety of bits to simulate multiple parallel softforks - test_blocks = self.generate_blocks(58, activated_version) # 0x20000001 (signalling ready) + test_blocks = self.generate_blocks(57, activated_version) # 0x20000001 (signalling ready) test_blocks = self.generate_blocks(26, 4, test_blocks) # 0x00000004 (signalling not) test_blocks = self.generate_blocks(50, activated_version, test_blocks) # 0x20000101 (signalling ready) test_blocks = self.generate_blocks(10, 4, test_blocks) # 0x20010000 (signalling not) yield TestInstance(test_blocks, sync_every_block=False) + # check counting stats and "possible" flag before last block of this period achieves LOCKED_IN... + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 143) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 107) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + assert_equal(self.get_bip9_status(bipName)['status'], 'started') + + # ...continue with Test 3 + test_blocks = self.generate_blocks(1, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + assert_equal(self.get_bip9_status(bipName)['status'], 'locked_in') - assert_equal(self.get_bip9_status(bipName)['since'], 432) + assert_equal(self.get_bip9_status(bipName)['since'], 576) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) @@ -147,7 +185,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): yield TestInstance(test_blocks, sync_every_block=False) assert_equal(self.get_bip9_status(bipName)['status'], 'locked_in') - assert_equal(self.get_bip9_status(bipName)['since'], 432) + assert_equal(self.get_bip9_status(bipName)['since'], 576) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) @@ -173,7 +211,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): yield TestInstance([[block, True]]) assert_equal(self.get_bip9_status(bipName)['status'], 'active') - assert_equal(self.get_bip9_status(bipName)['since'], 576) + assert_equal(self.get_bip9_status(bipName)['since'], 720) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName in tmpl['rules']) assert(bipName not in tmpl['vbavailable']) @@ -200,16 +238,14 @@ class BIP9SoftForksTest(ComparisonTestFramework): yield TestInstance([[block, False]]) # Restart all - self.test.block_store.close() - stop_nodes(self.nodes) - shutil.rmtree(self.options.tmpdir) + self.test.clear_all_connections() + self.stop_nodes() + shutil.rmtree(self.options.tmpdir + "/node0") self.setup_chain() self.setup_network() - self.test.block_store = BlockStore(self.options.tmpdir) - self.test.clear_all_connections() self.test.add_all_connections(self.nodes) - NetworkThread().start() # Start up network handling in another thread - + NetworkThread().start() + self.test.test_nodes[0].wait_for_verack() def get_tests(self): for test in itertools.chain( diff --git a/qa/rpc-tests/bipdersig-p2p.py b/test/functional/bipdersig-p2p.py index b82ef89395..31c7ebba90 100755 --- a/qa/rpc-tests/bipdersig-p2p.py +++ b/test/functional/bipdersig-p2p.py @@ -47,12 +47,6 @@ class BIP66Test(ComparisonTestFramework): super().__init__() self.num_nodes = 1 - def setup_network(self): - # Must set the blockversion for this test - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1', '-blockversion=2']], - binary=[self.options.testbinary]) - def run_test(self): test = TestManager(self, self.options.tmpdir) test.add_all_connections(self.nodes) diff --git a/qa/rpc-tests/bipdersig.py b/test/functional/bipdersig.py index fa54bc2749..41f88fb664 100755 --- a/qa/rpc-tests/bipdersig.py +++ b/test/functional/bipdersig.py @@ -12,15 +12,12 @@ class BIP66Test(BitcoinTestFramework): super().__init__() self.num_nodes = 3 self.setup_clean_chain = False + self.extra_args = [[], ["-blockversion=2"], ["-blockversion=3"]] def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, [])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-blockversion=2"])) - self.nodes.append(start_node(2, self.options.tmpdir, ["-blockversion=3"])) + self.setup_nodes() connect_nodes(self.nodes[1], 0) connect_nodes(self.nodes[2], 0) - self.is_network_split = False self.sync_all() def run_test(self): @@ -68,12 +65,8 @@ class BIP66Test(BitcoinTestFramework): if (self.nodes[0].getblockcount() != cnt + 1051): raise AssertionError("Failed to mine a version=3 block") - # Mine 1 old-version blocks - try: - self.nodes[1].generate(1) - raise AssertionError("Succeeded to mine a version=2 block after 950 version=3 blocks") - except JSONRPCException: - pass + # Mine 1 old-version blocks. This should fail + assert_raises_jsonrpc(-1, "CreateNewBlock: TestBlockValidity failed: bad-version(0x00000002)", self.nodes[1].generate, 1) self.sync_all() if (self.nodes[0].getblockcount() != cnt + 1051): raise AssertionError("Accepted a version=2 block after 950 version=3 blocks") diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py new file mode 100755 index 0000000000..6aef6d4489 --- /dev/null +++ b/test/functional/blockchain.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test RPCs related to blockchainstate. + +Test the following RPCs: + - gettxoutsetinfo + - getdifficulty + - getbestblockhash + - getblockhash + - getblockheader + - getchaintxstats + - getnetworkhashps + - verifychain + +Tests correspond to code in rpc/blockchain.cpp. +""" + +from decimal import Decimal + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_jsonrpc, + assert_is_hex_string, + assert_is_hash_string, +) + + +class BlockchainTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = False + self.num_nodes = 1 + + def run_test(self): + self._test_getchaintxstats() + self._test_gettxoutsetinfo() + self._test_getblockheader() + self._test_getdifficulty() + self._test_getnetworkhashps() + self.nodes[0].verifychain(4, 0) + + def _test_getchaintxstats(self): + chaintxstats = self.nodes[0].getchaintxstats(1) + # 200 txs plus genesis tx + assert_equal(chaintxstats['txcount'], 201) + # tx rate should be 1 per 10 minutes, or 1/600 + # we have to round because of binary math + assert_equal(round(chaintxstats['txrate'] * 600, 10), Decimal(1)) + + def _test_gettxoutsetinfo(self): + node = self.nodes[0] + res = node.gettxoutsetinfo() + + assert_equal(res['total_amount'], Decimal('8725.00000000')) + assert_equal(res['transactions'], 200) + assert_equal(res['height'], 200) + assert_equal(res['txouts'], 200) + assert_equal(res['bogosize'], 17000), + assert_equal(res['bestblock'], node.getblockhash(200)) + size = res['disk_size'] + assert size > 6400 + assert size < 64000 + assert_equal(len(res['bestblock']), 64) + assert_equal(len(res['hash_serialized_2']), 64) + + self.log.info("Test that gettxoutsetinfo() works for blockchain with just the genesis block") + b1hash = node.getblockhash(1) + node.invalidateblock(b1hash) + + res2 = node.gettxoutsetinfo() + assert_equal(res2['transactions'], 0) + assert_equal(res2['total_amount'], Decimal('0')) + assert_equal(res2['height'], 0) + assert_equal(res2['txouts'], 0) + assert_equal(res2['bogosize'], 0), + assert_equal(res2['bestblock'], node.getblockhash(0)) + assert_equal(len(res2['hash_serialized_2']), 64) + + self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") + node.reconsiderblock(b1hash) + + res3 = node.gettxoutsetinfo() + assert_equal(res['total_amount'], res3['total_amount']) + assert_equal(res['transactions'], res3['transactions']) + assert_equal(res['height'], res3['height']) + assert_equal(res['txouts'], res3['txouts']) + assert_equal(res['bogosize'], res3['bogosize']) + assert_equal(res['bestblock'], res3['bestblock']) + assert_equal(res['hash_serialized_2'], res3['hash_serialized_2']) + + def _test_getblockheader(self): + node = self.nodes[0] + + assert_raises_jsonrpc(-5, "Block not found", + node.getblockheader, "nonsense") + + besthash = node.getbestblockhash() + secondbesthash = node.getblockhash(199) + header = node.getblockheader(besthash) + + assert_equal(header['hash'], besthash) + assert_equal(header['height'], 200) + assert_equal(header['confirmations'], 1) + assert_equal(header['previousblockhash'], secondbesthash) + assert_is_hex_string(header['chainwork']) + assert_is_hash_string(header['hash']) + assert_is_hash_string(header['previousblockhash']) + assert_is_hash_string(header['merkleroot']) + assert_is_hash_string(header['bits'], length=None) + assert isinstance(header['time'], int) + assert isinstance(header['mediantime'], int) + assert isinstance(header['nonce'], int) + assert isinstance(header['version'], int) + assert isinstance(int(header['versionHex'], 16), int) + assert isinstance(header['difficulty'], Decimal) + + def _test_getdifficulty(self): + difficulty = self.nodes[0].getdifficulty() + # 1 hash in 2 should be valid, so difficulty should be 1/2**31 + # binary => decimal => binary math is why we do this check + assert abs(difficulty * 2**31 - 1) < 0.0001 + + def _test_getnetworkhashps(self): + hashes_per_second = self.nodes[0].getnetworkhashps() + # This should be 2 hashes every 10 minutes or 1/300 + assert abs(hashes_per_second * 300 - 1) < 0.0001 + +if __name__ == '__main__': + BlockchainTest().main() diff --git a/qa/rpc-tests/bumpfee.py b/test/functional/bumpfee.py index cc897a32c7..d42bab6cbf 100755 --- a/qa/rpc-tests/bumpfee.py +++ b/test/functional/bumpfee.py @@ -2,7 +2,17 @@ # Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the bumpfee RPC.""" +"""Test the bumpfee RPC. + +Verifies that the bumpfee RPC creates replacement transactions successfully when +its preconditions are met, and returns appropriate errors in other cases. + +This module consists of around a dozen individual test cases implemented in the +top-level functions named as test_<test_case_description>. The test functions +can be disabled or reordered if needed for debugging. If new test cases are +added in the the future, they should try to follow the same convention and not +make assumptions about execution order. +""" from segwit import send_to_witness from test_framework.test_framework import BitcoinTestFramework @@ -26,18 +36,17 @@ class BumpFeeTest(BitcoinTestFramework): self.setup_clean_chain = True def setup_network(self, split=False): - extra_args = [["-debug", "-prematurewitness", "-walletprematurewitness", "-walletrbf={}".format(i)] + extra_args = [["-prematurewitness", "-walletprematurewitness", "-walletrbf={}".format(i)] for i in range(self.num_nodes)] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) # Encrypt wallet for test_locked_wallet_fails test self.nodes[1].encryptwallet(WALLET_PASSPHRASE) bitcoind_processes[1].wait() - self.nodes[1] = start_node(1, self.options.tmpdir, extra_args[1]) + self.nodes[1] = self.start_node(1, self.options.tmpdir, extra_args[1]) self.nodes[1].walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) connect_nodes_bi(self.nodes, 0, 1) - self.is_network_split = False self.sync_all() def run_test(self): @@ -45,7 +54,7 @@ class BumpFeeTest(BitcoinTestFramework): rbf_node_address = rbf_node.getnewaddress() # fund rbf node with 10 coins of 0.001 btc (100,000 satoshis) - print("Mining blocks...") + self.log.info("Mining blocks...") peer_node.generate(110) self.sync_all() for i in range(25): @@ -55,30 +64,31 @@ class BumpFeeTest(BitcoinTestFramework): self.sync_all() assert_equal(rbf_node.getbalance(), Decimal("0.025")) - print("Running tests") + self.log.info("Running tests") dest_address = peer_node.getnewaddress() - test_small_output_fails(rbf_node, dest_address) - test_dust_to_fee(rbf_node, dest_address) test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address) test_segwit_bumpfee_succeeds(rbf_node, dest_address) test_nonrbf_bumpfee_fails(peer_node, dest_address) test_notmine_bumpfee_fails(rbf_node, peer_node, dest_address) test_bumpfee_with_descendant_fails(rbf_node, rbf_node_address, dest_address) + test_small_output_fails(rbf_node, dest_address) + test_dust_to_fee(rbf_node, dest_address) test_settxfee(rbf_node, dest_address) test_rebumping(rbf_node, dest_address) test_rebumping_not_replaceable(rbf_node, dest_address) test_unconfirmed_not_spendable(rbf_node, rbf_node_address) test_bumpfee_metadata(rbf_node, dest_address) test_locked_wallet_fails(rbf_node, dest_address) - print("Success") + self.log.info("Success") def test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address): - rbfid = create_fund_sign_send(rbf_node, {dest_address: 0.00090000}) + rbfid = spend_one_input(rbf_node, dest_address) rbftx = rbf_node.gettransaction(rbfid) sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() bumped_tx = rbf_node.bumpfee(rbfid) + assert_equal(bumped_tx["errors"], []) assert bumped_tx["fee"] - abs(rbftx["fee"]) > 0 # check that bumped_tx propogates, original tx was evicted and has a wallet conflict sync_mempools((rbf_node, peer_node)) @@ -102,7 +112,7 @@ def test_segwit_bumpfee_succeeds(rbf_node, dest_address): segwit_out = rbf_node.validateaddress(rbf_node.getnewaddress()) rbf_node.addwitnessaddress(segwit_out["address"]) segwitid = send_to_witness( - version=0, + use_p2wsh=False, node=rbf_node, utxo=segwit_in, pubkey=segwit_out["pubkey"], @@ -115,7 +125,7 @@ def test_segwit_bumpfee_succeeds(rbf_node, dest_address): 'vout': 0, "sequence": BIP125_SEQUENCE_NUMBER }], {dest_address: Decimal("0.0005"), - get_change_address(rbf_node): Decimal("0.0003")}) + rbf_node.getrawchangeaddress(): Decimal("0.0003")}) rbfsigned = rbf_node.signrawtransaction(rbfraw) rbfid = rbf_node.sendrawtransaction(rbfsigned["hex"]) assert rbfid in rbf_node.getrawmempool() @@ -127,8 +137,8 @@ def test_segwit_bumpfee_succeeds(rbf_node, dest_address): def test_nonrbf_bumpfee_fails(peer_node, dest_address): # cannot replace a non RBF transaction (from node which did not enable RBF) - not_rbfid = create_fund_sign_send(peer_node, {dest_address: 0.00090000}) - assert_raises_message(JSONRPCException, "not BIP 125 replaceable", peer_node.bumpfee, not_rbfid) + not_rbfid = peer_node.sendtoaddress(dest_address, Decimal("0.00090000")) + assert_raises_jsonrpc(-4, "not BIP 125 replaceable", peer_node.bumpfee, not_rbfid) def test_notmine_bumpfee_fails(rbf_node, peer_node, dest_address): @@ -148,82 +158,73 @@ def test_notmine_bumpfee_fails(rbf_node, peer_node, dest_address): signedtx = rbf_node.signrawtransaction(rawtx) signedtx = peer_node.signrawtransaction(signedtx["hex"]) rbfid = rbf_node.sendrawtransaction(signedtx["hex"]) - assert_raises_message(JSONRPCException, "Transaction contains inputs that don't belong to this wallet", + assert_raises_jsonrpc(-4, "Transaction contains inputs that don't belong to this wallet", rbf_node.bumpfee, rbfid) def test_bumpfee_with_descendant_fails(rbf_node, rbf_node_address, dest_address): # cannot bump fee if the transaction has a descendant # parent is send-to-self, so we don't have to check which output is change when creating the child tx - parent_id = create_fund_sign_send(rbf_node, {rbf_node_address: 0.00050000}) + parent_id = spend_one_input(rbf_node, rbf_node_address) tx = rbf_node.createrawtransaction([{"txid": parent_id, "vout": 0}], {dest_address: 0.00020000}) tx = rbf_node.signrawtransaction(tx) txid = rbf_node.sendrawtransaction(tx["hex"]) - assert_raises_message(JSONRPCException, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + assert_raises_jsonrpc(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) def test_small_output_fails(rbf_node, dest_address): # cannot bump fee with a too-small output - rbfid = spend_one_input(rbf_node, - Decimal("0.00100000"), - {dest_address: 0.00080000, - get_change_address(rbf_node): Decimal("0.00010000")}) - rbf_node.bumpfee(rbfid, {"totalFee": 20000}) + rbfid = spend_one_input(rbf_node, dest_address) + rbf_node.bumpfee(rbfid, {"totalFee": 50000}) - rbfid = spend_one_input(rbf_node, - Decimal("0.00100000"), - {dest_address: 0.00080000, - get_change_address(rbf_node): Decimal("0.00010000")}) - assert_raises_message(JSONRPCException, "Change output is too small", rbf_node.bumpfee, rbfid, {"totalFee": 20001}) + rbfid = spend_one_input(rbf_node, dest_address) + assert_raises_jsonrpc(-4, "Change output is too small", rbf_node.bumpfee, rbfid, {"totalFee": 50001}) def test_dust_to_fee(rbf_node, dest_address): # check that if output is reduced to dust, it will be converted to fee - # the bumped tx sets fee=9900, but it converts to 10,000 - rbfid = spend_one_input(rbf_node, - Decimal("0.00100000"), - {dest_address: 0.00080000, - get_change_address(rbf_node): Decimal("0.00010000")}) + # the bumped tx sets fee=49,900, but it converts to 50,000 + rbfid = spend_one_input(rbf_node, dest_address) fulltx = rbf_node.getrawtransaction(rbfid, 1) - bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 19900}) + bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 49900}) full_bumped_tx = rbf_node.getrawtransaction(bumped_tx["txid"], 1) - assert_equal(bumped_tx["fee"], Decimal("0.00020000")) + assert_equal(bumped_tx["fee"], Decimal("0.00050000")) assert_equal(len(fulltx["vout"]), 2) assert_equal(len(full_bumped_tx["vout"]), 1) #change output is eliminated def test_settxfee(rbf_node, dest_address): # check that bumpfee reacts correctly to the use of settxfee (paytxfee) - # increase feerate by 2.5x, test that fee increased at least 2x - rbf_node.settxfee(Decimal("0.00001000")) - rbfid = create_fund_sign_send(rbf_node, {dest_address: 0.00090000}) - rbftx = rbf_node.gettransaction(rbfid) - rbf_node.settxfee(Decimal("0.00002500")) + rbfid = spend_one_input(rbf_node, dest_address) + requested_feerate = Decimal("0.00025000") + rbf_node.settxfee(requested_feerate) bumped_tx = rbf_node.bumpfee(rbfid) - assert bumped_tx["fee"] > 2 * abs(rbftx["fee"]) + actual_feerate = bumped_tx["fee"] * 1000 / rbf_node.getrawtransaction(bumped_tx["txid"], True)["size"] + # Assert that the difference between the requested feerate and the actual + # feerate of the bumped transaction is small. + assert_greater_than(Decimal("0.00001000"), abs(requested_feerate - actual_feerate)) rbf_node.settxfee(Decimal("0.00000000")) # unset paytxfee def test_rebumping(rbf_node, dest_address): # check that re-bumping the original tx fails, but bumping the bumper succeeds - rbf_node.settxfee(Decimal("0.00001000")) - rbfid = create_fund_sign_send(rbf_node, {dest_address: 0.00090000}) - bumped = rbf_node.bumpfee(rbfid, {"totalFee": 1000}) - assert_raises_message(JSONRPCException, "already bumped", rbf_node.bumpfee, rbfid, {"totalFee": 2000}) - rbf_node.bumpfee(bumped["txid"], {"totalFee": 2000}) + rbfid = spend_one_input(rbf_node, dest_address) + bumped = rbf_node.bumpfee(rbfid, {"totalFee": 2000}) + assert_raises_jsonrpc(-4, "already bumped", rbf_node.bumpfee, rbfid, {"totalFee": 3000}) + rbf_node.bumpfee(bumped["txid"], {"totalFee": 3000}) def test_rebumping_not_replaceable(rbf_node, dest_address): # check that re-bumping a non-replaceable bump tx fails - rbfid = create_fund_sign_send(rbf_node, {dest_address: 0.00090000}) + rbfid = spend_one_input(rbf_node, dest_address) bumped = rbf_node.bumpfee(rbfid, {"totalFee": 10000, "replaceable": False}) - assert_raises_message(JSONRPCException, "Transaction is not BIP 125 replaceable", rbf_node.bumpfee, bumped["txid"], + assert_raises_jsonrpc(-4, "Transaction is not BIP 125 replaceable", rbf_node.bumpfee, bumped["txid"], {"totalFee": 20000}) def test_unconfirmed_not_spendable(rbf_node, rbf_node_address): # check that unconfirmed outputs from bumped transactions are not spendable - rbfid = create_fund_sign_send(rbf_node, {rbf_node_address: 0.00090000}) + rbfid = spend_one_input(rbf_node, rbf_node_address) rbftx = rbf_node.gettransaction(rbfid)["hex"] assert rbfid in rbf_node.getrawmempool() bumpid = rbf_node.bumpfee(rbfid)["txid"] @@ -258,7 +259,7 @@ def test_unconfirmed_not_spendable(rbf_node, rbf_node_address): def test_bumpfee_metadata(rbf_node, dest_address): - rbfid = rbf_node.sendtoaddress(dest_address, 0.00090000, "comment value", "to value") + rbfid = rbf_node.sendtoaddress(dest_address, Decimal("0.00100000"), "comment value", "to value") bumped_tx = rbf_node.bumpfee(rbfid) bumped_wtx = rbf_node.gettransaction(bumped_tx["txid"]) assert_equal(bumped_wtx["comment"], "comment value") @@ -266,43 +267,23 @@ def test_bumpfee_metadata(rbf_node, dest_address): def test_locked_wallet_fails(rbf_node, dest_address): - rbfid = create_fund_sign_send(rbf_node, {dest_address: 0.00090000}) + rbfid = spend_one_input(rbf_node, dest_address) rbf_node.walletlock() - assert_raises_message(JSONRPCException, "Please enter the wallet passphrase with walletpassphrase first.", + assert_raises_jsonrpc(-13, "Please enter the wallet passphrase with walletpassphrase first.", rbf_node.bumpfee, rbfid) -def create_fund_sign_send(node, outputs): - rawtx = node.createrawtransaction([], outputs) - fundtx = node.fundrawtransaction(rawtx) - signedtx = node.signrawtransaction(fundtx["hex"]) - txid = node.sendrawtransaction(signedtx["hex"]) - return txid - - -def spend_one_input(node, input_amount, outputs): - input = dict(sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in node.listunspent() if u["amount"] == input_amount)) - rawtx = node.createrawtransaction([input], outputs) +def spend_one_input(node, dest_address): + tx_input = dict( + sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in node.listunspent() if u["amount"] == Decimal("0.00100000"))) + rawtx = node.createrawtransaction( + [tx_input], {dest_address: Decimal("0.00050000"), + node.getrawchangeaddress(): Decimal("0.00049000")}) signedtx = node.signrawtransaction(rawtx) txid = node.sendrawtransaction(signedtx["hex"]) return txid -def get_change_address(node): - """Get a wallet change address. - - There is no wallet RPC to access unused change addresses, so this creates a - dummy transaction, calls fundrawtransaction to give add an input and change - output, then returns the change address.""" - dest_address = node.getnewaddress() - dest_amount = Decimal("0.00012345") - rawtx = node.createrawtransaction([], {dest_address: dest_amount}) - fundtx = node.fundrawtransaction(rawtx) - info = node.decoderawtransaction(fundtx["hex"]) - return next(address for out in info["vout"] - if out["value"] != dest_amount for address in out["scriptPubKey"]["addresses"]) - - def submit_block_with_tx(node, tx): ctx = CTransaction() ctx.deserialize(io.BytesIO(hex_str_to_bytes(tx))) @@ -315,9 +296,7 @@ def submit_block_with_tx(node, tx): block.rehash() block.hashMerkleRoot = block.calc_merkle_root() block.solve() - error = node.submitblock(bytes_to_hex_str(block.serialize(True))) - if error is not None: - raise Exception(error) + node.submitblock(bytes_to_hex_str(block.serialize(True))) return block diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py new file mode 100755 index 0000000000..3ca74ea35e --- /dev/null +++ b/test/functional/combine_logs.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Combine logs from multiple bitcoin nodes as well as the test_framework log. + +This streams the combined log output to stdout. Use combine_logs.py > outputfile +to write to an outputfile.""" + +import argparse +from collections import defaultdict, namedtuple +import heapq +import itertools +import os +import re +import sys + +# Matches on the date format at the start of the log event +TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}") + +LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event']) + +def main(): + """Main function. Parses args, reads the log files and renders them as text or html.""" + + parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__) + parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)') + parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2') + args, unknown_args = parser.parse_known_args() + + if args.color and os.name != 'posix': + print("Color output requires posix terminal colors.") + sys.exit(1) + + if args.html and args.color: + print("Only one out of --color or --html should be specified") + sys.exit(1) + + # There should only be one unknown argument - the path of the temporary test directory + if len(unknown_args) != 1: + print("Unexpected arguments" + str(unknown_args)) + sys.exit(1) + + log_events = read_logs(unknown_args[0]) + + print_logs(log_events, color=args.color, html=args.html) + +def read_logs(tmp_dir): + """Reads log files. + + Delegates to generator function get_log_events() to provide individual log events + for each of the input log files.""" + + files = [("test", "%s/test_framework.log" % tmp_dir)] + for i in itertools.count(): + logfile = "{}/node{}/regtest/debug.log".format(tmp_dir, i) + if not os.path.isfile(logfile): + break + files.append(("node%d" % i, logfile)) + + return heapq.merge(*[get_log_events(source, f) for source, f in files]) + +def get_log_events(source, logfile): + """Generator function that returns individual log events. + + Log events may be split over multiple lines. We use the timestamp + regex match as the marker for a new log event.""" + try: + with open(logfile, 'r') as infile: + event = '' + timestamp = '' + for line in infile: + # skip blank lines + if line == '\n': + continue + # if this line has a timestamp, it's the start of a new log event. + time_match = TIMESTAMP_PATTERN.match(line) + if time_match: + if event: + yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) + event = line + timestamp = time_match.group() + # if it doesn't have a timestamp, it's a continuation line of the previous log. + else: + event += "\n" + line + # Flush the final event + yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) + except FileNotFoundError: + print("File %s could not be opened. Continuing without it." % logfile, file=sys.stderr) + +def print_logs(log_events, color=False, html=False): + """Renders the iterator of log events into text or html.""" + if not html: + colors = defaultdict(lambda: '') + if color: + colors["test"] = "\033[0;36m" # CYAN + colors["node0"] = "\033[0;34m" # BLUE + colors["node1"] = "\033[0;32m" # GREEN + colors["node2"] = "\033[0;31m" # RED + colors["node3"] = "\033[0;33m" # YELLOW + colors["reset"] = "\033[0m" # Reset font color + + for event in log_events: + print("{0} {1: <5} {2} {3}".format(colors[event.source.rstrip()], event.source, event.event, colors["reset"])) + + else: + try: + import jinja2 + except ImportError: + print("jinja2 not found. Try `pip install jinja2`") + sys.exit(1) + print(jinja2.Environment(loader=jinja2.FileSystemLoader('./')) + .get_template('combined_log_template.html') + .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events])) + +if __name__ == '__main__': + main() diff --git a/test/functional/combined_log_template.html b/test/functional/combined_log_template.html new file mode 100644 index 0000000000..c0b854b080 --- /dev/null +++ b/test/functional/combined_log_template.html @@ -0,0 +1,40 @@ +<html lang="en"> +<head> + <title> {{ title }} </title> + <style> + ul { + list-style-type: none; + font-family: monospace; + } + li { + border: 1px solid slategray; + margin-bottom: 1px; + } + li:hover { + filter: brightness(85%); + } + li.log-test { + background-color: cyan; + } + li.log-node0 { + background-color: lightblue; + } + li.log-node1 { + background-color: lightgreen; + } + li.log-node2 { + background-color: lightsalmon; + } + li.log-node3 { + background-color: lightyellow; + } + </style> +</head> +<body> +<ul> +{% for event in log_events %} +<li class="log-{{ event.source }}"> {{ event.source }} {{ event.timestamp }} {{event.event}}</li> +{% endfor %} +</ul> +</body> +</html> diff --git a/qa/pull-tester/tests_config.ini.in b/test/functional/config.ini.in index e3e457d0b1..29586c555d 100644 --- a/qa/pull-tester/tests_config.ini.in +++ b/test/functional/config.ini.in @@ -3,7 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # These environment variables are set by the build process and read by -# rpc-tests.py +# test/functional/test_runner.py [environment] SRCDIR=@abs_top_srcdir@ diff --git a/qa/rpc-tests/create_cache.py b/test/functional/create_cache.py index 13eac92bbc..39c4c0f47e 100755 --- a/qa/rpc-tests/create_cache.py +++ b/test/functional/create_cache.py @@ -5,7 +5,7 @@ """Create a blockchain cache. Creating a cache of the blockchain speeds up test execution when running -multiple qa tests. This helper script is executed by rpc-tests when multiple +multiple functional tests. This helper script is executed by test_runner when multiple tests are being run in parallel. """ diff --git a/qa/rpc-tests/decodescript.py b/test/functional/decodescript.py index 5555e96c44..21a9f1223f 100755 --- a/qa/rpc-tests/decodescript.py +++ b/test/functional/decodescript.py @@ -16,10 +16,6 @@ class DecodeScriptTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - self.is_network_split = False - def decodescript_script_sig(self): signature = '304502207fa7a6d1e0ee81132a269ad84e68d695483745cde8b541e3bf630749894e342a022100c1f7ab20e13e22fb95281a870f3dcf38d782e53023ee313d741ad0cfbc0c509001' push_signature = '48' + signature diff --git a/qa/rpc-tests/disablewallet.py b/test/functional/disablewallet.py index ff7121659b..d344513414 100755 --- a/qa/rpc-tests/disablewallet.py +++ b/test/functional/disablewallet.py @@ -18,31 +18,20 @@ class DisableWalletTest (BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 - - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [['-disablewallet']]) - self.is_network_split = False - self.sync_all() + self.extra_args = [["-disablewallet"]] def run_test (self): + # Make sure wallet is really disabled + assert_raises_jsonrpc(-32601, 'Method not found', self.nodes[0].getwalletinfo) x = self.nodes[0].validateaddress('3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') assert(x['isvalid'] == False) x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') assert(x['isvalid'] == True) - # Checking mining to an address without a wallet - try: - self.nodes[0].generatetoaddress(1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - except JSONRPCException as e: - assert("Invalid address" not in e.error['message']) - assert("ProcessNewBlock, block not accepted" not in e.error['message']) - assert("Couldn't create new block" not in e.error['message']) - - try: - self.nodes[0].generatetoaddress(1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') - raise AssertionError("Must not mine to invalid address!") - except JSONRPCException as e: - assert("Invalid address" in e.error['message']) + # Checking mining to an address without a wallet. Generating to a valid address should succeed + # but generating to an invalid address will fail. + self.nodes[0].generatetoaddress(1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') + assert_raises_jsonrpc(-5, "Invalid address", self.nodes[0].generatetoaddress, 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') if __name__ == '__main__': DisableWalletTest ().main () diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py new file mode 100755 index 0000000000..89b68aeb25 --- /dev/null +++ b/test/functional/disconnect_ban.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test node disconnect and ban behavior""" +import time + +from test_framework.mininode import wait_until +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import (assert_equal, + assert_raises_jsonrpc, + connect_nodes_bi) + +class DisconnectBanTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False + + def run_test(self): + self.log.info("Test setban and listbanned RPCs") + + self.log.info("setban: successfully ban single IP address") + assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point + self.nodes[1].setban("127.0.0.1", "add") + assert wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) + assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point + assert_equal(len(self.nodes[1].listbanned()), 1) + + self.log.info("clearbanned: successfully clear ban list") + self.nodes[1].clearbanned() + assert_equal(len(self.nodes[1].listbanned()), 0) + self.nodes[1].setban("127.0.0.0/24", "add") + + self.log.info("setban: fail to ban an already banned subnet") + assert_equal(len(self.nodes[1].listbanned()), 1) + assert_raises_jsonrpc(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") + + self.log.info("setban: fail to ban an invalid subnet") + assert_raises_jsonrpc(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") + assert_equal(len(self.nodes[1].listbanned()), 1) # still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 + + self.log.info("setban remove: fail to unban a non-banned subnet") + assert_raises_jsonrpc(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") + assert_equal(len(self.nodes[1].listbanned()), 1) + + self.log.info("setban remove: successfully unban subnet") + self.nodes[1].setban("127.0.0.0/24", "remove") + assert_equal(len(self.nodes[1].listbanned()), 0) + self.nodes[1].clearbanned() + assert_equal(len(self.nodes[1].listbanned()), 0) + + self.log.info("setban: test persistence across node restart") + self.nodes[1].setban("127.0.0.0/32", "add") + self.nodes[1].setban("127.0.0.0/24", "add") + # Set the mocktime so we can control when bans expire + old_time = int(time.time()) + self.nodes[1].setmocktime(old_time) + self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1 seconds + self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds + listBeforeShutdown = self.nodes[1].listbanned() + assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) + # Move time forward by 3 seconds so the third ban has expired + self.nodes[1].setmocktime(old_time + 3) + assert_equal(len(self.nodes[1].listbanned()), 3) + + self.stop_node(1) + + self.nodes[1] = self.start_node(1, self.options.tmpdir) + listAfterShutdown = self.nodes[1].listbanned() + assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) + assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) + assert_equal("/19" in listAfterShutdown[2]['address'], True) + + # Clear ban lists + self.nodes[1].clearbanned() + connect_nodes_bi(self.nodes, 0, 1) + + self.log.info("Test disconnectnode RPCs") + + self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") + address1 = self.nodes[0].getpeerinfo()[0]['addr'] + node1 = self.nodes[0].getpeerinfo()[0]['addr'] + assert_raises_jsonrpc(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) + + self.log.info("disconnectnode: fail to disconnect when calling with junk address") + assert_raises_jsonrpc(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") + + self.log.info("disconnectnode: successfully disconnect node by address") + address1 = self.nodes[0].getpeerinfo()[0]['addr'] + self.nodes[0].disconnectnode(address=address1) + assert wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] + + self.log.info("disconnectnode: successfully reconnect node") + connect_nodes_bi(self.nodes, 0, 1) # reconnect the node + assert_equal(len(self.nodes[0].getpeerinfo()), 2) + assert [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] + + self.log.info("disconnectnode: successfully disconnect node by node id") + id1 = self.nodes[0].getpeerinfo()[0]['id'] + self.nodes[0].disconnectnode(nodeid=id1) + assert wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] + +if __name__ == '__main__': + DisconnectBanTest().main() diff --git a/qa/rpc-tests/forknotify.py b/test/functional/forknotify.py index c2724ba5df..3bcf0a6795 100755 --- a/qa/rpc-tests/forknotify.py +++ b/test/functional/forknotify.py @@ -3,6 +3,8 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -alertnotify option.""" +import os +import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * @@ -14,21 +16,18 @@ class ForkNotifyTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = False - alert_filename = None # Set by setup_network - def setup_network(self): self.nodes = [] self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") with open(self.alert_filename, 'w', encoding='utf8'): pass # Just open then close to create zero-length file - self.nodes.append(start_node(0, self.options.tmpdir, + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-blockversion=2", "-alertnotify=echo %s >> \"" + self.alert_filename + "\""])) # Node1 mines block.version=211 blocks - self.nodes.append(start_node(1, self.options.tmpdir, + self.nodes.append(self.start_node(1, self.options.tmpdir, ["-blockversion=211"])) connect_nodes(self.nodes[1], 0) - self.is_network_split = False self.sync_all() def run_test(self): @@ -41,12 +40,19 @@ class ForkNotifyTest(BitcoinTestFramework): self.nodes[1].generate(1) self.sync_all() + # Give bitcoind 10 seconds to write the alert notification + timeout = 10.0 + while timeout > 0: + if os.path.exists(self.alert_filename) and os.path.getsize(self.alert_filename): + break + time.sleep(0.1) + timeout -= 0.1 + else: + assert False, "-alertnotify did not warn of up-version blocks" + with open(self.alert_filename, 'r', encoding='utf8') as f: alert_text = f.read() - if len(alert_text) == 0: - raise AssertionError("-alertnotify did not warn of up-version blocks") - # Mine more up-version blocks, should not get more alerts: self.nodes[1].generate(1) self.sync_all() diff --git a/qa/rpc-tests/fundrawtransaction.py b/test/functional/fundrawtransaction.py index 7892e85e22..0a3166b89b 100755 --- a/qa/rpc-tests/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -23,19 +23,14 @@ class RawTransactionsTest(BitcoinTestFramework): self.num_nodes = 4 def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.setup_nodes() - connect_nodes_bi(self.nodes,0,1) - connect_nodes_bi(self.nodes,1,2) - connect_nodes_bi(self.nodes,0,2) - connect_nodes_bi(self.nodes,0,3) - - self.is_network_split=False - self.sync_all() + connect_nodes_bi(self.nodes, 0, 1) + connect_nodes_bi(self.nodes, 1, 2) + connect_nodes_bi(self.nodes, 0, 2) + connect_nodes_bi(self.nodes, 0, 3) def run_test(self): - print("Mining blocks...") - min_relay_tx_fee = self.nodes[0].getnetworkinfo()['relayfee'] # This test is not meant to test fee estimation and we'd like # to be sure all txs are sent at a consistent desired feerate @@ -55,6 +50,11 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(121) self.sync_all() + # ensure that setting changePosition in fundraw with an exact match is handled properly + rawmatch = self.nodes[2].createrawtransaction([], {self.nodes[2].getnewaddress():50}) + rawmatch = self.nodes[2].fundrawtransaction(rawmatch, {"changePosition":1, "subtractFeeFromOutputs":[0]}) + assert_equal(rawmatch["changepos"], -1) + watchonly_address = self.nodes[0].getnewaddress() watchonly_pubkey = self.nodes[0].validateaddress(watchonly_address)["pubkey"] watchonly_amount = Decimal(200) @@ -182,12 +182,7 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - try: - self.nodes[2].fundrawtransaction(rawtx, {'foo': 'bar'}) - raise AssertionError("Accepted invalid option foo") - except JSONRPCException as e: - assert("Unexpected key foo" in e.error['message']) - + assert_raises_jsonrpc(-3, "Unexpected key foo", self.nodes[2].fundrawtransaction, rawtx, {'foo':'bar'}) ############################################################ # test a fundrawtransaction with an invalid change address # @@ -200,12 +195,7 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - try: - self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': 'foobar'}) - raise AssertionError("Accepted invalid bitcoin address") - except JSONRPCException as e: - assert("changeAddress must be a valid bitcoin address" in e.error['message']) - + assert_raises_jsonrpc(-5, "changeAddress must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) ############################################################ # test a fundrawtransaction with a provided change address # @@ -219,12 +209,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) change = self.nodes[2].getnewaddress() - try: - rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': change, 'changePosition': 2}) - except JSONRPCException as e: - assert('changePosition out of bounds' == e.error['message']) - else: - assert(False) + assert_raises_jsonrpc(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':change, 'changePosition':2}) rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': change, 'changePosition': 0}) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) out = dec_tx['vout'][0] @@ -333,19 +318,14 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[2].createrawtransaction(inputs, outputs) dec_tx = self.nodes[2].decoderawtransaction(rawtx) - try: - rawtxfund = self.nodes[2].fundrawtransaction(rawtx) - raise AssertionError("Spent more than available") - except JSONRPCException as e: - assert("Insufficient" in e.error['message']) - + assert_raises_jsonrpc(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx) ############################################################ #compare fee of a standard pubkeyhash transaction inputs = [] outputs = {self.nodes[1].getnewaddress():1.1} - rawTx = self.nodes[0].createrawtransaction(inputs, outputs) - fundedTx = self.nodes[0].fundrawtransaction(rawTx) + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + fundedTx = self.nodes[0].fundrawtransaction(rawtx) #create same transaction over sendtoaddress txId = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1.1) @@ -360,8 +340,8 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee of a standard pubkeyhash transaction with multiple outputs inputs = [] outputs = {self.nodes[1].getnewaddress():1.1,self.nodes[1].getnewaddress():1.2,self.nodes[1].getnewaddress():0.1,self.nodes[1].getnewaddress():1.3,self.nodes[1].getnewaddress():0.2,self.nodes[1].getnewaddress():0.3} - rawTx = self.nodes[0].createrawtransaction(inputs, outputs) - fundedTx = self.nodes[0].fundrawtransaction(rawTx) + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + fundedTx = self.nodes[0].fundrawtransaction(rawtx) #create same transaction over sendtoaddress txId = self.nodes[0].sendmany("", outputs) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] @@ -386,8 +366,8 @@ class RawTransactionsTest(BitcoinTestFramework): inputs = [] outputs = {mSigObj:1.1} - rawTx = self.nodes[0].createrawtransaction(inputs, outputs) - fundedTx = self.nodes[0].fundrawtransaction(rawTx) + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + fundedTx = self.nodes[0].fundrawtransaction(rawtx) #create same transaction over sendtoaddress txId = self.nodes[0].sendtoaddress(mSigObj, 1.1) @@ -419,8 +399,8 @@ class RawTransactionsTest(BitcoinTestFramework): inputs = [] outputs = {mSigObj:1.1} - rawTx = self.nodes[0].createrawtransaction(inputs, outputs) - fundedTx = self.nodes[0].fundrawtransaction(rawTx) + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + fundedTx = self.nodes[0].fundrawtransaction(rawtx) #create same transaction over sendtoaddress txId = self.nodes[0].sendtoaddress(mSigObj, 1.1) @@ -454,8 +434,8 @@ class RawTransactionsTest(BitcoinTestFramework): oldBalance = self.nodes[1].getbalance() inputs = [] outputs = {self.nodes[1].getnewaddress():1.1} - rawTx = self.nodes[2].createrawtransaction(inputs, outputs) - fundedTx = self.nodes[2].fundrawtransaction(rawTx) + rawtx = self.nodes[2].createrawtransaction(inputs, outputs) + fundedTx = self.nodes[2].fundrawtransaction(rawtx) signedTx = self.nodes[2].signrawtransaction(fundedTx['hex']) txId = self.nodes[2].sendrawtransaction(signedTx['hex']) @@ -468,13 +448,13 @@ class RawTransactionsTest(BitcoinTestFramework): ############################################################ # locked wallet test + self.stop_node(0) + self.stop_node(2) + self.stop_node(3) self.nodes[1].encryptwallet("test") - self.nodes.pop(1) - stop_node(self.nodes[0], 0) - stop_node(self.nodes[1], 2) - stop_node(self.nodes[2], 3) + bitcoind_processes[1].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) # This test is not meant to test fee estimation and we'd like # to be sure all txs are sent at a consistent desired feerate for node in self.nodes: @@ -484,38 +464,31 @@ class RawTransactionsTest(BitcoinTestFramework): connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) connect_nodes_bi(self.nodes,0,3) - self.is_network_split=False self.sync_all() # drain the keypool self.nodes[1].getnewaddress() + self.nodes[1].getrawchangeaddress() inputs = [] outputs = {self.nodes[0].getnewaddress():1.1} - rawTx = self.nodes[1].createrawtransaction(inputs, outputs) + rawtx = self.nodes[1].createrawtransaction(inputs, outputs) # fund a transaction that requires a new key for the change output # creating the key must be impossible because the wallet is locked - try: - fundedTx = self.nodes[1].fundrawtransaction(rawTx) - raise AssertionError("Wallet unlocked without passphrase") - except JSONRPCException as e: - assert('Keypool ran out' in e.error['message']) + assert_raises_jsonrpc(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) #refill the keypool self.nodes[1].walletpassphrase("test", 100) + self.nodes[1].keypoolrefill(8) #need to refill the keypool to get an internal change address self.nodes[1].walletlock() - try: - self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.2) - raise AssertionError("Wallet unlocked without passphrase") - except JSONRPCException as e: - assert('walletpassphrase' in e.error['message']) + assert_raises_jsonrpc(-13, "walletpassphrase", self.nodes[1].sendtoaddress, self.nodes[0].getnewaddress(), 1.2) oldBalance = self.nodes[0].getbalance() inputs = [] outputs = {self.nodes[0].getnewaddress():1.1} - rawTx = self.nodes[1].createrawtransaction(inputs, outputs) - fundedTx = self.nodes[1].fundrawtransaction(rawTx) + rawtx = self.nodes[1].createrawtransaction(inputs, outputs) + fundedTx = self.nodes[1].fundrawtransaction(rawtx) #now we need to unlock self.nodes[1].walletpassphrase("test", 600) @@ -546,8 +519,8 @@ class RawTransactionsTest(BitcoinTestFramework): #fund a tx with ~20 small inputs inputs = [] outputs = {self.nodes[0].getnewaddress():0.15,self.nodes[0].getnewaddress():0.04} - rawTx = self.nodes[1].createrawtransaction(inputs, outputs) - fundedTx = self.nodes[1].fundrawtransaction(rawTx) + rawtx = self.nodes[1].createrawtransaction(inputs, outputs) + fundedTx = self.nodes[1].fundrawtransaction(rawtx) #create same transaction over sendtoaddress txId = self.nodes[1].sendmany("", outputs) @@ -578,8 +551,8 @@ class RawTransactionsTest(BitcoinTestFramework): inputs = [] outputs = {self.nodes[0].getnewaddress():0.15,self.nodes[0].getnewaddress():0.04} - rawTx = self.nodes[1].createrawtransaction(inputs, outputs) - fundedTx = self.nodes[1].fundrawtransaction(rawTx) + rawtx = self.nodes[1].createrawtransaction(inputs, outputs) + fundedTx = self.nodes[1].fundrawtransaction(rawtx) fundedAndSignedTx = self.nodes[1].signrawtransaction(fundedTx['hex']) txId = self.nodes[1].sendrawtransaction(fundedAndSignedTx['hex']) self.sync_all() @@ -674,7 +647,7 @@ class RawTransactionsTest(BitcoinTestFramework): if out['value'] > 1.0: changeaddress += out['scriptPubKey']['addresses'][0] assert(changeaddress != "") - nextaddr = self.nodes[3].getnewaddress() + nextaddr = self.nodes[3].getrawchangeaddress() # frt should not have removed the key from the keypool assert(changeaddress == nextaddr) @@ -721,7 +694,6 @@ class RawTransactionsTest(BitcoinTestFramework): inputs = [] outputs = {self.nodes[2].getnewaddress(): value for value in (1.0, 1.1, 1.2, 1.3)} - keys = list(outputs.keys()) rawtx = self.nodes[3].createrawtransaction(inputs, outputs) result = [self.nodes[3].fundrawtransaction(rawtx), diff --git a/qa/rpc-tests/getblocktemplate_longpoll.py b/test/functional/getblocktemplate_longpoll.py index b73fea0695..bbe1dda5f7 100755 --- a/qa/rpc-tests/getblocktemplate_longpoll.py +++ b/test/functional/getblocktemplate_longpoll.py @@ -29,7 +29,7 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): self.setup_clean_chain = False def run_test(self): - print("Warning: this test will take about 70 seconds in the best case. Be patient.") + self.log.info("Warning: this test will take about 70 seconds in the best case. Be patient.") self.nodes[0].generate(10) templat = self.nodes[0].getblocktemplate() longpollid = templat['longpollid'] @@ -61,7 +61,9 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): thr = LongpollThread(self.nodes[0]) thr.start() # generate a random transaction and submit it - (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), Decimal("0.0"), Decimal("0.001"), 20) + min_relay_fee = self.nodes[0].getnetworkinfo()["relayfee"] + # min_relay_fee is fee per 1000 bytes, which should be more than enough. + (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), min_relay_fee, Decimal("0.001"), 20) # after one minute, every 10 seconds the mempool is probed, so in 80 seconds it should have returned thr.join(60 + 20) assert(not thr.is_alive()) diff --git a/qa/rpc-tests/getblocktemplate_proposals.py b/test/functional/getblocktemplate_proposals.py index 3b17dfcfb5..fca99c7df5 100755 --- a/qa/rpc-tests/getblocktemplate_proposals.py +++ b/test/functional/getblocktemplate_proposals.py @@ -73,10 +73,6 @@ class GetBlockTemplateProposalTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = False - def setup_network(self): - self.nodes = self.setup_nodes() - connect_nodes_bi(self.nodes, 0, 1) - def run_test(self): node = self.nodes[0] node.generate(1) # Mine a block to leave initial block download @@ -105,7 +101,7 @@ class GetBlockTemplateProposalTest(BitcoinTestFramework): # Test 3: Truncated final tx lastbyte = txlist[-1].pop() - assert_raises(JSONRPCException, assert_template, node, tmpl, txlist, 'n/a') + assert_raises_jsonrpc(-22, "Block decode failed", assert_template, node, tmpl, txlist, 'n/a') txlist[-1].append(lastbyte) # Test 4: Add an invalid tx to the end (duplicate of gen tx) @@ -126,7 +122,7 @@ class GetBlockTemplateProposalTest(BitcoinTestFramework): # Test 7: Bad tx count txlist.append(b'') - assert_raises(JSONRPCException, assert_template, node, tmpl, txlist, 'n/a') + assert_raises_jsonrpc(-22, 'Block decode failed', assert_template, node, tmpl, txlist, 'n/a') txlist.pop() # Test 8: Bad bits diff --git a/qa/rpc-tests/getchaintips.py b/test/functional/getchaintips.py index 14222334a6..15f96c565f 100755 --- a/qa/rpc-tests/getchaintips.py +++ b/test/functional/getchaintips.py @@ -31,7 +31,7 @@ class GetChainTipsTest (BitcoinTestFramework): self.split_network () self.nodes[0].generate(10) self.nodes[2].generate(20) - self.sync_all () + self.sync_all([self.nodes[:2], self.nodes[2:]]) tips = self.nodes[1].getchaintips () assert_equal (len (tips), 1) diff --git a/qa/rpc-tests/httpbasics.py b/test/functional/httpbasics.py index 8f35f0ab87..4b32e8d9ca 100755 --- a/qa/rpc-tests/httpbasics.py +++ b/test/functional/httpbasics.py @@ -17,7 +17,7 @@ class HTTPBasicsTest (BitcoinTestFramework): self.setup_clean_chain = False def setup_network(self): - self.nodes = self.setup_nodes() + self.setup_nodes() def run_test(self): diff --git a/qa/rpc-tests/import-rescan.py b/test/functional/import-rescan.py index 7ca61824c9..4fc5078217 100755 --- a/qa/rpc-tests/import-rescan.py +++ b/test/functional/import-rescan.py @@ -7,11 +7,11 @@ Test rescan behavior of importaddress, importpubkey, importprivkey, and importmulti RPCs with different types of keys and rescan options. -In the first part of the test, node 1 creates an address for each type of -import RPC call and node 0 sends BTC to it. Then other nodes import the -addresses, and the test makes listtransactions and getbalance calls to confirm -that the importing node either did or did not execute rescans picking up the -send transactions. +In the first part of the test, node 0 creates an address for each type of +import RPC call and sends BTC to it. Then other nodes import the addresses, +and the test makes listtransactions and getbalance calls to confirm that the +importing node either did or did not execute rescans picking up the send +transactions. In the second part of the test, node 0 sends more BTC to each address, and the test makes more listtransactions and getbalance calls to confirm that the @@ -21,8 +21,7 @@ happened previously. from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_nodes, connect_nodes, sync_blocks, assert_equal, set_node_times) -from decimal import Decimal +from test_framework.util import (connect_nodes, sync_blocks, assert_equal, set_node_times) import collections import enum @@ -56,7 +55,7 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")): "scriptPubKey": { "address": self.address["address"] }, - "timestamp": timestamp + RESCAN_WINDOW + (1 if self.rescan == Rescan.late_timestamp else 0), + "timestamp": timestamp + TIMESTAMP_WINDOW + (1 if self.rescan == Rescan.late_timestamp else 0), "pubkeys": [self.address["pubkey"]] if self.data == Data.pub else [], "keys": [self.key] if self.data == Data.priv else [], "label": self.label, @@ -108,7 +107,7 @@ ImportNode = collections.namedtuple("ImportNode", "prune rescan") IMPORT_NODES = [ImportNode(*fields) for fields in itertools.product((False, True), repeat=2)] # Rescans start at the earliest block up to 2 hours before the key timestamp. -RESCAN_WINDOW = 2 * 60 * 60 +TIMESTAMP_WINDOW = 2 * 60 * 60 class ImportRescanTest(BitcoinTestFramework): @@ -117,12 +116,12 @@ class ImportRescanTest(BitcoinTestFramework): self.num_nodes = 2 + len(IMPORT_NODES) def setup_network(self): - extra_args = [["-debug=1"] for _ in range(self.num_nodes)] + extra_args = [[] for _ in range(self.num_nodes)] for i, import_node in enumerate(IMPORT_NODES, 2): if import_node.prune: extra_args[i] += ["-prune=1"] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) for i in range(1, self.num_nodes): connect_nodes(self.nodes[i], 0) @@ -141,7 +140,7 @@ class ImportRescanTest(BitcoinTestFramework): self.nodes[0].generate(1) assert_equal(self.nodes[0].getrawmempool(), []) timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] - set_node_times(self.nodes, timestamp + RESCAN_WINDOW + 1) + set_node_times(self.nodes, timestamp + TIMESTAMP_WINDOW + 1) self.nodes[0].generate(1) sync_blocks(self.nodes) diff --git a/qa/rpc-tests/importmulti.py b/test/functional/importmulti.py index 015336effe..e83e85de13 100755 --- a/qa/rpc-tests/importmulti.py +++ b/test/functional/importmulti.py @@ -12,12 +12,11 @@ class ImportMultiTest (BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True - def setup_network(self, split=False): - self.nodes = start_nodes(2, self.options.tmpdir) - self.is_network_split=False + def setup_network(self): + self.setup_nodes() def run_test (self): - print ("Mining blocks...") + self.log.info("Mining blocks...") self.nodes[0].generate(1) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] @@ -48,7 +47,7 @@ class ImportMultiTest (BitcoinTestFramework): # RPC importmulti ----------------------------------------------- # Bitcoin Address - print("Should import an address") + self.log.info("Should import an address") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": { @@ -64,7 +63,7 @@ class ImportMultiTest (BitcoinTestFramework): watchonly_address = address['address'] watchonly_timestamp = timestamp - print("Should not import an invalid address") + self.log.info("Should not import an invalid address") result = self.nodes[1].importmulti([{ "scriptPubKey": { "address": "not valid address", @@ -76,7 +75,7 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal(result[0]['error']['message'], 'Invalid address') # ScriptPubKey + internal - print("Should import a scriptPubKey with internal flag") + self.log.info("Should import a scriptPubKey with internal flag") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": address['scriptPubKey'], @@ -90,7 +89,7 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal(address_assert['timestamp'], timestamp) # ScriptPubKey + !internal - print("Should not import a scriptPubKey without internal flag") + self.log.info("Should not import a scriptPubKey without internal flag") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": address['scriptPubKey'], @@ -106,7 +105,7 @@ class ImportMultiTest (BitcoinTestFramework): # Address + Public key + !Internal - print("Should import an address with public key") + self.log.info("Should import an address with public key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": { @@ -123,7 +122,7 @@ class ImportMultiTest (BitcoinTestFramework): # ScriptPubKey + Public key + internal - print("Should import a scriptPubKey with internal and with public key") + self.log.info("Should import a scriptPubKey with internal and with public key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) request = [{ "scriptPubKey": address['scriptPubKey'], @@ -139,7 +138,7 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal(address_assert['timestamp'], timestamp) # ScriptPubKey + Public key + !internal - print("Should not import a scriptPubKey without internal and with public key") + self.log.info("Should not import a scriptPubKey without internal and with public key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) request = [{ "scriptPubKey": address['scriptPubKey'], @@ -156,7 +155,7 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal('timestamp' in address_assert, False) # Address + Private key + !watchonly - print("Should import an address with private key") + self.log.info("Should import an address with private key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": { @@ -172,7 +171,7 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal(address_assert['timestamp'], timestamp) # Address + Private key + watchonly - print("Should not import an address with private key and with watchonly") + self.log.info("Should not import an address with private key and with watchonly") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": { @@ -191,7 +190,7 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal('timestamp' in address_assert, False) # ScriptPubKey + Private key + internal - print("Should import a scriptPubKey with internal and with private key") + self.log.info("Should import a scriptPubKey with internal and with private key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": address['scriptPubKey'], @@ -206,7 +205,7 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal(address_assert['timestamp'], timestamp) # ScriptPubKey + Private key + !internal - print("Should not import a scriptPubKey without internal and with private key") + self.log.info("Should not import a scriptPubKey without internal and with private key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": address['scriptPubKey'], @@ -233,7 +232,7 @@ class ImportMultiTest (BitcoinTestFramework): timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] transaction = self.nodes[1].gettransaction(transactionid) - print("Should import a p2sh") + self.log.info("Should import a p2sh") result = self.nodes[1].importmulti([{ "scriptPubKey": { "address": multi_sig_script['address'] @@ -261,7 +260,7 @@ class ImportMultiTest (BitcoinTestFramework): timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] transaction = self.nodes[1].gettransaction(transactionid) - print("Should import a p2sh with respective redeem script") + self.log.info("Should import a p2sh with respective redeem script") result = self.nodes[1].importmulti([{ "scriptPubKey": { "address": multi_sig_script['address'] @@ -289,7 +288,7 @@ class ImportMultiTest (BitcoinTestFramework): timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] transaction = self.nodes[1].gettransaction(transactionid) - print("Should import a p2sh with respective redeem script and private keys") + self.log.info("Should import a p2sh with respective redeem script and private keys") result = self.nodes[1].importmulti([{ "scriptPubKey": { "address": multi_sig_script['address'] @@ -314,9 +313,10 @@ class ImportMultiTest (BitcoinTestFramework): self.nodes[1].generate(100) transactionid = self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) self.nodes[1].generate(1) + timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] transaction = self.nodes[1].gettransaction(transactionid) - print("Should import a p2sh with respective redeem script and private keys") + self.log.info("Should import a p2sh with respective redeem script and private keys") result = self.nodes[1].importmulti([{ "scriptPubKey": { "address": multi_sig_script['address'] @@ -332,7 +332,7 @@ class ImportMultiTest (BitcoinTestFramework): # Address + Public key + !Internal + Wrong pubkey - print("Should not import an address with a wrong public key") + self.log.info("Should not import an address with a wrong public key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) address2 = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ @@ -352,7 +352,7 @@ class ImportMultiTest (BitcoinTestFramework): # ScriptPubKey + Public key + internal + Wrong pubkey - print("Should not import a scriptPubKey with internal and with a wrong public key") + self.log.info("Should not import a scriptPubKey with internal and with a wrong public key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) address2 = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) request = [{ @@ -372,7 +372,7 @@ class ImportMultiTest (BitcoinTestFramework): # Address + Private key + !watchonly + Wrong private key - print("Should not import an address with a wrong private key") + self.log.info("Should not import an address with a wrong private key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) address2 = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ @@ -392,7 +392,7 @@ class ImportMultiTest (BitcoinTestFramework): # ScriptPubKey + Private key + internal + Wrong private key - print("Should not import a scriptPubKey with internal and with a wrong private key") + self.log.info("Should not import a scriptPubKey with internal and with a wrong private key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) address2 = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ @@ -409,16 +409,34 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal(address_assert['ismine'], False) assert_equal('timestamp' in address_assert, False) + + # Importing existing watch only address with new timestamp should replace saved timestamp. + assert_greater_than(timestamp, watchonly_timestamp) + self.log.info("Should replace previously saved watch only timestamp.") + result = self.nodes[1].importmulti([{ + "scriptPubKey": { + "address": watchonly_address, + }, + "timestamp": "now", + }]) + assert_equal(result[0]['success'], True) + address_assert = self.nodes[1].validateaddress(watchonly_address) + assert_equal(address_assert['iswatchonly'], True) + assert_equal(address_assert['ismine'], False) + assert_equal(address_assert['timestamp'], timestamp) + watchonly_timestamp = timestamp + + # restart nodes to check for proper serialization/deserialization of watch only address - stop_nodes(self.nodes) - self.nodes = start_nodes(2, self.options.tmpdir) + self.stop_nodes() + self.nodes = self.start_nodes(2, self.options.tmpdir) address_assert = self.nodes[1].validateaddress(watchonly_address) assert_equal(address_assert['iswatchonly'], True) assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], watchonly_timestamp); + assert_equal(address_assert['timestamp'], watchonly_timestamp) # Bad or missing timestamps - print("Should throw on invalid or missing timestamp values") + self.log.info("Should throw on invalid or missing timestamp values") assert_raises_message(JSONRPCException, 'Missing required timestamp field for key', self.nodes[1].importmulti, [{ "scriptPubKey": address['scriptPubKey'], diff --git a/qa/rpc-tests/importprunedfunds.py b/test/functional/importprunedfunds.py index a941124656..94753fe431 100755 --- a/qa/rpc-tests/importprunedfunds.py +++ b/test/functional/importprunedfunds.py @@ -14,14 +14,8 @@ class ImportPrunedFundsTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 2 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes,0,1) - self.is_network_split=False - self.sync_all() - def run_test(self): - print("Mining blocks...") + self.log.info("Mining blocks...") self.nodes[0].generate(101) self.sync_all() @@ -76,12 +70,7 @@ class ImportPrunedFundsTest(BitcoinTestFramework): self.sync_all() #Import with no affiliated address - try: - self.nodes[1].importprunedfunds(rawtxn1, proof1) - except JSONRPCException as e: - assert('No addresses' in e.error['message']) - else: - assert(False) + assert_raises_jsonrpc(-5, "No addresses", self.nodes[1].importprunedfunds, rawtxn1, proof1) balance1 = self.nodes[1].getbalance("", 0, True) assert_equal(balance1, Decimal(0)) @@ -112,12 +101,7 @@ class ImportPrunedFundsTest(BitcoinTestFramework): assert_equal(address_info['ismine'], True) #Remove transactions - try: - self.nodes[1].removeprunedfunds(txnid1) - except JSONRPCException as e: - assert('does not exist' in e.error['message']) - else: - assert(False) + assert_raises_jsonrpc(-8, "Transaction does not exist in wallet.", self.nodes[1].removeprunedfunds, txnid1) balance1 = self.nodes[1].getbalance("*", 0, True) assert_equal(balance1, Decimal('0.075')) diff --git a/qa/rpc-tests/invalidateblock.py b/test/functional/invalidateblock.py index 92e65927d5..c499d57b90 100755 --- a/qa/rpc-tests/invalidateblock.py +++ b/test/functional/invalidateblock.py @@ -8,63 +8,56 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * class InvalidateTest(BitcoinTestFramework): - - + def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 def setup_network(self): - self.nodes = [] - self.is_network_split = False - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug"])) - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) - + self.setup_nodes() + def run_test(self): - print("Make sure we repopulate setBlockIndexCandidates after InvalidateBlock:") - print("Mine 4 blocks on Node 0") + self.log.info("Make sure we repopulate setBlockIndexCandidates after InvalidateBlock:") + self.log.info("Mine 4 blocks on Node 0") self.nodes[0].generate(4) assert(self.nodes[0].getblockcount() == 4) besthash = self.nodes[0].getbestblockhash() - print("Mine competing 6 blocks on Node 1") + self.log.info("Mine competing 6 blocks on Node 1") self.nodes[1].generate(6) assert(self.nodes[1].getblockcount() == 6) - print("Connect nodes to force a reorg") + self.log.info("Connect nodes to force a reorg") connect_nodes_bi(self.nodes,0,1) sync_blocks(self.nodes[0:2]) assert(self.nodes[0].getblockcount() == 6) badhash = self.nodes[1].getblockhash(2) - print("Invalidate block 2 on node 0 and verify we reorg to node 0's original chain") + self.log.info("Invalidate block 2 on node 0 and verify we reorg to node 0's original chain") self.nodes[0].invalidateblock(badhash) newheight = self.nodes[0].getblockcount() newhash = self.nodes[0].getbestblockhash() if (newheight != 4 or newhash != besthash): raise AssertionError("Wrong tip for node0, hash %s, height %d"%(newhash,newheight)) - print("\nMake sure we won't reorg to a lower work chain:") + self.log.info("Make sure we won't reorg to a lower work chain:") connect_nodes_bi(self.nodes,1,2) - print("Sync node 2 to node 1 so both have 6 blocks") + self.log.info("Sync node 2 to node 1 so both have 6 blocks") sync_blocks(self.nodes[1:3]) assert(self.nodes[2].getblockcount() == 6) - print("Invalidate block 5 on node 1 so its tip is now at 4") + self.log.info("Invalidate block 5 on node 1 so its tip is now at 4") self.nodes[1].invalidateblock(self.nodes[1].getblockhash(5)) assert(self.nodes[1].getblockcount() == 4) - print("Invalidate block 3 on node 2, so its tip is now 2") + self.log.info("Invalidate block 3 on node 2, so its tip is now 2") self.nodes[2].invalidateblock(self.nodes[2].getblockhash(3)) assert(self.nodes[2].getblockcount() == 2) - print("..and then mine a block") + self.log.info("..and then mine a block") self.nodes[2].generate(1) - print("Verify all nodes are at the right height") + self.log.info("Verify all nodes are at the right height") time.sleep(5) - for i in range(3): - print(i,self.nodes[i].getblockcount()) - assert(self.nodes[2].getblockcount() == 3) - assert(self.nodes[0].getblockcount() == 4) + assert_equal(self.nodes[2].getblockcount(), 3) + assert_equal(self.nodes[0].getblockcount(), 4) node1height = self.nodes[1].getblockcount() if node1height < 4: raise AssertionError("Node 1 reorged to a lower height: %d"%node1height) diff --git a/qa/rpc-tests/invalidblockrequest.py b/test/functional/invalidblockrequest.py index eabc0db8df..eabc0db8df 100755 --- a/qa/rpc-tests/invalidblockrequest.py +++ b/test/functional/invalidblockrequest.py diff --git a/qa/rpc-tests/invalidtxrequest.py b/test/functional/invalidtxrequest.py index a9ac231f09..a9ac231f09 100755 --- a/qa/rpc-tests/invalidtxrequest.py +++ b/test/functional/invalidtxrequest.py diff --git a/qa/rpc-tests/keypool.py b/test/functional/keypool.py index 4b9936a1fd..f23a427d1f 100755 --- a/qa/rpc-tests/keypool.py +++ b/test/functional/keypool.py @@ -20,43 +20,49 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].encryptwallet('test') bitcoind_processes[0].wait() # Restart node 0 - nodes[0] = start_node(0, self.options.tmpdir) + nodes[0] = self.start_node(0, self.options.tmpdir) # Keep creating keys addr = nodes[0].getnewaddress() addr_data = nodes[0].validateaddress(addr) wallet_info = nodes[0].getwalletinfo() assert(addr_before_encrypting_data['hdmasterkeyid'] != wallet_info['hdmasterkeyid']) assert(addr_data['hdmasterkeyid'] == wallet_info['hdmasterkeyid']) - - try: - addr = nodes[0].getnewaddress() - raise AssertionError('Keypool should be exhausted after one address') - except JSONRPCException as e: - assert(e.error['code']==-12) + assert_raises_jsonrpc(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) - # put three new keys in the keypool + # put six (plus 2) new keys in the keypool (100% external-, +100% internal-keys, 1 in min) nodes[0].walletpassphrase('test', 12000) - nodes[0].keypoolrefill(3) + nodes[0].keypoolrefill(6) nodes[0].walletlock() + wi = nodes[0].getwalletinfo() + assert_equal(wi['keypoolsize_hd_internal'], 6) + assert_equal(wi['keypoolsize'], 6) - # drain the keys + # drain the internal keys + nodes[0].getrawchangeaddress() + nodes[0].getrawchangeaddress() + nodes[0].getrawchangeaddress() + nodes[0].getrawchangeaddress() + nodes[0].getrawchangeaddress() + nodes[0].getrawchangeaddress() addr = set() - addr.add(nodes[0].getrawchangeaddress()) - addr.add(nodes[0].getrawchangeaddress()) - addr.add(nodes[0].getrawchangeaddress()) - addr.add(nodes[0].getrawchangeaddress()) - # assert that four unique addresses were returned - assert(len(addr) == 4) # the next one should fail - try: - addr = nodes[0].getrawchangeaddress() - raise AssertionError('Keypool should be exhausted after three addresses') - except JSONRPCException as e: - assert(e.error['code']==-12) + assert_raises_jsonrpc(-12, "Keypool ran out", nodes[0].getrawchangeaddress) + + # drain the external keys + addr.add(nodes[0].getnewaddress()) + addr.add(nodes[0].getnewaddress()) + addr.add(nodes[0].getnewaddress()) + addr.add(nodes[0].getnewaddress()) + addr.add(nodes[0].getnewaddress()) + addr.add(nodes[0].getnewaddress()) + assert(len(addr) == 6) + # the next one should fail + assert_raises_jsonrpc(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # refill keypool with three new addresses nodes[0].walletpassphrase('test', 1) nodes[0].keypoolrefill(3) + # test walletpassphrase timeout time.sleep(1.1) assert_equal(nodes[0].getwalletinfo()["unlocked_until"], 0) @@ -65,20 +71,18 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].generate(1) nodes[0].generate(1) nodes[0].generate(1) - nodes[0].generate(1) - try: - nodes[0].generate(1) - raise AssertionError('Keypool should be exhausted after three addesses') - except JSONRPCException as e: - assert(e.error['code']==-12) + assert_raises_jsonrpc(-12, "Keypool ran out", nodes[0].generate, 1) + + nodes[0].walletpassphrase('test', 100) + nodes[0].keypoolrefill(100) + wi = nodes[0].getwalletinfo() + assert_equal(wi['keypoolsize_hd_internal'], 100) + assert_equal(wi['keypoolsize'], 100) def __init__(self): super().__init__() self.setup_clean_chain = False self.num_nodes = 1 - def setup_network(self): - self.nodes = self.setup_nodes() - if __name__ == '__main__': KeyPoolTest().main() diff --git a/qa/rpc-tests/listsinceblock.py b/test/functional/listsinceblock.py index 88304af5b0..f3d41e573e 100755 --- a/qa/rpc-tests/listsinceblock.py +++ b/test/functional/listsinceblock.py @@ -43,7 +43,6 @@ class ListSinceBlockTest (BitcoinTestFramework): This test only checks that [tx0] is present. ''' - assert_equal(self.is_network_split, False) self.nodes[2].generate(101) self.sync_all() @@ -54,7 +53,6 @@ class ListSinceBlockTest (BitcoinTestFramework): # Split network into two self.split_network() - assert_equal(self.is_network_split, True) # send to nodes[0] from nodes[2] senttx = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) @@ -62,9 +60,9 @@ class ListSinceBlockTest (BitcoinTestFramework): # generate on both sides lastblockhash = self.nodes[1].generate(6)[5] self.nodes[2].generate(7) - print('lastblockhash=%s' % (lastblockhash)) + self.log.info('lastblockhash=%s' % (lastblockhash)) - self.sync_all() + self.sync_all([self.nodes[:2], self.nodes[2:]]) self.join_network() diff --git a/qa/rpc-tests/listtransactions.py b/test/functional/listtransactions.py index 92fb96c809..f69f1c5724 100755 --- a/qa/rpc-tests/listtransactions.py +++ b/test/functional/listtransactions.py @@ -24,7 +24,7 @@ class ListTransactionsTest(BitcoinTestFramework): def setup_nodes(self): #This test requires mocktime enable_mocktime() - return start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): # Simple send, 0 to 1: @@ -126,7 +126,11 @@ class ListTransactionsTest(BitcoinTestFramework): assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_1}, {"bip125-replaceable":"no"}) # Tx2 will build off txid_1, still not opting in to RBF. + utxo_to_use = get_unconfirmed_utxo_entry(self.nodes[0], txid_1) + assert_equal(utxo_to_use["safe"], True) utxo_to_use = get_unconfirmed_utxo_entry(self.nodes[1], txid_1) + utxo_to_use = get_unconfirmed_utxo_entry(self.nodes[1], txid_1) + assert_equal(utxo_to_use["safe"], False) # Create tx2 using createrawtransaction inputs = [{"txid":utxo_to_use["txid"], "vout":utxo_to_use["vout"]}] diff --git a/qa/rpc-tests/maxuploadtarget.py b/test/functional/maxuploadtarget.py index 757aa60afd..66e5bd29e6 100755 --- a/qa/rpc-tests/maxuploadtarget.py +++ b/test/functional/maxuploadtarget.py @@ -10,72 +10,24 @@ if uploadtarget has been reached. if uploadtarget has been reached. * Verify that the upload counters are reset after 24 hours. """ +from collections import defaultdict +import time from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -import time -# TestNode: bare-bones "peer". Used mostly as a conduit for a test to sending -# p2p messages to a node, generating the messages in the main testing logic. class TestNode(NodeConnCB): def __init__(self): - NodeConnCB.__init__(self) - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() - self.block_receive_map = {} - - def add_connection(self, conn): - self.connection = conn - self.peer_disconnected = False + super().__init__() + self.block_receive_map = defaultdict(int) def on_inv(self, conn, message): pass - # Track the last getdata message we receive (used in the test) - def on_getdata(self, conn, message): - self.last_getdata = message - def on_block(self, conn, message): message.block.calc_sha256() - try: - self.block_receive_map[message.block.sha256] += 1 - except KeyError as e: - self.block_receive_map[message.block.sha256] = 1 - - # Spin until verack message is received from the node. - # We use this to signal that our test can begin. This - # is called from the testing thread, so it needs to acquire - # the global lock. - def wait_for_verack(self): - def veracked(): - return self.verack_received - return wait_until(veracked, timeout=10) - - def wait_for_disconnect(self): - def disconnected(): - return self.peer_disconnected - return wait_until(disconnected, timeout=10) - - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_pong(self, conn, message): - self.last_pong = message - - def on_close(self, conn): - self.peer_disconnected = True - - # Sync up with the node after delivery of a block - def sync_with_ping(self, timeout=30): - def received_pong(): - return (self.last_pong.nonce == self.ping_counter) - self.connection.send_message(msg_ping(nonce=self.ping_counter)) - success = wait_until(received_pong, timeout=timeout) - self.ping_counter += 1 - return success + self.block_receive_map[message.block.sha256] += 1 class MaxUploadTest(BitcoinTestFramework): @@ -83,15 +35,11 @@ class MaxUploadTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 + self.extra_args = [["-maxuploadtarget=800", "-blockmaxsize=999000"]] # Cache for utxos, as the listunspent may take a long time later in the test self.utxo_cache = [] - def setup_network(self): - # Start a node with maxuploadtarget of 200 MB (/24h) - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-maxuploadtarget=800", "-blockmaxsize=999000"])) - def run_test(self): # Before we connect anything, we first set the time on the node # to be in the past, otherwise things break because the CNode @@ -161,7 +109,7 @@ class MaxUploadTest(BitcoinTestFramework): test_nodes[0].send_message(getdata_request) test_nodes[0].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 2) - print("Peer 0 disconnected after downloading old block too many times") + self.log.info("Peer 0 disconnected after downloading old block too many times") # Requesting the current block on test_nodes[1] should succeed indefinitely, # even when over the max upload target. @@ -172,7 +120,7 @@ class MaxUploadTest(BitcoinTestFramework): test_nodes[1].sync_with_ping() assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) - print("Peer 1 able to repeatedly download new block") + self.log.info("Peer 1 able to repeatedly download new block") # But if test_nodes[1] tries for an old block, it gets disconnected too. getdata_request.inv = [CInv(2, big_old_block)] @@ -180,9 +128,9 @@ class MaxUploadTest(BitcoinTestFramework): test_nodes[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 1) - print("Peer 1 disconnected after trying to download old block") + self.log.info("Peer 1 disconnected after trying to download old block") - print("Advancing system time on node to clear counters...") + self.log.info("Advancing system time on node to clear counters...") # If we advance the time by 24 hours, then the counters should reset, # and test_nodes[2] should be able to retrieve the old block. @@ -192,42 +140,35 @@ class MaxUploadTest(BitcoinTestFramework): test_nodes[2].sync_with_ping() assert_equal(test_nodes[2].block_receive_map[big_old_block], 1) - print("Peer 2 able to download old block") + self.log.info("Peer 2 able to download old block") [c.disconnect_node() for c in connections] #stop and start node 0 with 1MB maxuploadtarget, whitelist 127.0.0.1 - print("Restarting nodes with -whitelist=127.0.0.1") - stop_node(self.nodes[0], 0) - self.nodes[0] = start_node(0, self.options.tmpdir, ["-debug", "-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) + self.log.info("Restarting nodes with -whitelist=127.0.0.1") + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) - #recreate/reconnect 3 test nodes - test_nodes = [] - connections = [] - - for i in range(3): - test_nodes.append(TestNode()) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[i])) - test_nodes[i].add_connection(connections[i]) + #recreate/reconnect a test node + test_nodes = [TestNode()] + connections = [NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[0])] + test_nodes[0].add_connection(connections[0]) NetworkThread().start() # Start up network handling in another thread - [x.wait_for_verack() for x in test_nodes] + test_nodes[0].wait_for_verack() #retrieve 20 blocks which should be enough to break the 1MB limit getdata_request.inv = [CInv(2, big_new_block)] for i in range(20): - test_nodes[1].send_message(getdata_request) - test_nodes[1].sync_with_ping() - assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) + test_nodes[0].send_message(getdata_request) + test_nodes[0].sync_with_ping() + assert_equal(test_nodes[0].block_receive_map[big_new_block], i+1) getdata_request.inv = [CInv(2, big_old_block)] - test_nodes[1].send_message(getdata_request) - test_nodes[1].wait_for_disconnect() - assert_equal(len(self.nodes[0].getpeerinfo()), 3) #node is still connected because of the whitelist + test_nodes[0].send_and_ping(getdata_request) + assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the whitelist - print("Peer 1 still connected after trying to download old block (whitelisted)") - - [c.disconnect_node() for c in connections] + self.log.info("Peer still connected after trying to download old block (whitelisted)") if __name__ == '__main__': MaxUploadTest().main() diff --git a/qa/rpc-tests/mempool_limit.py b/test/functional/mempool_limit.py index 8fae92ad2b..2777291dd0 100755 --- a/qa/rpc-tests/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -9,30 +9,25 @@ from test_framework.util import * class MempoolLimitTest(BitcoinTestFramework): - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxmempool=5", "-spendzeroconfchange=0", "-debug"])) - self.is_network_split = False - self.sync_all() - self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] - def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 - - self.txouts = gen_return_txouts() + self.extra_args = [["-maxmempool=5", "-spendzeroconfchange=0"]] def run_test(self): + txouts = gen_return_txouts() + relayfee = self.nodes[0].getnetworkinfo()['relayfee'] + txids = [] - utxos = create_confirmed_utxos(self.relayfee, self.nodes[0], 91) + utxos = create_confirmed_utxos(relayfee, self.nodes[0], 91) #create a mempool tx that will be evicted us0 = utxos.pop() inputs = [{ "txid" : us0["txid"], "vout" : us0["vout"]}] outputs = {self.nodes[0].getnewaddress() : 0.0001} tx = self.nodes[0].createrawtransaction(inputs, outputs) - self.nodes[0].settxfee(self.relayfee) # specifically fund this tx with low fee + self.nodes[0].settxfee(relayfee) # specifically fund this tx with low fee txF = self.nodes[0].fundrawtransaction(tx) self.nodes[0].settxfee(0) # return to automatic fee selection txFS = self.nodes[0].signrawtransaction(txF['hex']) @@ -42,7 +37,7 @@ class MempoolLimitTest(BitcoinTestFramework): base_fee = relayfee*100 for i in range (3): txids.append([]) - txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) + txids[i] = create_lots_of_big_transactions(self.nodes[0], txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) # by now, the tx should be evicted, check confirmation state assert(txid not in self.nodes[0].getrawmempool()) diff --git a/qa/rpc-tests/mempool_packages.py b/test/functional/mempool_packages.py index 388889d07c..e225493816 100755 --- a/qa/rpc-tests/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -16,14 +16,7 @@ class MempoolPackagesTest(BitcoinTestFramework): super().__init__() self.num_nodes = 2 self.setup_clean_chain = False - - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-debug"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-maxorphantx=1000", "-limitancestorcount=5", "-debug"])) - connect_nodes(self.nodes[0], 1) - self.is_network_split = False - self.sync_all() + self.extra_args = [["-maxorphantx=1000"], ["-maxorphantx=1000", "-limitancestorcount=5"]] # Build a transaction that spends parent_txid:vout # Return amount sent @@ -101,9 +94,21 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x], v_descendants[x]) assert(chain[0] not in v_descendants.keys()) + # Check that ancestor modified fees includes fee deltas from + # prioritisetransaction + self.nodes[0].prioritisetransaction(txid=chain[0], fee_delta=1000) + mempool = self.nodes[0].getrawmempool(True) + ancestor_fees = 0 + for x in chain: + ancestor_fees += mempool[x]['fee'] + assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN + 1000) + + # Undo the prioritisetransaction for later tests + self.nodes[0].prioritisetransaction(txid=chain[0], fee_delta=-1000) + # Check that descendant modified fees includes fee deltas from # prioritisetransaction - self.nodes[0].prioritisetransaction(chain[-1], 0, 1000) + self.nodes[0].prioritisetransaction(txid=chain[-1], fee_delta=1000) mempool = self.nodes[0].getrawmempool(True) descendant_fees = 0 @@ -112,10 +117,7 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 1000) # Adding one more transaction on to the chain should fail. - try: - self.chain_transaction(self.nodes[0], txid, vout, value, fee, 1) - except JSONRPCException as e: - print("too-long-ancestor-chain successfully rejected") + assert_raises_jsonrpc(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], txid, vout, value, fee, 1) # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. @@ -124,7 +126,7 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getrawmempool()), 0) # Prioritise a transaction that has been mined, then add it back to the # mempool by using invalidateblock. - self.nodes[0].prioritisetransaction(chain[-1], 0, 2000) + self.nodes[0].prioritisetransaction(txid=chain[-1], fee_delta=2000) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # Keep node1's tip synced with node0 self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash()) @@ -155,19 +157,19 @@ class MempoolPackagesTest(BitcoinTestFramework): for i in range(10): transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value}) - for i in range(MAX_DESCENDANTS): + # Sign and send up to MAX_DESCENDANT transactions chained off the parent tx + for i in range(MAX_DESCENDANTS - 1): utxo = transaction_package.pop(0) - try: - (txid, sent_value) = self.chain_transaction(self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) - for j in range(10): - transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value}) - if i == MAX_DESCENDANTS - 2: - mempool = self.nodes[0].getrawmempool(True) - assert_equal(mempool[parent_transaction]['descendantcount'], MAX_DESCENDANTS) - except JSONRPCException as e: - print(e.error['message']) - assert_equal(i, MAX_DESCENDANTS - 1) - print("tx that would create too large descendant package successfully rejected") + (txid, sent_value) = self.chain_transaction(self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) + for j in range(10): + transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value}) + + mempool = self.nodes[0].getrawmempool(True) + assert_equal(mempool[parent_transaction]['descendantcount'], MAX_DESCENDANTS) + + # Sending one more chained transaction will fail + utxo = transaction_package.pop(0) + assert_raises_jsonrpc(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) # TODO: check that node1's mempool is as expected diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py new file mode 100755 index 0000000000..e0889fd5e9 --- /dev/null +++ b/test/functional/mempool_persist.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test mempool persistence. + +By default, bitcoind will dump mempool on shutdown and +then reload it on startup. This can be overridden with +the -persistmempool=0 command line option. + +Test is as follows: + + - start node0, node1 and node2. node1 has -persistmempool=0 + - create 5 transactions on node2 to its own address. Note that these + are not sent to node0 or node1 addresses because we don't want + them to be saved in the wallet. + - check that node0 and node1 have 5 transactions in their mempools + - shutdown all nodes. + - startup node0. Verify that it still has 5 transactions + in its mempool. Shutdown node0. This tests that by default the + mempool is persistent. + - startup node1. Verify that its mempool is empty. Shutdown node1. + This tests that with -persistmempool=0, the mempool is not + dumped to disk when the node is shut down. + - Restart node0 with -persistmempool=0. Verify that its mempool is + empty. Shutdown node0. This tests that with -persistmempool=0, + the mempool is not loaded from disk on start up. + - Restart node0 with -persistmempool. Verify that it has 5 + transactions in its mempool. This tests that -persistmempool=0 + does not overwrite a previously valid mempool stored on disk. + +""" +import time + +from test_framework.mininode import wait_until +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class MempoolPersistTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + # We need 3 nodes for this test. Node1 does not have a persistent mempool. + self.num_nodes = 3 + self.setup_clean_chain = False + self.extra_args = [[], ["-persistmempool=0"], []] + + def run_test(self): + chain_height = self.nodes[0].getblockcount() + assert_equal(chain_height, 200) + + self.log.debug("Mine a single block to get out of IBD") + self.nodes[0].generate(1) + self.sync_all() + + self.log.debug("Send 5 transactions from node2 (to its own address)") + for i in range(5): + self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) + self.sync_all() + + self.log.debug("Verify that node0 and node1 have 5 transactions in their mempools") + assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[1].getrawmempool()), 5) + + self.log.debug("Stop-start node0 and node1. Verify that node0 has the transactions in its mempool and node1 does not.") + self.stop_nodes() + self.nodes = [] + self.nodes.append(self.start_node(0, self.options.tmpdir)) + self.nodes.append(self.start_node(1, self.options.tmpdir)) + # Give bitcoind a second to reload the mempool + time.sleep(1) + assert wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) + assert_equal(len(self.nodes[1].getrawmempool()), 0) + + self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") + self.stop_nodes() + self.nodes = [] + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-persistmempool=0"])) + # Give bitcoind a second to reload the mempool + time.sleep(1) + assert_equal(len(self.nodes[0].getrawmempool()), 0) + + self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.") + self.stop_nodes() + self.nodes = [] + self.nodes.append(self.start_node(0, self.options.tmpdir)) + assert wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) + +if __name__ == '__main__': + MempoolPersistTest().main() diff --git a/qa/rpc-tests/mempool_reorg.py b/test/functional/mempool_reorg.py index 2cd5573277..937bf4bab5 100755 --- a/qa/rpc-tests/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -17,22 +17,15 @@ class MempoolCoinbaseTest(BitcoinTestFramework): super().__init__() self.num_nodes = 2 self.setup_clean_chain = False + self.extra_args = [["-checkmempool"]] * 2 alert_filename = None # Set by setup_network - def setup_network(self): - args = ["-checkmempool", "-debug=mempool"] - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, args)) - self.nodes.append(start_node(1, self.options.tmpdir, args)) - connect_nodes(self.nodes[1], 0) - self.is_network_split = False - self.sync_all() - def run_test(self): - start_count = self.nodes[0].getblockcount() + # Start with a 200 block chain + assert_equal(self.nodes[0].getblockcount(), 200) - # Mine three blocks. After this, nodes[0] blocks + # Mine four blocks. After this, nodes[0] blocks # 101, 102, and 103 are spend-able. new_blocks = self.nodes[1].generate(4) self.sync_all() @@ -52,19 +45,21 @@ class MempoolCoinbaseTest(BitcoinTestFramework): spend_102_raw = create_tx(self.nodes[0], coinbase_txids[2], node0_address, 49.99) spend_103_raw = create_tx(self.nodes[0], coinbase_txids[3], node0_address, 49.99) - # Create a block-height-locked transaction which will be invalid after reorg + # Create a transaction which is time-locked to two blocks in the future timelock_tx = self.nodes[0].createrawtransaction([{"txid": coinbase_txids[0], "vout": 0}], {node0_address: 49.99}) # Set the time lock timelock_tx = timelock_tx.replace("ffffffff", "11111191", 1) timelock_tx = timelock_tx[:-8] + hex(self.nodes[0].getblockcount() + 2)[2:] + "000000" timelock_tx = self.nodes[0].signrawtransaction(timelock_tx)["hex"] - assert_raises(JSONRPCException, self.nodes[0].sendrawtransaction, timelock_tx) + # This will raise an exception because the timelock transaction is too immature to spend + assert_raises_jsonrpc(-26, "non-final", self.nodes[0].sendrawtransaction, timelock_tx) # Broadcast and mine spend_102 and 103: spend_102_id = self.nodes[0].sendrawtransaction(spend_102_raw) spend_103_id = self.nodes[0].sendrawtransaction(spend_103_raw) self.nodes[0].generate(1) - assert_raises(JSONRPCException, self.nodes[0].sendrawtransaction, timelock_tx) + # Time-locked transaction is still too immature to spend + assert_raises_jsonrpc(-26,'non-final', self.nodes[0].sendrawtransaction, timelock_tx) # Create 102_1 and 103_1: spend_102_1_raw = create_tx(self.nodes[0], spend_102_id, node1_address, 49.98) @@ -73,6 +68,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): # Broadcast and mine 103_1: spend_103_1_id = self.nodes[0].sendrawtransaction(spend_103_1_raw) last_block = self.nodes[0].generate(1) + # Time-locked transaction can now be spent timelock_tx_id = self.nodes[0].sendrawtransaction(timelock_tx) # ... now put spend_101 and spend_102_1 in memory pools: @@ -85,6 +81,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework): for node in self.nodes: node.invalidateblock(last_block[0]) + # Time-locked transaction is now too immature and has been removed from the mempool + # spend_103_1 has been re-orged out of the chain and is back in the mempool assert_equal(set(self.nodes[0].getrawmempool()), {spend_101_id, spend_102_1_id, spend_103_1_id}) # Use invalidateblock to re-org back and make all those coinbase spends diff --git a/qa/rpc-tests/mempool_resurrect_test.py b/test/functional/mempool_resurrect_test.py index c7f37aaa44..a2f6228df9 100755 --- a/qa/rpc-tests/mempool_resurrect_test.py +++ b/test/functional/mempool_resurrect_test.py @@ -14,13 +14,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework): super().__init__() self.num_nodes = 1 self.setup_clean_chain = False - - def setup_network(self): # Just need one node for this test - args = ["-checkmempool", "-debug=mempool"] - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, args)) - self.is_network_split = False + self.extra_args = [["-checkmempool"]] def run_test(self): node0_address = self.nodes[0].getnewaddress() diff --git a/qa/rpc-tests/mempool_spendcoinbase.py b/test/functional/mempool_spendcoinbase.py index da8485e2a2..277ea45ad5 100755 --- a/qa/rpc-tests/mempool_spendcoinbase.py +++ b/test/functional/mempool_spendcoinbase.py @@ -22,13 +22,7 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): super().__init__() self.num_nodes = 1 self.setup_clean_chain = False - - def setup_network(self): - # Just need one node for this test - args = ["-checkmempool", "-debug=mempool"] - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, args)) - self.is_network_split = False + self.extra_args = [["-checkmempool"]] def run_test(self): chain_height = self.nodes[0].getblockcount() @@ -45,7 +39,7 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): spend_101_id = self.nodes[0].sendrawtransaction(spends_raw[0]) # coinbase at height 102 should be too immature to spend - assert_raises(JSONRPCException, self.nodes[0].sendrawtransaction, spends_raw[1]) + assert_raises_jsonrpc(-26,"bad-txns-premature-spend-of-coinbase", self.nodes[0].sendrawtransaction, spends_raw[1]) # mempool should have just spend_101: assert_equal(self.nodes[0].getrawmempool(), [ spend_101_id ]) diff --git a/qa/rpc-tests/merkle_blocks.py b/test/functional/merkle_blocks.py index c4b45425d7..06af72ef10 100755 --- a/qa/rpc-tests/merkle_blocks.py +++ b/test/functional/merkle_blocks.py @@ -13,24 +13,19 @@ class MerkleBlockTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 4 + # Nodes 0/1 are "wallet" nodes, Nodes 2/3 are used for testing + self.extra_args = [[], [], [], ["-txindex"]] def setup_network(self): - self.nodes = [] - # Nodes 0/1 are "wallet" nodes - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug"])) - # Nodes 2/3 are used for testing - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) - self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-txindex"])) + self.setup_nodes() connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) - self.is_network_split = False self.sync_all() def run_test(self): - print("Mining blocks...") + self.log.info("Mining blocks...") self.nodes[0].generate(105) self.sync_all() diff --git a/qa/rpc-tests/multi_rpc.py b/test/functional/multi_rpc.py index 3b74bf1c46..6ff91a960b 100755 --- a/qa/rpc-tests/multi_rpc.py +++ b/test/functional/multi_rpc.py @@ -27,9 +27,6 @@ class HTTPBasicsTest (BitcoinTestFramework): f.write(rpcauth+"\n") f.write(rpcauth2+"\n") - def setup_network(self): - self.nodes = self.setup_nodes() - def run_test(self): ################################################## @@ -41,11 +38,9 @@ class HTTPBasicsTest (BitcoinTestFramework): authpair = url.username + ':' + url.password #New authpair generated via share/rpcuser tool - rpcauth = "rpcauth=rt:93648e835a54c573682c2eb19f882535$7681e9c5b74bdd85e78166031d2058e1069b3ed7ed967c93fc63abba06f31144" password = "cA773lm788buwYe4g4WT+05pKyNruVKjQ25x3n0DQcM=" #Second authpair with different username - rpcauth2 = "rpcauth=rt2:f8607b1a88861fac29dfccf9b52ff9f$ff36a0c23c8c62b4846112e50fa888416e94c17bfd4c42f88fd8f55ec6a3137e" password2 = "8/F3uMDw4KSEbw96U3CA1C4X05dkHDN2BPFjTgZW4KI=" authpairnew = "rt:"+password diff --git a/test/functional/net.py b/test/functional/net.py new file mode 100755 index 0000000000..3ba3764cf9 --- /dev/null +++ b/test/functional/net.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test RPC calls related to net. + +Tests correspond to code in rpc/net.cpp. +""" + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_jsonrpc, + connect_nodes_bi, + p2p_port, +) + + +class NetTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 2 + + def run_test(self): + self._test_connection_count() + self._test_getnettotals() + self._test_getnetworkinginfo() + self._test_getaddednodeinfo() + self._test_getpeerinfo() + + def _test_connection_count(self): + # connect_nodes_bi connects each node to the other + assert_equal(self.nodes[0].getconnectioncount(), 2) + + def _test_getnettotals(self): + # check that getnettotals totalbytesrecv and totalbytessent + # are consistent with getpeerinfo + peer_info = self.nodes[0].getpeerinfo() + assert_equal(len(peer_info), 2) + net_totals = self.nodes[0].getnettotals() + assert_equal(sum([peer['bytesrecv'] for peer in peer_info]), + net_totals['totalbytesrecv']) + assert_equal(sum([peer['bytessent'] for peer in peer_info]), + net_totals['totalbytessent']) + # test getnettotals and getpeerinfo by doing a ping + # the bytes sent/received should change + # note ping and pong are 32 bytes each + self.nodes[0].ping() + time.sleep(0.1) + peer_info_after_ping = self.nodes[0].getpeerinfo() + net_totals_after_ping = self.nodes[0].getnettotals() + for before, after in zip(peer_info, peer_info_after_ping): + assert_equal(before['bytesrecv_per_msg']['pong'] + 32, after['bytesrecv_per_msg']['pong']) + assert_equal(before['bytessent_per_msg']['ping'] + 32, after['bytessent_per_msg']['ping']) + assert_equal(net_totals['totalbytesrecv'] + 32*2, net_totals_after_ping['totalbytesrecv']) + assert_equal(net_totals['totalbytessent'] + 32*2, net_totals_after_ping['totalbytessent']) + + def _test_getnetworkinginfo(self): + assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) + assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) + + self.nodes[0].setnetworkactive(False) + assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) + timeout = 3 + while self.nodes[0].getnetworkinfo()['connections'] != 0: + # Wait a bit for all sockets to close + assert timeout > 0, 'not all connections closed in time' + timeout -= 0.1 + time.sleep(0.1) + + self.nodes[0].setnetworkactive(True) + connect_nodes_bi(self.nodes, 0, 1) + assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) + assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) + + def _test_getaddednodeinfo(self): + assert_equal(self.nodes[0].getaddednodeinfo(), []) + # add a node (node2) to node0 + ip_port = "127.0.0.1:{}".format(p2p_port(2)) + self.nodes[0].addnode(ip_port, 'add') + # check that the node has indeed been added + added_nodes = self.nodes[0].getaddednodeinfo(ip_port) + assert_equal(len(added_nodes), 1) + assert_equal(added_nodes[0]['addednode'], ip_port) + # check that a non-existant node returns an error + assert_raises_jsonrpc(-24, "Node has not been added", + self.nodes[0].getaddednodeinfo, '1.1.1.1') + + def _test_getpeerinfo(self): + peer_info = [x.getpeerinfo() for x in self.nodes] + # check both sides of bidirectional connection between nodes + # the address bound to on one side will be the source address for the other node + assert_equal(peer_info[0][0]['addrbind'], peer_info[1][0]['addr']) + assert_equal(peer_info[1][0]['addrbind'], peer_info[0][0]['addr']) + +if __name__ == '__main__': + NetTest().main() diff --git a/qa/rpc-tests/nulldummy.py b/test/functional/nulldummy.py index 66c4e90f21..9717add272 100755 --- a/qa/rpc-tests/nulldummy.py +++ b/test/functional/nulldummy.py @@ -41,11 +41,7 @@ class NULLDUMMYTest(BitcoinTestFramework): super().__init__() self.num_nodes = 1 self.setup_clean_chain = True - - def setup_network(self): - # Must set the blockversion for this test - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1', '-walletprematurewitness']]) + self.extra_args = [['-whitelist=127.0.0.1', '-walletprematurewitness']] def run_test(self): self.address = self.nodes[0].getnewaddress() @@ -64,40 +60,40 @@ class NULLDUMMYTest(BitcoinTestFramework): self.lastblockheight = 429 self.lastblocktime = int(time.time()) + 429 - print ("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]") + self.log.info("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]") test1txs = [self.create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, 49)] - txid1 = self.tx_submit(self.nodes[0], test1txs[0]) + txid1 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[0].serialize_with_witness()), True) test1txs.append(self.create_transaction(self.nodes[0], txid1, self.ms_address, 48)) - txid2 = self.tx_submit(self.nodes[0], test1txs[1]) + txid2 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[1].serialize_with_witness()), True) test1txs.append(self.create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, 49)) - txid3 = self.tx_submit(self.nodes[0], test1txs[2]) + txid3 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[2].serialize_with_witness()), True) self.block_submit(self.nodes[0], test1txs, False, True) - print ("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") - test2tx = self.create_transaction(self.nodes[0], txid2, self.ms_address, 48) + self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") + test2tx = self.create_transaction(self.nodes[0], txid2, self.ms_address, 47) trueDummy(test2tx) - txid4 = self.tx_submit(self.nodes[0], test2tx, NULLDUMMY_ERROR) + assert_raises_jsonrpc(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize_with_witness()), True) - print ("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") + self.log.info("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") self.block_submit(self.nodes[0], [test2tx], False, True) - print ("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") - test4tx = self.create_transaction(self.nodes[0], txid4, self.address, 47) + self.log.info("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") + test4tx = self.create_transaction(self.nodes[0], test2tx.hash, self.address, 46) test6txs=[CTransaction(test4tx)] trueDummy(test4tx) - self.tx_submit(self.nodes[0], test4tx, NULLDUMMY_ERROR) + assert_raises_jsonrpc(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize_with_witness()), True) self.block_submit(self.nodes[0], [test4tx]) - print ("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") + self.log.info("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") test5tx = self.create_transaction(self.nodes[0], txid3, self.wit_address, 48) test6txs.append(CTransaction(test5tx)) test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' - self.tx_submit(self.nodes[0], test5tx, NULLDUMMY_ERROR) + assert_raises_jsonrpc(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test5tx.serialize_with_witness()), True) self.block_submit(self.nodes[0], [test5tx], True) - print ("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]") + self.log.info("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]") for i in test6txs: - self.tx_submit(self.nodes[0], i) + self.nodes[0].sendrawtransaction(bytes_to_hex_str(i.serialize_with_witness()), True) self.block_submit(self.nodes[0], test6txs, True, True) @@ -112,17 +108,6 @@ class NULLDUMMYTest(BitcoinTestFramework): return tx - def tx_submit(self, node, tx, msg = ""): - tx.rehash() - try: - node.sendrawtransaction(bytes_to_hex_str(tx.serialize_with_witness()), True) - except JSONRPCException as exp: - assert_equal(exp.error["message"], msg) - else: - assert_equal('', msg) - return tx.hash - - def block_submit(self, node, txs, witness = False, accept = False): block = create_block(self.tip, create_coinbase(self.lastblockheight + 1), self.lastblocktime + 1) block.nVersion = 4 diff --git a/qa/rpc-tests/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index 2f21fa149a..322cb767db 100755 --- a/qa/rpc-tests/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -54,55 +54,6 @@ from test_framework.util import * import time from test_framework.blocktools import create_block, create_coinbase -# TestNode: bare-bones "peer". Used mostly as a conduit for a test to sending -# p2p messages to a node, generating the messages in the main testing logic. -class TestNode(NodeConnCB): - def __init__(self): - NodeConnCB.__init__(self) - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() - - def add_connection(self, conn): - self.connection = conn - - # Track the last getdata message we receive (used in the test) - def on_getdata(self, conn, message): - self.last_getdata = message - - # Spin until verack message is received from the node. - # We use this to signal that our test can begin. This - # is called from the testing thread, so it needs to acquire - # the global lock. - def wait_for_verack(self): - while True: - with mininode_lock: - if self.verack_received: - return - time.sleep(0.05) - - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_pong(self, conn, message): - self.last_pong = message - - # Sync up with the node after delivery of a block - def sync_with_ping(self, timeout=30): - self.connection.send_message(msg_ping(nonce=self.ping_counter)) - received_pong = False - sleep_time = 0.05 - while not received_pong and timeout > 0: - time.sleep(sleep_time) - timeout -= sleep_time - with mininode_lock: - if self.last_pong.nonce == self.ping_counter: - received_pong = True - self.ping_counter += 1 - return received_pong - - class AcceptBlockTest(BitcoinTestFramework): def add_options(self, parser): parser.add_option("--testbinary", dest="testbinary", @@ -113,22 +64,18 @@ class AcceptBlockTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 2 + self.extra_args = [[], ["-whitelist=127.0.0.1"]] def setup_network(self): # Node0 will be used to test behavior of processing unrequested blocks # from peers which are not whitelisted, while Node1 will be used for # the whitelisted case. - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"], - binary=self.options.testbinary)) - self.nodes.append(start_node(1, self.options.tmpdir, - ["-debug", "-whitelist=127.0.0.1"], - binary=self.options.testbinary)) + self.setup_nodes() def run_test(self): # Setup the p2p connections and start up the network thread. - test_node = TestNode() # connects to node0 (not whitelisted) - white_node = TestNode() # connects to node1 (whitelisted) + test_node = NodeConnCB() # connects to node0 (not whitelisted) + white_node = NodeConnCB() # connects to node1 (whitelisted) connections = [] connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) @@ -160,7 +107,7 @@ class AcceptBlockTest(BitcoinTestFramework): [ x.sync_with_ping() for x in [test_node, white_node] ] assert_equal(self.nodes[0].getblockcount(), 2) assert_equal(self.nodes[1].getblockcount(), 2) - print("First height 2 block accepted by both nodes") + self.log.info("First height 2 block accepted by both nodes") # 3. Send another block that builds on the original tip. blocks_h2f = [] # Blocks at height 2 that fork off the main chain @@ -179,7 +126,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == blocks_h2f[1].hash: assert_equal(x['status'], "valid-headers") - print("Second height 2 block accepted only from whitelisted peer") + self.log.info("Second height 2 block accepted only from whitelisted peer") # 4. Now send another block that builds on the forking chain. blocks_h3 = [] @@ -197,15 +144,12 @@ class AcceptBlockTest(BitcoinTestFramework): assert_equal(x['status'], "headers-only") # But this block should be accepted by node0 since it has more work. - try: - self.nodes[0].getblock(blocks_h3[0].hash) - print("Unrequested more-work block accepted from non-whitelisted peer") - except: - raise AssertionError("Unrequested more work block was not processed") + self.nodes[0].getblock(blocks_h3[0].hash) + self.log.info("Unrequested more-work block accepted from non-whitelisted peer") # Node1 should have accepted and reorged. assert_equal(self.nodes[1].getblockcount(), 3) - print("Successfully reorged to length 3 chain from whitelisted peer") + self.log.info("Successfully reorged to length 3 chain from whitelisted peer") # 4b. Now mine 288 more blocks and deliver; all should be processed but # the last (height-too-high) on node0. Node1 should process the tip if @@ -225,26 +169,17 @@ class AcceptBlockTest(BitcoinTestFramework): tips[j] = next_block time.sleep(2) - for x in all_blocks: - try: - self.nodes[0].getblock(x.hash) - if x == all_blocks[287]: - raise AssertionError("Unrequested block too far-ahead should have been ignored") - except: - if x == all_blocks[287]: - print("Unrequested block too far-ahead not processed") - else: - raise AssertionError("Unrequested block with more work should have been accepted") + # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead + for x in all_blocks[:-1]: + self.nodes[0].getblock(x.hash) + assert_raises_jsonrpc(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) headers_message.headers.pop() # Ensure the last block is unrequested white_node.send_message(headers_message) # Send headers leading to tip white_node.send_message(msg_block(tips[1])) # Now deliver the tip - try: - white_node.sync_with_ping() - self.nodes[1].getblock(tips[1].hash) - print("Unrequested block far ahead of tip accepted from whitelisted peer") - except: - raise AssertionError("Unrequested block from whitelisted peer not accepted") + white_node.sync_with_ping() + self.nodes[1].getblock(tips[1].hash) + self.log.info("Unrequested block far ahead of tip accepted from whitelisted peer") # 5. Test handling of unrequested block on the node that didn't process # Should still not be processed (even though it has a child that has more @@ -258,30 +193,30 @@ class AcceptBlockTest(BitcoinTestFramework): # a getdata request for this block. test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) - print("Unrequested block that would complete more-work chain was ignored") + self.log.info("Unrequested block that would complete more-work chain was ignored") # 6. Try to get node to request the missing block. # Poke the node with an inv for block at height 3 and see if that # triggers a getdata on block 2 (it should if block 2 is missing). with mininode_lock: # Clear state so we can check the getdata request - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) test_node.send_message(msg_inv([CInv(2, blocks_h3[0].sha256)])) test_node.sync_with_ping() with mininode_lock: - getdata = test_node.last_getdata + getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block assert_equal(getdata.inv[0].hash, blocks_h2f[0].sha256) - print("Inv at tip triggered getdata for unprocessed block") + self.log.info("Inv at tip triggered getdata for unprocessed block") # 7. Send the missing block for the third time (now it is requested) test_node.send_message(msg_block(blocks_h2f[0])) test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 290) - print("Successfully reorged to longer chain from non-whitelisted peer") + self.log.info("Successfully reorged to longer chain from non-whitelisted peer") [ c.disconnect_node() for c in connections ] diff --git a/qa/rpc-tests/p2p-compactblocks.py b/test/functional/p2p-compactblocks.py index 79561f35f7..ff76e49fba 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/test/functional/p2p-compactblocks.py @@ -15,61 +15,35 @@ from test_framework.blocktools import create_block, create_coinbase, add_witness from test_framework.script import CScript, OP_TRUE # TestNode: A peer we use to send messages to bitcoind, and store responses. -class TestNode(SingleNodeConnCB): +class TestNode(NodeConnCB): def __init__(self): - SingleNodeConnCB.__init__(self) + super().__init__() self.last_sendcmpct = [] - self.last_headers = None - self.last_inv = None - self.last_cmpctblock = None self.block_announced = False - self.last_getdata = None - self.last_getheaders = None - self.last_getblocktxn = None - self.last_block = None - self.last_blocktxn = None # Store the hashes of blocks we've seen announced. # This is for synchronizing the p2p message traffic, # so we can eg wait until a particular block is announced. - self.set_announced_blockhashes = set() + self.announced_blockhashes = set() def on_sendcmpct(self, conn, message): self.last_sendcmpct.append(message) - def on_block(self, conn, message): - self.last_block = message - def on_cmpctblock(self, conn, message): - self.last_cmpctblock = message self.block_announced = True - self.last_cmpctblock.header_and_shortids.header.calc_sha256() - self.set_announced_blockhashes.add(self.last_cmpctblock.header_and_shortids.header.sha256) + self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() + self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256) def on_headers(self, conn, message): - self.last_headers = message self.block_announced = True - for x in self.last_headers.headers: + for x in self.last_message["headers"].headers: x.calc_sha256() - self.set_announced_blockhashes.add(x.sha256) + self.announced_blockhashes.add(x.sha256) def on_inv(self, conn, message): - self.last_inv = message - for x in self.last_inv.inv: + for x in self.last_message["inv"].inv: if x.type == 2: self.block_announced = True - self.set_announced_blockhashes.add(x.hash) - - def on_getdata(self, conn, message): - self.last_getdata = message - - def on_getheaders(self, conn, message): - self.last_getheaders = message - - def on_getblocktxn(self, conn, message): - self.last_getblocktxn = message - - def on_blocktxn(self, conn, message): - self.last_blocktxn = message + self.announced_blockhashes.add(x.hash) # Requires caller to hold mininode_lock def received_block_announcement(self): @@ -78,9 +52,9 @@ class TestNode(SingleNodeConnCB): def clear_block_announcement(self): with mininode_lock: self.block_announced = False - self.last_inv = None - self.last_headers = None - self.last_cmpctblock = None + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) + self.last_message.pop("cmpctblock", None) def get_headers(self, locator, hashstop): msg = msg_getheaders() @@ -96,34 +70,37 @@ class TestNode(SingleNodeConnCB): def request_headers_and_sync(self, locator, hashstop=0): self.clear_block_announcement() self.get_headers(locator, hashstop) - assert(wait_until(self.received_block_announcement, timeout=30)) - assert(self.received_block_announcement()) + assert wait_until(self.received_block_announcement, timeout=30) self.clear_block_announcement() # Block until a block announcement for a particular block hash is # received. def wait_for_block_announcement(self, block_hash, timeout=30): def received_hash(): - return (block_hash in self.set_announced_blockhashes) + return (block_hash in self.announced_blockhashes) return wait_until(received_hash, timeout=timeout) + def send_await_disconnect(self, message, timeout=30): + """Sends a message to the node and wait for disconnect. + + This is used when we want to send a message into the node that we expect + will get us disconnected, eg an invalid block.""" + self.send_message(message) + success = wait_until(lambda: not self.connected, timeout=timeout) + if not success: + logger.error("send_await_disconnect failed!") + raise AssertionError("send_await_disconnect failed!") + return success + class CompactBlocksTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True # Node0 = pre-segwit, node1 = segwit-aware self.num_nodes = 2 + self.extra_args = [["-vbparams=segwit:0:0"], ["-txindex"]] self.utxos = [] - def setup_network(self): - self.nodes = [] - - # Start up node0 to be a version 1, pre-segwit node. - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"], - ["-debug", "-logtimemicros", "-txindex"]]) - connect_nodes(self.nodes[0], 1) - def build_block_on_tip(self, node, segwit=False): height = node.getblockcount() tip = node.getbestblockhash() @@ -195,14 +172,14 @@ class CompactBlocksTest(BitcoinTestFramework): with mininode_lock: assert predicate(peer), ( "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( - block_hash, peer.last_cmpctblock, peer.last_inv)) + block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) # We shouldn't get any block announcements via cmpctblock yet. - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Try one more time, this time after requesting headers. test_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) # Test a few ways of using sendcmpct that should NOT # result in compact block announcements. @@ -214,7 +191,7 @@ class CompactBlocksTest(BitcoinTestFramework): sendcmpct.version = preferred_version+1 sendcmpct.announce = True test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. test_node.request_headers_and_sync(locator=[tip]) @@ -223,7 +200,7 @@ class CompactBlocksTest(BitcoinTestFramework): sendcmpct.version = preferred_version sendcmpct.announce = False test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. test_node.request_headers_and_sync(locator=[tip]) @@ -232,26 +209,26 @@ class CompactBlocksTest(BitcoinTestFramework): sendcmpct.version = preferred_version sendcmpct.announce = True test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time (no headers sync should be needed!) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time, after turning on sendheaders test_node.send_and_ping(msg_sendheaders()) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time, after sending a version-1, announce=false message. sendcmpct.version = preferred_version-1 sendcmpct.announce = False test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Now turn off announcements sendcmpct.version = preferred_version sendcmpct.announce = False test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) if old_node is not None: # Verify that a peer using an older protocol version can receive @@ -261,7 +238,7 @@ class CompactBlocksTest(BitcoinTestFramework): old_node.send_and_ping(sendcmpct) # Header sync old_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, old_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message) # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. def test_invalid_cmpctblock_message(self): @@ -274,8 +251,8 @@ class CompactBlocksTest(BitcoinTestFramework): # This index will be too high prefilled_txn = PrefilledTransaction(1, block.vtx[0]) cmpct_block.prefilled_txn = [prefilled_txn] - self.test_node.send_and_ping(msg_cmpctblock(cmpct_block)) - assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock) + self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) # Compare the generated shortids to what we expect based on BIP 152, given # bitcoind's choice of nonce. @@ -326,9 +303,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert(test_node.last_cmpctblock is not None) + assert("cmpctblock" in test_node.last_message) # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) + header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) # Now fetch the compact block using a normal non-announce getdata @@ -343,9 +320,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert(test_node.last_cmpctblock is not None) + assert("cmpctblock" in test_node.last_message) # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) + header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): @@ -405,20 +382,20 @@ class CompactBlocksTest(BitcoinTestFramework): for announce in ["inv", "header"]: block = self.build_block_on_tip(node, segwit=segwit) with mininode_lock: - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) if announce == "inv": test_node.send_message(msg_inv([CInv(2, block.sha256)])) - success = wait_until(lambda: test_node.last_getheaders is not None, timeout=30) + success = wait_until(lambda: "getheaders" in test_node.last_message, timeout=30) assert(success) test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) - success = wait_until(lambda: test_node.last_getdata is not None, timeout=30) + success = wait_until(lambda: "getdata" in test_node.last_message, timeout=30) assert(success) - assert_equal(len(test_node.last_getdata.inv), 1) - assert_equal(test_node.last_getdata.inv[0].type, 4) - assert_equal(test_node.last_getdata.inv[0].hash, block.sha256) + assert_equal(len(test_node.last_message["getdata"].inv), 1) + assert_equal(test_node.last_message["getdata"].inv[0].type, 4) + assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Send back a compactblock message that omits the coinbase comp_block = HeaderAndShortIDs() @@ -434,8 +411,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. with mininode_lock: - assert(test_node.last_getblocktxn is not None) - absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute() + assert("getblocktxn" in test_node.last_message) + absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request # Send the coinbase, and verify that the tip advances. @@ -474,8 +451,8 @@ class CompactBlocksTest(BitcoinTestFramework): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) with mininode_lock: - assert(peer.last_getblocktxn is not None) - absolute_indexes = peer.last_getblocktxn.block_txn_request.to_absolute() + assert("getblocktxn" in peer.last_message) + absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) def test_tip_after_message(node, peer, msg, tip): @@ -539,14 +516,14 @@ class CompactBlocksTest(BitcoinTestFramework): # Clear out last request. with mininode_lock: - test_node.last_getblocktxn = None + test_node.last_message.pop("getblocktxn", None) # Send compact block comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with mininode_lock: # Shouldn't have gotten a request for any transaction - assert(test_node.last_getblocktxn is None) + assert("getblocktxn" not in test_node.last_message) # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. @@ -572,8 +549,8 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with mininode_lock: - assert(test_node.last_getblocktxn is not None) - absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute() + assert("getblocktxn" in test_node.last_message) + absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) # Now give an incorrect response. @@ -594,11 +571,11 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # We should receive a getdata request - success = wait_until(lambda: test_node.last_getdata is not None, timeout=10) + success = wait_until(lambda: "getdata" in test_node.last_message, timeout=10) assert(success) - assert_equal(len(test_node.last_getdata.inv), 1) - assert(test_node.last_getdata.inv[0].type == 2 or test_node.last_getdata.inv[0].type == 2|MSG_WITNESS_FLAG) - assert_equal(test_node.last_getdata.inv[0].hash, block.sha256) + assert_equal(len(test_node.last_message["getdata"].inv), 1) + assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2|MSG_WITNESS_FLAG) + assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block if version==2: @@ -622,15 +599,15 @@ class CompactBlocksTest(BitcoinTestFramework): num_to_request = random.randint(1, len(block.vtx)) msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) test_node.send_message(msg) - success = wait_until(lambda: test_node.last_blocktxn is not None, timeout=10) + success = wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10) assert(success) [tx.calc_sha256() for tx in block.vtx] with mininode_lock: - assert_equal(test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) + assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) all_indices = msg.block_txn_request.to_absolute() for index in all_indices: - tx = test_node.last_blocktxn.block_transactions.transactions.pop(0) + tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) tx.calc_sha256() assert_equal(tx.sha256, block.vtx[index].sha256) if version == 1: @@ -639,7 +616,7 @@ class CompactBlocksTest(BitcoinTestFramework): else: # Check that the witness matches assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) - test_node.last_blocktxn = None + test_node.last_message.pop("blocktxn", None) current_height -= 1 # Next request should send a full block response, as we're past the @@ -647,13 +624,13 @@ class CompactBlocksTest(BitcoinTestFramework): block_hash = node.getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) with mininode_lock: - test_node.last_block = None - test_node.last_blocktxn = None + test_node.last_message.pop("block", None) + test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) with mininode_lock: - test_node.last_block.block.calc_sha256() - assert_equal(test_node.last_block.block.sha256, int(block_hash, 16)) - assert_equal(test_node.last_blocktxn, None) + test_node.last_message["block"].block.calc_sha256() + assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) + assert "blocktxn" not in test_node.last_message def test_compactblocks_not_at_tip(self, node, test_node): # Test that requesting old compactblocks doesn't work. @@ -666,7 +643,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.clear_block_announcement() test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - success = wait_until(lambda: test_node.last_cmpctblock is not None, timeout=30) + success = wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) assert(success) test_node.clear_block_announcement() @@ -674,13 +651,13 @@ class CompactBlocksTest(BitcoinTestFramework): wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() with mininode_lock: - test_node.last_block = None + test_node.last_message.pop("block", None) test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - success = wait_until(lambda: test_node.last_block is not None, timeout=30) + success = wait_until(lambda: "block" in test_node.last_message, timeout=30) assert(success) with mininode_lock: - test_node.last_block.block.calc_sha256() - assert_equal(test_node.last_block.block.sha256, int(new_blocks[0], 16)) + test_node.last_message["block"].block.calc_sha256() + assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) # Generate an old compactblock, and verify that it's not accepted. cur_height = node.getblockcount() @@ -707,10 +684,10 @@ class CompactBlocksTest(BitcoinTestFramework): msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) with mininode_lock: - test_node.last_blocktxn = None + test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) with mininode_lock: - assert(test_node.last_blocktxn is None) + assert "blocktxn" not in test_node.last_message def activate_segwit(self, node): node.generate(144*3) @@ -731,9 +708,9 @@ class CompactBlocksTest(BitcoinTestFramework): wait_until(lambda: l.received_block_announcement(), timeout=30) with mininode_lock: for l in listeners: - assert(l.last_cmpctblock is not None) - l.last_cmpctblock.header_and_shortids.header.calc_sha256() - assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256) + assert "cmpctblock" in l.last_message + l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() + assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. @@ -785,7 +762,7 @@ class CompactBlocksTest(BitcoinTestFramework): msg = msg_cmpctblock(cmpct_block.to_p2p()) peer.send_and_ping(msg) with mininode_lock: - assert(peer.last_getblocktxn is not None) + assert "getblocktxn" in peer.last_message return block, cmpct_block block, cmpct_block = announce_cmpct_block(node, stalling_peer) @@ -846,102 +823,102 @@ class CompactBlocksTest(BitcoinTestFramework): # We will need UTXOs to construct transactions in later tests. self.make_utxos() - print("Running tests, pre-segwit activation:") + self.log.info("Running tests, pre-segwit activation:") - print("\tTesting SENDCMPCT p2p message... ") + self.log.info("Testing SENDCMPCT p2p message... ") self.test_sendcmpct(self.nodes[0], self.test_node, 1) sync_blocks(self.nodes) self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node) sync_blocks(self.nodes) - print("\tTesting compactblock construction...") + self.log.info("Testing compactblock construction...") self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False) sync_blocks(self.nodes) self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False) sync_blocks(self.nodes) - print("\tTesting compactblock requests... ") + self.log.info("Testing compactblock requests... ") self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False) sync_blocks(self.nodes) self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False) sync_blocks(self.nodes) - print("\tTesting getblocktxn requests...") + self.log.info("Testing getblocktxn requests...") self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) sync_blocks(self.nodes) self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) sync_blocks(self.nodes) - print("\tTesting getblocktxn handler...") + self.log.info("Testing getblocktxn handler...") self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) sync_blocks(self.nodes) self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) sync_blocks(self.nodes) - print("\tTesting compactblock requests/announcements not at chain tip...") + self.log.info("Testing compactblock requests/announcements not at chain tip...") self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) sync_blocks(self.nodes) self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node) self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) sync_blocks(self.nodes) - print("\tTesting handling of incorrect blocktxn responses...") + self.log.info("Testing handling of incorrect blocktxn responses...") self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) sync_blocks(self.nodes) self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2) sync_blocks(self.nodes) # End-to-end block relay tests - print("\tTesting end-to-end block relay...") + self.log.info("Testing end-to-end block relay...") self.request_cb_announcements(self.test_node, self.nodes[0], 1) self.request_cb_announcements(self.old_node, self.nodes[1], 1) self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) - print("\tTesting handling of invalid compact blocks...") + self.log.info("Testing handling of invalid compact blocks...") self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) - print("\tTesting reconstructing compact blocks from all peers...") + self.log.info("Testing reconstructing compact blocks from all peers...") self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node) sync_blocks(self.nodes) # Advance to segwit activation - print ("\nAdvancing to segwit activation\n") + self.log.info("Advancing to segwit activation") self.activate_segwit(self.nodes[1]) - print ("Running tests, post-segwit activation...") + self.log.info("Running tests, post-segwit activation...") - print("\tTesting compactblock construction...") + self.log.info("Testing compactblock construction...") self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True) self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True) sync_blocks(self.nodes) - print("\tTesting compactblock requests (unupgraded node)... ") + self.log.info("Testing compactblock requests (unupgraded node)... ") self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True) - print("\tTesting getblocktxn requests (unupgraded node)...") + self.log.info("Testing getblocktxn requests (unupgraded node)...") self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) # Need to manually sync node0 and node1, because post-segwit activation, # node1 will not download blocks from node0. - print("\tSyncing nodes...") + self.log.info("Syncing nodes...") assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash()) while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1) self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) - print("\tTesting compactblock requests (segwit node)... ") + self.log.info("Testing compactblock requests (segwit node)... ") self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True) - print("\tTesting getblocktxn requests (segwit node)...") + self.log.info("Testing getblocktxn requests (segwit node)...") self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) sync_blocks(self.nodes) - print("\tTesting getblocktxn handler (segwit node should return witnesses)...") + self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) @@ -949,18 +926,18 @@ class CompactBlocksTest(BitcoinTestFramework): # announcement to all peers. # (Post-segwit activation, blocks won't propagate from node0 to node1 # automatically, so don't bother testing a block announced to node0.) - print("\tTesting end-to-end block relay...") + self.log.info("Testing end-to-end block relay...") self.request_cb_announcements(self.test_node, self.nodes[0], 1) self.request_cb_announcements(self.old_node, self.nodes[1], 1) self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) - print("\tTesting handling of invalid compact blocks...") + self.log.info("Testing handling of invalid compact blocks...") self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) - print("\tTesting invalid index in cmpctblock message...") + self.log.info("Testing invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/qa/rpc-tests/p2p-feefilter.py b/test/functional/p2p-feefilter.py index 1c62961db0..dbccb633a5 100755 --- a/qa/rpc-tests/p2p-feefilter.py +++ b/test/functional/p2p-feefilter.py @@ -22,11 +22,9 @@ def allInvsMatch(invsExpected, testnode): time.sleep(1) return False -# TestNode: bare-bones "peer". Used to track which invs are received from a node -# and to send the node feefilter messages. -class TestNode(SingleNodeConnCB): +class TestNode(NodeConnCB): def __init__(self): - SingleNodeConnCB.__init__(self) + super().__init__() self.txinvs = [] def on_inv(self, conn, message): @@ -38,10 +36,6 @@ class TestNode(SingleNodeConnCB): with mininode_lock: self.txinvs = [] - def send_filter(self, feerate): - self.send_message(msg_feefilter(feerate)) - self.sync_with_ping() - class FeeFilterTest(BitcoinTestFramework): def __init__(self): @@ -49,14 +43,6 @@ class FeeFilterTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = False - def setup_network(self): - # Node1 will be used to generate txs which should be relayed from Node0 - # to our test node - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-logtimemicros"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-logtimemicros"])) - connect_nodes(self.nodes[0], 1) - def run_test(self): node1 = self.nodes[1] node0 = self.nodes[0] @@ -78,7 +64,7 @@ class FeeFilterTest(BitcoinTestFramework): test_node.clear_invs() # Set a filter of 15 sat/byte - test_node.send_filter(15000) + test_node.send_and_ping(msg_feefilter(15000)) # Test that txs are still being received (paying 20 sat/byte) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] @@ -103,7 +89,7 @@ class FeeFilterTest(BitcoinTestFramework): test_node.clear_invs() # Remove fee filter and check that txs are received again - test_node.send_filter(0) + test_node.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] assert(allInvsMatch(txids, test_node)) test_node.clear_invs() diff --git a/qa/rpc-tests/p2p-fullblocktest.py b/test/functional/p2p-fullblocktest.py index 274dbb8a92..e7fe7372c8 100755 --- a/qa/rpc-tests/p2p-fullblocktest.py +++ b/test/functional/p2p-fullblocktest.py @@ -398,7 +398,7 @@ class FullBlockTest(ComparisonTestFramework): # Extend the b26 chain to make sure bitcoind isn't accepting b26 b27 = block(27, spend=out[7]) - yield rejected(RejectResult(0, b'bad-prevblk')) + yield rejected(False) # Now try a too-large-coinbase script tip(15) @@ -410,7 +410,7 @@ class FullBlockTest(ComparisonTestFramework): # Extend the b28 chain to make sure bitcoind isn't accepting b28 b29 = block(29, spend=out[7]) - yield rejected(RejectResult(0, b'bad-prevblk')) + yield rejected(False) # b30 has a max-sized coinbase scriptSig. tip(23) diff --git a/qa/rpc-tests/p2p-leaktests.py b/test/functional/p2p-leaktests.py index bf08ea32f4..33b57ef33d 100755 --- a/qa/rpc-tests/p2p-leaktests.py +++ b/test/functional/p2p-leaktests.py @@ -19,23 +19,17 @@ banscore = 10 class CLazyNode(NodeConnCB): def __init__(self): - self.connection = None - self.unexpected_msg = False - self.connected = False super().__init__() - - def add_connection(self, conn): - self.connection = conn - - def send_message(self, message): - self.connection.send_message(message) + self.unexpected_msg = False + self.ever_connected = False def bad_message(self, message): self.unexpected_msg = True - print("should not have received message: %s" % message.command) + self.log.info("should not have received message: %s" % message.command) def on_open(self, conn): self.connected = True + self.ever_connected = True def on_version(self, conn, message): self.bad_message(message) def on_verack(self, conn, message): self.bad_message(message) @@ -63,9 +57,6 @@ class CLazyNode(NodeConnCB): # Node that never sends a version. We'll use this to send a bunch of messages # anyway, and eventually get disconnected. class CNodeNoVersionBan(CLazyNode): - def __init__(self): - super().__init__() - # send a bunch of veracks without sending a message. This should get us disconnected. # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes def on_open(self, conn): @@ -101,10 +92,7 @@ class P2PLeakTest(BitcoinTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - def setup_network(self): - extra_args = [['-debug', '-banscore='+str(banscore)] - for i in range(self.num_nodes)] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.extra_args = [['-banscore='+str(banscore)]] def run_test(self): no_version_bannode = CNodeNoVersionBan() @@ -121,7 +109,9 @@ class P2PLeakTest(BitcoinTestFramework): NetworkThread().start() # Start up network handling in another thread - assert(wait_until(lambda: no_version_bannode.connected and no_version_idlenode.connected and no_verack_idlenode.version_received, timeout=10)) + assert wait_until(lambda: no_version_bannode.ever_connected, timeout=10) + assert wait_until(lambda: no_version_idlenode.ever_connected, timeout=10) + assert wait_until(lambda: no_verack_idlenode.version_received, timeout=10) # Mine a block and make sure that it's not sent to the connected nodes self.nodes[0].generate(1) @@ -130,7 +120,7 @@ class P2PLeakTest(BitcoinTestFramework): time.sleep(5) #This node should have been banned - assert(no_version_bannode.connection.state == "closed") + assert not no_version_bannode.connected [conn.disconnect_node() for conn in connections] diff --git a/test/functional/p2p-mempool.py b/test/functional/p2p-mempool.py new file mode 100755 index 0000000000..34ef249eea --- /dev/null +++ b/test/functional/p2p-mempool.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test p2p mempool message. + +Test that nodes are disconnected if they send mempool messages when bloom +filters are not enabled. +""" + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class P2PMempoolTests(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [["-peerbloomfilters=0"]] + + def run_test(self): + #connect a mininode + aTestNode = NodeConnCB() + node = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], aTestNode) + aTestNode.add_connection(node) + NetworkThread().start() + aTestNode.wait_for_verack() + + #request mempool + aTestNode.send_message(msg_mempool()) + aTestNode.wait_for_disconnect() + + #mininode must be disconnected at this point + assert_equal(len(self.nodes[0].getpeerinfo()), 0) + +if __name__ == '__main__': + P2PMempoolTests().main() diff --git a/qa/rpc-tests/p2p-segwit.py b/test/functional/p2p-segwit.py index 470a5398b6..dbc61d21fc 100755 --- a/qa/rpc-tests/p2p-segwit.py +++ b/test/functional/p2p-segwit.py @@ -8,7 +8,7 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.script import * -from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, WITNESS_COMMITMENT_HEADER +from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER from test_framework.key import CECKey, CPubKey import time import random @@ -32,91 +32,25 @@ def get_virtual_size(witness_block): vsize = int((3*base_size + total_size + 3)/4) return vsize -# Note: we can reduce code by using SingleNodeConnCB (in master, not 0.12) class TestNode(NodeConnCB): def __init__(self): - NodeConnCB.__init__(self) - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong(0) - self.sleep_time = 0.05 + super().__init__() self.getdataset = set() - self.last_reject = None - - def add_connection(self, conn): - self.connection = conn - - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_inv(self, conn, message): - self.last_inv = message - - def on_block(self, conn, message): - self.last_block = message.block - self.last_block.calc_sha256() def on_getdata(self, conn, message): for inv in message.inv: self.getdataset.add(inv.hash) - self.last_getdata = message - - def on_getheaders(self, conn, message): - self.last_getheaders = message - - def on_pong(self, conn, message): - self.last_pong = message - - def on_reject(self, conn, message): - self.last_reject = message - #print (message) - - # Syncing helpers - def sync(self, test_function, timeout=60): - while timeout > 0: - with mininode_lock: - if test_function(): - return - time.sleep(self.sleep_time) - timeout -= self.sleep_time - raise AssertionError("Sync failed to complete") - - def sync_with_ping(self, timeout=60): - self.send_message(msg_ping(nonce=self.ping_counter)) - test_function = lambda: self.last_pong.nonce == self.ping_counter - self.sync(test_function, timeout) - self.ping_counter += 1 - return - - def wait_for_block(self, blockhash, timeout=60): - test_function = lambda: self.last_block != None and self.last_block.sha256 == blockhash - self.sync(test_function, timeout) - return - - def wait_for_getdata(self, timeout=60): - test_function = lambda: self.last_getdata != None - self.sync(test_function, timeout) - - def wait_for_getheaders(self, timeout=60): - test_function = lambda: self.last_getheaders != None - self.sync(test_function, timeout) - - def wait_for_inv(self, expected_inv, timeout=60): - test_function = lambda: self.last_inv != expected_inv - self.sync(test_function, timeout) def announce_tx_and_wait_for_getdata(self, tx, timeout=60): with mininode_lock: - self.last_getdata = None + self.last_message.pop("getdata", None) self.send_message(msg_inv(inv=[CInv(1, tx.sha256)])) self.wait_for_getdata(timeout) - return def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): with mininode_lock: - self.last_getdata = None - self.last_getheaders = None + self.last_message.pop("getdata", None) + self.last_message.pop("getheaders", None) msg = msg_headers() msg.headers = [ CBlockHeader(block) ] if use_header: @@ -126,24 +60,13 @@ class TestNode(NodeConnCB): self.wait_for_getheaders() self.send_message(msg) self.wait_for_getdata() - return - - def announce_block(self, block, use_header): - with mininode_lock: - self.last_getdata = None - if use_header: - msg = msg_headers() - msg.headers = [ CBlockHeader(block) ] - self.send_message(msg) - else: - self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) def request_block(self, blockhash, inv_type, timeout=60): with mininode_lock: - self.last_block = None + self.last_message.pop("block", None) self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) self.wait_for_block(blockhash, timeout) - return self.last_block + return self.last_message["block"].block def test_transaction_acceptance(self, tx, with_witness, accepted, reason=None): tx_message = msg_tx(tx) @@ -155,7 +78,7 @@ class TestNode(NodeConnCB): if (reason != None and not accepted): # Check the rejection reason as well. with mininode_lock: - assert_equal(self.last_reject.reason, reason) + assert_equal(self.last_message["reject"].reason, reason) # Test whether a witness block had the correct effect on the tip def test_witness_block(self, block, accepted, with_witness=True): @@ -166,7 +89,6 @@ class TestNode(NodeConnCB): self.sync_with_ping() assert_equal(self.connection.rpc.getbestblockhash() == block.hash, accepted) - # Used to keep track of anyone-can-spend outputs that we can use in the tests class UTXO(object): def __init__(self, sha256, n, nValue): @@ -192,17 +114,13 @@ class SegWitTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [["-whitelist=127.0.0.1"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0"], ["-whitelist=127.0.0.1", "-vbparams=segwit:0:0"]] def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-logtimemicros=1", "-whitelist=127.0.0.1"])) - # Start a node for testing IsStandard rules. - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-logtimemicros=1", "-whitelist=127.0.0.1", "-acceptnonstdtxn=0"])) + self.setup_nodes() connect_nodes(self.nodes[0], 1) - - # Disable segwit's bip9 parameter to simulate upgrading after activation. - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-whitelist=127.0.0.1", "-bip9params=segwit:0:0"])) connect_nodes(self.nodes[0], 2) + self.sync_all() ''' Helpers ''' # Build a block on top of node0's tip. @@ -224,7 +142,7 @@ class SegWitTest(BitcoinTestFramework): ''' Individual tests ''' def test_witness_services(self): - print("\tVerifying NODE_WITNESS service bit") + self.log.info("Verifying NODE_WITNESS service bit") assert((self.test_node.connection.nServices & NODE_WITNESS) != 0) @@ -233,7 +151,7 @@ class SegWitTest(BitcoinTestFramework): def test_non_witness_transaction(self): # Mine a block with an anyone-can-spend coinbase, # let it mature, then try to spend it. - print("\tTesting non-witness transaction") + self.log.info("Testing non-witness transaction") block = self.build_next_block(nVersion=1) block.solve() self.test_node.send_message(msg_block(block)) @@ -262,7 +180,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that blocks with witnesses are rejected before activation. def test_unnecessary_witness_before_segwit_activation(self): - print("\tTesting behavior of unnecessary witnesses") + self.log.info("Testing behavior of unnecessary witnesses") # For now, rely on earlier tests to have created at least one utxo for # us to use assert(len(self.utxo) > 0) @@ -288,7 +206,7 @@ class SegWitTest(BitcoinTestFramework): # TODO: fix synchronization so we can test reject reason # Right now, bitcoind delays sending reject messages for blocks # until the future, making synchronization here difficult. - #assert_equal(self.test_node.last_reject.reason, "unexpected-witness") + #assert_equal(self.test_node.last_message["reject"].reason, "unexpected-witness") # But it should not be permanently marked bad... # Resend without witness information. @@ -389,7 +307,7 @@ class SegWitTest(BitcoinTestFramework): # This test can only be run after segwit has activated def test_witness_commitments(self): - print("\tTesting witness commitments") + self.log.info("Testing witness commitments") # First try a correct witness commitment. block = self.build_next_block() @@ -478,7 +396,7 @@ class SegWitTest(BitcoinTestFramework): def test_block_malleability(self): - print("\tTesting witness block malleability") + self.log.info("Testing witness block malleability") # Make sure that a block that has too big a virtual size # because of a too-large coinbase witness is not permanently @@ -519,7 +437,7 @@ class SegWitTest(BitcoinTestFramework): def test_witness_block_size(self): - print("\tTesting witness block size limit") + self.log.info("Testing witness block size limit") # TODO: Test that non-witness carrying blocks can't exceed 1MB # Skipping this test for now; this is covered in p2p-fullblocktest.py @@ -636,7 +554,7 @@ class SegWitTest(BitcoinTestFramework): # Consensus tests of extra witness data in a transaction. def test_extra_witness_data(self): - print("\tTesting extra witness data in tx") + self.log.info("Testing extra witness data in tx") assert(len(self.utxo) > 0) @@ -712,7 +630,7 @@ class SegWitTest(BitcoinTestFramework): def test_max_witness_push_length(self): ''' Should only allow up to 520 byte pushes in witness stack ''' - print("\tTesting maximum witness push size") + self.log.info("Testing maximum witness push size") MAX_SCRIPT_ELEMENT_SIZE = 520 assert(len(self.utxo)) @@ -752,7 +670,7 @@ class SegWitTest(BitcoinTestFramework): def test_max_witness_program_length(self): # Can create witness outputs that are long, but can't be greater than # 10k bytes to successfully spend - print("\tTesting maximum witness program length") + self.log.info("Testing maximum witness program length") assert(len(self.utxo)) MAX_PROGRAM_LENGTH = 10000 @@ -801,7 +719,7 @@ class SegWitTest(BitcoinTestFramework): def test_witness_input_length(self): ''' Ensure that vin length must match vtxinwit length ''' - print("\tTesting witness input length") + self.log.info("Testing witness input length") assert(len(self.utxo)) witness_program = CScript([OP_DROP, OP_TRUE]) @@ -884,7 +802,7 @@ class SegWitTest(BitcoinTestFramework): def test_witness_tx_relay_before_segwit_activation(self): - print("\tTesting relay of witness transactions") + self.log.info("Testing relay of witness transactions") # Generate a transaction that doesn't require a witness, but send it # with a witness. Should be rejected for premature-witness, but should # not be added to recently rejected list. @@ -902,13 +820,13 @@ class SegWitTest(BitcoinTestFramework): # Verify that if a peer doesn't set nServices to include NODE_WITNESS, # the getdata is just for the non-witness portion. self.old_node.announce_tx_and_wait_for_getdata(tx) - assert(self.old_node.last_getdata.inv[0].type == 1) + assert(self.old_node.last_message["getdata"].inv[0].type == 1) # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. try: self.test_node.announce_tx_and_wait_for_getdata(tx, timeout=2) - print("Error: duplicate tx getdata!") + self.log.error("Error: duplicate tx getdata!") assert(False) except AssertionError as e: pass @@ -936,7 +854,7 @@ class SegWitTest(BitcoinTestFramework): # - accepts transactions with valid witnesses # and that witness transactions are relayed to non-upgraded peers. def test_tx_relay_after_segwit_activation(self): - print("\tTesting relay of witness transactions") + self.log.info("Testing relay of witness transactions") # Generate a transaction that doesn't require a witness, but send it # with a witness. Should be rejected because we can't use a witness # when spending a non-witness output. @@ -998,9 +916,9 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit[0].scriptWitness.stack = [ witness_program ] # Also check that old_node gets a tx announcement, even though this is # a witness transaction. - self.old_node.wait_for_inv(CInv(1, tx2.sha256)) # wait until tx2 was inv'ed + self.old_node.wait_for_inv([CInv(1, tx2.sha256)]) # wait until tx2 was inv'ed self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) - self.old_node.wait_for_inv(CInv(1, tx3.sha256)) + self.old_node.wait_for_inv([CInv(1, tx3.sha256)]) # Test that getrawtransaction returns correct witness information # hash, size, vsize @@ -1025,7 +943,7 @@ class SegWitTest(BitcoinTestFramework): # This is true regardless of segwit activation. # Also test that we don't ask for blocks from unupgraded peers def test_block_relay(self, segwit_activated): - print("\tTesting block relay") + self.log.info("Testing block relay") blocktype = 2|MSG_WITNESS_FLAG @@ -1037,20 +955,20 @@ class SegWitTest(BitcoinTestFramework): block1.solve() self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) - assert(self.test_node.last_getdata.inv[0].type == blocktype) + assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) self.test_node.test_witness_block(block1, True) block2 = self.build_next_block(nVersion=4) block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) - assert(self.test_node.last_getdata.inv[0].type == blocktype) + assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) self.test_node.test_witness_block(block2, True) block3 = self.build_next_block(nVersion=(VB_TOP_BITS | (1<<15))) block3.solve() self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) - assert(self.test_node.last_getdata.inv[0].type == blocktype) + assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) self.test_node.test_witness_block(block3, True) # Check that we can getdata for witness blocks or regular blocks, @@ -1101,19 +1019,24 @@ class SegWitTest(BitcoinTestFramework): block4 = self.build_next_block(nVersion=4) block4.solve() self.old_node.getdataset = set() + # Blocks can be requested via direct-fetch (immediately upon processing the announcement) # or via parallel download (with an indeterminate delay from processing the announcement) # so to test that a block is NOT requested, we could guess a time period to sleep for, # and then check. We can avoid the sleep() by taking advantage of transaction getdata's # being processed after block getdata's, and announce a transaction as well, # and then check to see if that particular getdata has been received. - self.old_node.announce_block(block4, use_header=False) + # Since 0.14, inv's will only be responded to with a getheaders, so send a header + # to announce this block. + msg = msg_headers() + msg.headers = [ CBlockHeader(block4) ] + self.old_node.send_message(msg) self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) assert(block4.sha256 not in self.old_node.getdataset) # V0 segwit outputs should be standard after activation, but not before. def test_standardness_v0(self, segwit_activated): - print("\tTesting standardness of v0 outputs (%s activation)" % ("after" if segwit_activated else "before")) + self.log.info("Testing standardness of v0 outputs (%s activation)" % ("after" if segwit_activated else "before")) assert(len(self.utxo)) witness_program = CScript([OP_TRUE]) @@ -1190,7 +1113,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that future segwit upgraded transactions are non-standard, # but valid in blocks. Can run this before and after segwit activation. def test_segwit_versions(self): - print("\tTesting standardness/consensus for segwit versions (0-16)") + self.log.info("Testing standardness/consensus for segwit versions (0-16)") assert(len(self.utxo)) NUM_TESTS = 17 # will test OP_0, OP1, ..., OP_16 if (len(self.utxo) < NUM_TESTS): @@ -1261,7 +1184,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=False) self.test_node.sync_with_ping() with mininode_lock: - assert(b"reserved for soft-fork upgrades" in self.test_node.last_reject.reason) + assert(b"reserved for soft-fork upgrades" in self.test_node.last_message["reject"].reason) # Building a block with the transaction must be valid, however. block = self.build_next_block() @@ -1274,7 +1197,7 @@ class SegWitTest(BitcoinTestFramework): def test_premature_coinbase_witness_spend(self): - print("\tTesting premature coinbase witness spend") + self.log.info("Testing premature coinbase witness spend") block = self.build_next_block() # Change the output of the block to be a witness output. witness_program = CScript([OP_TRUE]) @@ -1309,7 +1232,7 @@ class SegWitTest(BitcoinTestFramework): def test_signature_version_1(self): - print("\tTesting segwit signature hash version 1") + self.log.info("Testing segwit signature hash version 1") key = CECKey() key.set_secretbytes(b"9") pubkey = CPubKey(key.get_pubkey()) @@ -1428,7 +1351,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() if (not used_sighash_single_out_of_bounds): - print("WARNING: this test run didn't attempt SIGHASH_SINGLE with out-of-bounds index value") + self.log.info("WARNING: this test run didn't attempt SIGHASH_SINGLE with out-of-bounds index value") # Test the transactions we've added to the block if (len(block.vtx) > 1): self.update_witness_block_with_transactions(block, []) @@ -1491,7 +1414,7 @@ class SegWitTest(BitcoinTestFramework): # Test P2SH wrapped witness programs. def test_p2sh_witness(self, segwit_activated): - print("\tTesting P2SH witness transactions") + self.log.info("Testing P2SH witness transactions") assert(len(self.utxo)) @@ -1564,7 +1487,7 @@ class SegWitTest(BitcoinTestFramework): # To enable this test, pass --oldbinary=<path-to-pre-segwit-bitcoind> to # the test. def test_upgrade_after_activation(self, node, node_id): - print("\tTesting software upgrade after softfork activation") + self.log.info("Testing software upgrade after softfork activation") assert(node_id != 0) # node0 is assumed to be a segwit-active bitcoind @@ -1572,8 +1495,8 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) # Restart with the new binary - stop_node(node, node_id) - self.nodes[node_id] = start_node(node_id, self.options.tmpdir, ["-debug"]) + self.stop_node(node_id) + self.nodes[node_id] = self.start_node(node_id, self.options.tmpdir) connect_nodes(self.nodes[0], node_id) sync_blocks(self.nodes) @@ -1592,7 +1515,7 @@ class SegWitTest(BitcoinTestFramework): def test_witness_sigops(self): '''Ensure sigop counting is correct inside witnesses.''' - print("\tTesting sigops limit") + self.log.info("Testing sigops limit") assert(len(self.utxo)) @@ -1694,14 +1617,16 @@ class SegWitTest(BitcoinTestFramework): # TODO: test p2sh sigop counting def test_getblocktemplate_before_lockin(self): - print("\tTesting getblocktemplate setting of segwit versionbit (before lockin)") + self.log.info("Testing getblocktemplate setting of segwit versionbit (before lockin)") # Node0 is segwit aware, node2 is not. for node in [self.nodes[0], self.nodes[2]]: gbt_results = node.getblocktemplate() block_version = gbt_results['version'] - # If we're not indicating segwit support, we should not be signalling - # for segwit activation, nor should we get a witness commitment. - assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) + # If we're not indicating segwit support, we will still be + # signalling for segwit activation. + assert_equal((block_version & (1 << VB_WITNESS_BIT) != 0), node == self.nodes[0]) + # If we don't specify the segwit rule, then we won't get a default + # commitment. assert('default_witness_commitment' not in gbt_results) # Workaround: @@ -1728,15 +1653,10 @@ class SegWitTest(BitcoinTestFramework): assert('default_witness_commitment' in gbt_results) witness_commitment = gbt_results['default_witness_commitment'] - # TODO: this duplicates some code from blocktools.py, would be nice - # to refactor. # Check that default_witness_commitment is present. - block = CBlock() - witness_root = block.get_merkle_root([ser_uint256(0), ser_uint256(txid)]) - check_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(0))) - from test_framework.blocktools import WITNESS_COMMITMENT_HEADER - output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(check_commitment) - script = CScript([OP_RETURN, output_data]) + witness_root = CBlock.get_merkle_root([ser_uint256(0), + ser_uint256(txid)]) + script = get_witness_script(witness_root, 0) assert_equal(witness_commitment, bytes_to_hex_str(script)) # undo mocktime @@ -1746,7 +1666,7 @@ class SegWitTest(BitcoinTestFramework): # Uncompressed pubkeys are no longer supported in default relay policy, # but (for now) are still valid in blocks. def test_uncompressed_pubkey(self): - print("\tTesting uncompressed pubkeys") + self.log.info("Testing uncompressed pubkeys") # Segwit transactions using uncompressed pubkeys are not accepted # under default policy, but should still pass consensus. key = CECKey() @@ -1848,7 +1768,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.append(UTXO(tx5.sha256, 0, tx5.vout[0].nValue)) def test_non_standard_witness(self): - print("\tTesting detection of non-standard P2WSH witness") + self.log.info("Testing detection of non-standard P2WSH witness") pad = chr(1).encode('latin-1') # Create scripts for tests @@ -1972,7 +1892,7 @@ class SegWitTest(BitcoinTestFramework): # Test logic begins here self.test_node.wait_for_verack() - print("\nStarting tests before segwit lock in:") + self.log.info("Starting tests before segwit lock in:") self.test_witness_services() # Verifies NODE_WITNESS self.test_non_witness_transaction() # non-witness tx's are accepted @@ -1987,7 +1907,7 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) # At lockin, nothing should change. - print("\nTesting behavior post lockin, pre-activation") + self.log.info("Testing behavior post lockin, pre-activation") self.advance_to_segwit_lockin() # Retest unnecessary witnesses @@ -2000,7 +1920,7 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) # Now activate segwit - print("\nTesting behavior after segwit activation") + self.log.info("Testing behavior after segwit activation") self.advance_to_segwit_active() sync_blocks(self.nodes) diff --git a/qa/rpc-tests/p2p-timeouts.py b/test/functional/p2p-timeouts.py index 7f596b6e4b..c3b29c215b 100755 --- a/qa/rpc-tests/p2p-timeouts.py +++ b/test/functional/p2p-timeouts.py @@ -27,21 +27,10 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class TestNode(SingleNodeConnCB): - def __init__(self): - SingleNodeConnCB.__init__(self) - self.connected = False - self.received_version = False - - def on_open(self, conn): - self.connected = True - - def on_close(self, conn): - self.connected = False - +class TestNode(NodeConnCB): def on_version(self, conn, message): # Don't send a verack in response - self.received_version = True + pass class TimeoutsTest(BitcoinTestFramework): def __init__(self): @@ -49,13 +38,6 @@ class TimeoutsTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self): - self.nodes = [] - - # Start up node0 to be a version 1, pre-segwit node. - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - [["-debug", "-logtimemicros=1"]]) - def run_test(self): # Setup the p2p connections and start up the network thread. self.no_verack_node = TestNode() # never send verack @@ -84,7 +66,7 @@ class TimeoutsTest(BitcoinTestFramework): sleep(30) - assert(self.no_verack_node.received_version) + assert "version" in self.no_verack_node.last_message assert(self.no_verack_node.connected) assert(self.no_version_node.connected) diff --git a/qa/rpc-tests/p2p-versionbits-warning.py b/test/functional/p2p-versionbits-warning.py index 8e3e361fc1..df7e8ce5c1 100755 --- a/qa/rpc-tests/p2p-versionbits-warning.py +++ b/test/functional/p2p-versionbits-warning.py @@ -12,7 +12,6 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * import re -import time from test_framework.blocktools import create_block, create_coinbase VB_PERIOD = 144 # versionbits period length for regtest @@ -24,43 +23,10 @@ WARN_UNKNOWN_RULES_MINED = "Unknown block versions being mined! It's possible un WARN_UNKNOWN_RULES_ACTIVE = "unknown new rules activated (versionbit {})".format(VB_UNKNOWN_BIT) VB_PATTERN = re.compile("^Warning.*versionbit") -# TestNode: bare-bones "peer". Used mostly as a conduit for a test to sending -# p2p messages to a node, generating the messages in the main testing logic. class TestNode(NodeConnCB): - def __init__(self): - NodeConnCB.__init__(self) - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() - - def add_connection(self, conn): - self.connection = conn - def on_inv(self, conn, message): pass - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_pong(self, conn, message): - self.last_pong = message - - # Sync up with the node after delivery of a block - def sync_with_ping(self, timeout=30): - self.connection.send_message(msg_ping(nonce=self.ping_counter)) - received_pong = False - sleep_time = 0.05 - while not received_pong and timeout > 0: - time.sleep(sleep_time) - timeout -= sleep_time - with mininode_lock: - if self.last_pong.nonce == self.ping_counter: - received_pong = True - self.ping_counter += 1 - return received_pong - - class VersionBitsWarningTest(BitcoinTestFramework): def __init__(self): super().__init__() @@ -72,8 +38,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Open and close to create zero-length file with open(self.alert_filename, 'w', encoding='utf8') as _: pass - self.extra_args = [["-debug", "-logtimemicros=1", "-alertnotify=echo %s >> \"" + self.alert_filename + "\""]] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.extra_args = [["-alertnotify=echo %s >> \"" + self.alert_filename + "\""]] + self.setup_nodes() # Send numblocks blocks via peer with nVersionToUse set. def send_blocks_with_version(self, peer, numblocks, nVersionToUse): @@ -142,22 +108,22 @@ class VersionBitsWarningTest(BitcoinTestFramework): # is cleared, and restart the node. This should move the versionbit state # to ACTIVE. self.nodes[0].generate(VB_PERIOD) - stop_nodes(self.nodes) + self.stop_nodes() # Empty out the alert file with open(self.alert_filename, 'w', encoding='utf8') as _: pass - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) # Connecting one block should be enough to generate an error. self.nodes[0].generate(1) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getinfo()["errors"]) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getmininginfo()["errors"]) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getnetworkinfo()["warnings"]) - stop_nodes(self.nodes) + self.stop_nodes() self.test_versionbits_in_alert_file() # Test framework expects the node to still be running... - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) if __name__ == '__main__': VersionBitsWarningTest().main() diff --git a/qa/rpc-tests/preciousblock.py b/test/functional/preciousblock.py index dde164f329..04b41e76ba 100755 --- a/qa/rpc-tests/preciousblock.py +++ b/test/functional/preciousblock.py @@ -39,13 +39,12 @@ class PreciousTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-debug"]] * self.num_nodes def setup_network(self): - self.nodes = self.setup_nodes() + self.setup_nodes() def run_test(self): - print("Ensure submitblock can in principle reorg to a competing chain") + self.log.info("Ensure submitblock can in principle reorg to a competing chain") self.nodes[0].generate(1) assert_equal(self.nodes[0].getblockcount(), 1) (hashY, hashZ) = self.nodes[1].generate(2) @@ -53,62 +52,62 @@ class PreciousTest(BitcoinTestFramework): node_sync_via_rpc(self.nodes[0:3]) assert_equal(self.nodes[0].getbestblockhash(), hashZ) - print("Mine blocks A-B-C on Node 0") + self.log.info("Mine blocks A-B-C on Node 0") (hashA, hashB, hashC) = self.nodes[0].generate(3) assert_equal(self.nodes[0].getblockcount(), 5) - print("Mine competing blocks E-F-G on Node 1") + self.log.info("Mine competing blocks E-F-G on Node 1") (hashE, hashF, hashG) = self.nodes[1].generate(3) assert_equal(self.nodes[1].getblockcount(), 5) assert(hashC != hashG) - print("Connect nodes and check no reorg occurs") + self.log.info("Connect nodes and check no reorg occurs") # Submit competing blocks via RPC so any reorg should occur before we proceed (no way to wait on inaction for p2p sync) node_sync_via_rpc(self.nodes[0:2]) connect_nodes_bi(self.nodes,0,1) assert_equal(self.nodes[0].getbestblockhash(), hashC) assert_equal(self.nodes[1].getbestblockhash(), hashG) - print("Make Node0 prefer block G") + self.log.info("Make Node0 prefer block G") self.nodes[0].preciousblock(hashG) assert_equal(self.nodes[0].getbestblockhash(), hashG) - print("Make Node0 prefer block C again") + self.log.info("Make Node0 prefer block C again") self.nodes[0].preciousblock(hashC) assert_equal(self.nodes[0].getbestblockhash(), hashC) - print("Make Node1 prefer block C") + self.log.info("Make Node1 prefer block C") self.nodes[1].preciousblock(hashC) sync_chain(self.nodes[0:2]) # wait because node 1 may not have downloaded hashC assert_equal(self.nodes[1].getbestblockhash(), hashC) - print("Make Node1 prefer block G again") + self.log.info("Make Node1 prefer block G again") self.nodes[1].preciousblock(hashG) assert_equal(self.nodes[1].getbestblockhash(), hashG) - print("Make Node0 prefer block G again") + self.log.info("Make Node0 prefer block G again") self.nodes[0].preciousblock(hashG) assert_equal(self.nodes[0].getbestblockhash(), hashG) - print("Make Node1 prefer block C again") + self.log.info("Make Node1 prefer block C again") self.nodes[1].preciousblock(hashC) assert_equal(self.nodes[1].getbestblockhash(), hashC) - print("Mine another block (E-F-G-)H on Node 0 and reorg Node 1") + self.log.info("Mine another block (E-F-G-)H on Node 0 and reorg Node 1") self.nodes[0].generate(1) assert_equal(self.nodes[0].getblockcount(), 6) sync_blocks(self.nodes[0:2]) hashH = self.nodes[0].getbestblockhash() assert_equal(self.nodes[1].getbestblockhash(), hashH) - print("Node1 should not be able to prefer block C anymore") + self.log.info("Node1 should not be able to prefer block C anymore") self.nodes[1].preciousblock(hashC) assert_equal(self.nodes[1].getbestblockhash(), hashH) - print("Mine competing blocks I-J-K-L on Node 2") + self.log.info("Mine competing blocks I-J-K-L on Node 2") self.nodes[2].generate(4) assert_equal(self.nodes[2].getblockcount(), 6) hashL = self.nodes[2].getbestblockhash() - print("Connect nodes and check no reorg occurs") + self.log.info("Connect nodes and check no reorg occurs") node_sync_via_rpc(self.nodes[1:3]) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) assert_equal(self.nodes[0].getbestblockhash(), hashH) assert_equal(self.nodes[1].getbestblockhash(), hashH) assert_equal(self.nodes[2].getbestblockhash(), hashL) - print("Make Node1 prefer block L") + self.log.info("Make Node1 prefer block L") self.nodes[1].preciousblock(hashL) assert_equal(self.nodes[1].getbestblockhash(), hashL) - print("Make Node2 prefer block H") + self.log.info("Make Node2 prefer block H") self.nodes[2].preciousblock(hashH) assert_equal(self.nodes[2].getbestblockhash(), hashH) diff --git a/qa/rpc-tests/prioritise_transaction.py b/test/functional/prioritise_transaction.py index 9a63d0838f..4fc03d2547 100755 --- a/qa/rpc-tests/prioritise_transaction.py +++ b/test/functional/prioritise_transaction.py @@ -13,18 +13,13 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True - self.num_nodes = 1 + self.num_nodes = 2 + self.extra_args = [["-printpriority=1"], ["-printpriority=1"]] + def run_test(self): self.txouts = gen_return_txouts() - - def setup_network(self): - self.nodes = [] - self.is_network_split = False - - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-printpriority=1"])) self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] - def run_test(self): utxo_count = 90 utxos = create_confirmed_utxos(self.relayfee, self.nodes[0], utxo_count) base_fee = self.relayfee*100 # our transactions are smaller than 100kb @@ -50,15 +45,13 @@ class PrioritiseTransactionTest(BitcoinTestFramework): assert(sizes[i] > MAX_BLOCK_BASE_SIZE) # Fail => raise utxo_count # add a fee delta to something in the cheapest bucket and make sure it gets mined - # also check that a different entry in the cheapest bucket is NOT mined (lower - # the priority to ensure its not mined due to priority) - self.nodes[0].prioritisetransaction(txids[0][0], 0, int(3*base_fee*COIN)) - self.nodes[0].prioritisetransaction(txids[0][1], -1e15, 0) + # also check that a different entry in the cheapest bucket is NOT mined + self.nodes[0].prioritisetransaction(txid=txids[0][0], fee_delta=int(3*base_fee*COIN)) self.nodes[0].generate(1) mempool = self.nodes[0].getrawmempool() - print("Assert that prioritised transaction was mined") + self.log.info("Assert that prioritised transaction was mined") assert(txids[0][0] not in mempool) assert(txids[0][1] in mempool) @@ -72,7 +65,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). - self.nodes[0].prioritisetransaction(high_fee_tx, -1e15, -int(2*base_fee*COIN)) + self.nodes[0].prioritisetransaction(txid=high_fee_tx, fee_delta=-int(2*base_fee*COIN)) # Add everything back to mempool self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) @@ -90,13 +83,13 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # High fee transaction should not have been mined, but other high fee rate # transactions should have been. mempool = self.nodes[0].getrawmempool() - print("Assert that de-prioritised transaction is still in mempool") + self.log.info("Assert that de-prioritised transaction is still in mempool") assert(high_fee_tx in mempool) for x in txids[2]: if (x != high_fee_tx): assert(x not in mempool) - # Create a free, low priority transaction. Should be rejected. + # Create a free transaction. Should be rejected. utxo_list = self.nodes[0].listunspent() assert(len(utxo_list) > 0) utxo = utxo_list[0] @@ -104,37 +97,34 @@ class PrioritiseTransactionTest(BitcoinTestFramework): inputs = [] outputs = {} inputs.append({"txid" : utxo["txid"], "vout" : utxo["vout"]}) - outputs[self.nodes[0].getnewaddress()] = utxo["amount"] - self.relayfee + outputs[self.nodes[0].getnewaddress()] = utxo["amount"] raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) tx_hex = self.nodes[0].signrawtransaction(raw_tx)["hex"] - txid = self.nodes[0].sendrawtransaction(tx_hex) + tx_id = self.nodes[0].decoderawtransaction(tx_hex)["txid"] - # A tx that spends an in-mempool tx has 0 priority, so we can use it to - # test the effect of using prioritise transaction for mempool acceptance - inputs = [] - inputs.append({"txid": txid, "vout": 0}) - outputs = {} - outputs[self.nodes[0].getnewaddress()] = utxo["amount"] - self.relayfee - raw_tx2 = self.nodes[0].createrawtransaction(inputs, outputs) - tx2_hex = self.nodes[0].signrawtransaction(raw_tx2)["hex"] - tx2_id = self.nodes[0].decoderawtransaction(tx2_hex)["txid"] - - try: - self.nodes[0].sendrawtransaction(tx2_hex) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) # insufficient fee - assert(tx2_id not in self.nodes[0].getrawmempool()) - else: - assert(False) + # This will raise an exception due to min relay fee not being met + assert_raises_jsonrpc(-26, "66: min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) + assert(tx_id not in self.nodes[0].getrawmempool()) # This is a less than 1000-byte transaction, so just set the fee # to be the minimum for a 1000 byte transaction and check that it is # accepted. - self.nodes[0].prioritisetransaction(tx2_id, 0, int(self.relayfee*COIN)) - - print("Assert that prioritised free transaction is accepted to mempool") - assert_equal(self.nodes[0].sendrawtransaction(tx2_hex), tx2_id) - assert(tx2_id in self.nodes[0].getrawmempool()) + self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=int(self.relayfee*COIN)) + + self.log.info("Assert that prioritised free transaction is accepted to mempool") + assert_equal(self.nodes[0].sendrawtransaction(tx_hex), tx_id) + assert(tx_id in self.nodes[0].getrawmempool()) + + # Test that calling prioritisetransaction is sufficient to trigger + # getblocktemplate to (eventually) return a new block. + mock_time = int(time.time()) + self.nodes[0].setmocktime(mock_time) + template = self.nodes[0].getblocktemplate() + self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=-int(self.relayfee*COIN)) + self.nodes[0].setmocktime(mock_time+10) + new_template = self.nodes[0].getblocktemplate() + + assert(template != new_template) if __name__ == '__main__': PrioritiseTransactionTest().main() diff --git a/qa/rpc-tests/proxy_test.py b/test/functional/proxy_test.py index e4e231f312..ae6f843ddc 100755 --- a/qa/rpc-tests/proxy_test.py +++ b/test/functional/proxy_test.py @@ -35,7 +35,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( PORT_MIN, PORT_RANGE, - start_nodes, assert_equal, ) from test_framework.netutil import test_ipv6_local @@ -49,6 +48,7 @@ class ProxyTest(BitcoinTestFramework): self.num_nodes = 4 self.setup_clean_chain = False + def setup_nodes(self): self.have_ipv6 = test_ipv6_local() # Create two proxies on different ports # ... one unauthenticated @@ -69,7 +69,7 @@ class ProxyTest(BitcoinTestFramework): self.conf3.unauth = True self.conf3.auth = True else: - print("Warning: testing without local IPv6 support") + self.log.warning("Testing without local IPv6 support") self.serv1 = Socks5Server(self.conf1) self.serv1.start() @@ -79,18 +79,17 @@ class ProxyTest(BitcoinTestFramework): self.serv3 = Socks5Server(self.conf3) self.serv3.start() - def setup_nodes(self): # Note: proxies are not used to connect to local nodes # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost args = [ - ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], - ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], - ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], + ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], + ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], + ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], [] ] if self.have_ipv6: - args[3] = ['-listen', '-debug=net', '-debug=proxy', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] - return start_nodes(self.num_nodes, self.options.tmpdir, extra_args=args) + args[3] = ['-listen', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args=args) def node_test(self, node, proxies, auth, test_onion=True): rv = [] diff --git a/qa/rpc-tests/pruning.py b/test/functional/pruning.py index ace8ced422..4c3501ad77 100755 --- a/qa/rpc-tests/pruning.py +++ b/test/functional/pruning.py @@ -19,7 +19,7 @@ MIN_BLOCKS_TO_KEEP = 288 # Rescans start at the earliest block up to 2 hours before a key timestamp, so # the manual prune RPC avoids pruning blocks in the same window to be # compatible with pruning based on key creation time. -RESCAN_WINDOW = 2 * 60 * 60 +TIMESTAMP_WINDOW = 2 * 60 * 60 def calc_usage(blockdir): @@ -32,31 +32,22 @@ class PruneTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 6 - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache_0 = [] - self.utxo_cache_1 = [] - - def setup_network(self): - self.nodes = [] - self.is_network_split = False - - # Create nodes 0 and 1 to mine - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) - - # Create node 2 to test pruning - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900)) - self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/" - + # Create nodes 0 and 1 to mine. + # Create node 2 to test pruning. + self.full_node_default_args = ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000" ] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) - self.nodes.append(start_node(3, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900)) - self.nodes.append(start_node(4, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900)) - # Create nodes 5 to test wallet in prune mode, but do not connect - self.nodes.append(start_node(5, self.options.tmpdir, ["-debug=0", "-prune=550"])) + self.extra_args = [self.full_node_default_args, + self.full_node_default_args, + ["-maxreceivebuffer=20000", "-prune=550"], + ["-maxreceivebuffer=20000", "-blockmaxsize=999000"], + ["-maxreceivebuffer=20000", "-blockmaxsize=999000"], + ["-prune=550"]] - # Determine default relay fee - self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] + def setup_network(self): + self.setup_nodes() + + self.prunedir = self.options.tmpdir + "/node2/regtest/blocks/" connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) @@ -79,9 +70,9 @@ class PruneTest(BitcoinTestFramework): def test_height_min(self): if not os.path.isfile(self.prunedir+"blk00000.dat"): raise AssertionError("blk00000.dat is missing, pruning too early") - print("Success") - print("Though we're already using more than 550MiB, current usage:", calc_usage(self.prunedir)) - print("Mining 25 more blocks should cause the first block file to be pruned") + self.log.info("Success") + self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir)) + self.log.info("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this for i in range(25): mine_large_block(self.nodes[0], self.utxo_cache_0) @@ -92,27 +83,30 @@ class PruneTest(BitcoinTestFramework): if time.time() - waitstart > 30: raise AssertionError("blk00000.dat not pruned when it should be") - print("Success") + self.log.info("Success") usage = calc_usage(self.prunedir) - print("Usage should be below target:", usage) + self.log.info("Usage should be below target: %d" % usage) if (usage > 550): raise AssertionError("Pruning target not being met") def create_chain_with_staleblocks(self): # Create stale blocks in manageable sized chunks - print("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds") + self.log.info("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds") for j in range(12): # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine self.stop_node(0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900) + self.nodes[0]=self.start_node(0, self.options.tmpdir, self.full_node_default_args, timewait=900) # Mine 24 blocks in node 1 for i in range(24): if j == 0: mine_large_block(self.nodes[1], self.utxo_cache_1) else: + # Add node1's wallet transactions back to the mempool, to + # avoid the mined blocks from being too small. + self.nodes[1].resendwallettransactions() self.nodes[1].generate(1) #tx's already in mempool from previous disconnects # Reorg back with 25 block chain from node 0 @@ -124,7 +118,7 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[2], 0) sync_blocks(self.nodes[0:3]) - print("Usage can be over target because of high stale rate:", calc_usage(self.prunedir)) + self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir)) def reorg_test(self): # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip @@ -132,14 +126,14 @@ class PruneTest(BitcoinTestFramework): # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) self.stop_node(1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) + self.nodes[1] = self.start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) height = self.nodes[1].getblockcount() - print("Current block height:", height) + self.log.info("Current block height: %d" % height) invalidheight = height-287 badhash = self.nodes[1].getblockhash(invalidheight) - print("Invalidating block at height:",invalidheight,badhash) + self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight)) self.nodes[1].invalidateblock(badhash) # We've now switched to our previously mined-24 block fork on node 1, but thats not what we want @@ -151,24 +145,29 @@ class PruneTest(BitcoinTestFramework): curhash = self.nodes[1].getblockhash(invalidheight - 1) assert(self.nodes[1].getblockcount() == invalidheight - 1) - print("New best height", self.nodes[1].getblockcount()) + self.log.info("New best height: %d" % self.nodes[1].getblockcount()) # Reboot node1 to clear those giant tx's from mempool self.stop_node(1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) + self.nodes[1] = self.start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) - print("Generating new longer chain of 300 more blocks") + self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) - print("Reconnect nodes") + self.log.info("Reconnect nodes") connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[2], 1) sync_blocks(self.nodes[0:3], timeout=120) - print("Verify height on node 2:",self.nodes[2].getblockcount()) - print("Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir)) + self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount()) + self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir)) + + self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") + + # Get node0's wallet transactions back in its mempool, to avoid the + # mined blocks from being too small. + self.nodes[0].resendwallettransactions() - print("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") for i in range(22): # This can be slow, so do this in multiple RPC calls to avoid # RPC timeouts. @@ -176,7 +175,7 @@ class PruneTest(BitcoinTestFramework): sync_blocks(self.nodes[0:3], timeout=300) usage = calc_usage(self.prunedir) - print("Usage should be below target:", usage) + self.log.info("Usage should be below target: %d" % usage) if (usage > 550): raise AssertionError("Pruning target not being met") @@ -184,11 +183,8 @@ class PruneTest(BitcoinTestFramework): def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away - try: - self.nodes[2].getblock(self.forkhash) - raise AssertionError("Old block wasn't pruned so can't test redownload") - except JSONRPCException as e: - print("Will need to redownload block",self.forkheight) + assert_raises_jsonrpc(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) + self.log.info("Will need to redownload block %d" % self.forkheight) # Verify that we have enough history to reorg back to the fork point # Although this is more than 288 blocks, because this chain was written more recently @@ -212,14 +208,14 @@ class PruneTest(BitcoinTestFramework): # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg if self.nodes[2].getblockcount() < self.mainchainheight: blocks_to_mine = first_reorg_height + 1 - self.mainchainheight - print("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine) + self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine) self.nodes[0].invalidateblock(curchainhash) assert(self.nodes[0].getblockcount() == self.mainchainheight) assert(self.nodes[0].getbestblockhash() == self.mainchainhash2) goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 - print("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") + self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") waitstart = time.time() while self.nodes[2].getblockcount() < goalbestheight: time.sleep(0.1) @@ -231,18 +227,18 @@ class PruneTest(BitcoinTestFramework): def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode - node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0"], timewait=900) + node = self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, timewait=900) assert_equal(node.getblockcount(), 995) - assert_raises_message(JSONRPCException, "not in prune mode", node.pruneblockchain, 500) + assert_raises_jsonrpc(-1, "not in prune mode", node.pruneblockchain, 500) self.stop_node(node_number) # now re-start in manual pruning mode - node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0","-prune=1"], timewait=900) + node = self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, ["-prune=1"], timewait=900) assert_equal(node.getblockcount(), 995) def height(index): if use_timestamp: - return node.getblockheader(node.getblockhash(index))["time"] + RESCAN_WINDOW + return node.getblockheader(node.getblockhash(index))["time"] + TIMESTAMP_WINDOW else: return index @@ -265,14 +261,14 @@ class PruneTest(BitcoinTestFramework): return os.path.isfile(self.options.tmpdir + "/node{}/regtest/blocks/blk{:05}.dat".format(node_number, index)) # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) - assert_raises_message(JSONRPCException, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) + assert_raises_jsonrpc(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight) node.generate(6) assert_equal(node.getblockchaininfo()["blocks"], 1001) # negative heights should raise an exception - assert_raises_message(JSONRPCException, "Negative", node.pruneblockchain, -10) + assert_raises_jsonrpc(-8, "Negative", node.pruneblockchain, -10) # height=100 too low to prune first block file so this is a no-op prune(100) @@ -311,36 +307,38 @@ class PruneTest(BitcoinTestFramework): # stop node, start back up with auto-prune at 550MB, make sure still runs self.stop_node(node_number) - self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0","-prune=550"], timewait=900) + self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, ["-prune=550"], timewait=900) - print("Success") + self.log.info("Success") def wallet_test(self): # check that the pruning node's wallet is still in good shape - print("Stop and start pruning node to trigger wallet rescan") - try: - self.stop_node(2) - start_node(2, self.options.tmpdir, ["-debug=1","-prune=550"]) - print("Success") - except Exception as detail: - raise AssertionError("Wallet test: unable to re-start the pruning node") + self.log.info("Stop and start pruning node to trigger wallet rescan") + self.stop_node(2) + self.start_node(2, self.options.tmpdir, ["-prune=550"]) + self.log.info("Success") # check that wallet loads loads successfully when restarting a pruned node after IBD. # this was reported to fail in #7494. - print ("Syncing node 5 to test wallet") + self.log.info("Syncing node 5 to test wallet") connect_nodes(self.nodes[0], 5) nds = [self.nodes[0], self.nodes[5]] sync_blocks(nds, wait=5, timeout=300) - try: - self.stop_node(5) #stop and start to trigger rescan - start_node(5, self.options.tmpdir, ["-debug=1","-prune=550"]) - print ("Success") - except Exception as detail: - raise AssertionError("Wallet test: unable to re-start node5") + self.stop_node(5) #stop and start to trigger rescan + self.start_node(5, self.options.tmpdir, ["-prune=550"]) + self.log.info("Success") def run_test(self): - print("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") - print("Mining a big blockchain of 995 blocks") + self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") + self.log.info("Mining a big blockchain of 995 blocks") + + # Determine default relay fee + self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] + + # Cache for utxos, as the listunspent may take a long time later in the test + self.utxo_cache_0 = [] + self.utxo_cache_1 = [] + self.create_big_chain() # Chain diagram key: # * blocks on main chain @@ -355,12 +353,12 @@ class PruneTest(BitcoinTestFramework): self.stop_node(3) self.stop_node(4) - print("Check that we haven't started pruning yet because we're below PruneAfterHeight") + self.log.info("Check that we haven't started pruning yet because we're below PruneAfterHeight") self.test_height_min() # Extend this chain past the PruneAfterHeight # N0=N1=N2 **...*(1020) - print("Check that we'll exceed disk space target if we have a very high stale block rate") + self.log.info("Check that we'll exceed disk space target if we have a very high stale block rate") self.create_chain_with_staleblocks() # Disconnect N0 # And mine a 24 block chain on N1 and a separate 25 block chain on N0 @@ -384,7 +382,7 @@ class PruneTest(BitcoinTestFramework): self.mainchainheight = self.nodes[2].getblockcount() #1320 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) - print("Check that we can survive a 288 block reorg still") + self.log.info("Check that we can survive a 288 block reorg still") (self.forkheight,self.forkhash) = self.reorg_test() #(1033, ) # Now create a 288 block reorg by mining a longer chain on N1 # First disconnect N1 @@ -417,7 +415,7 @@ class PruneTest(BitcoinTestFramework): # \ # *...**(1320) - print("Test that we can rerequest a block we previously pruned if needed for a reorg") + self.log.info("Test that we can rerequest a block we previously pruned if needed for a reorg") self.reorg_back() # Verify that N2 still has block 1033 on current chain (@), but not on main chain (*) # Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to @@ -437,16 +435,16 @@ class PruneTest(BitcoinTestFramework): # # N1 doesn't change because 1033 on main chain (*) is invalid - print("Test manual pruning with block indices") + self.log.info("Test manual pruning with block indices") self.manual_test(3, use_timestamp=False) - print("Test manual pruning with timestamps") + self.log.info("Test manual pruning with timestamps") self.manual_test(4, use_timestamp=True) - print("Test wallet re-scan") + self.log.info("Test wallet re-scan") self.wallet_test() - print("Done") + self.log.info("Done") if __name__ == '__main__': PruneTest().main() diff --git a/qa/rpc-tests/rawtransactions.py b/test/functional/rawtransactions.py index 5adef31207..35debf9cab 100755 --- a/qa/rpc-tests/rawtransactions.py +++ b/test/functional/rawtransactions.py @@ -24,21 +24,9 @@ class RawTransactionsTest(BitcoinTestFramework): self.num_nodes = 3 def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - - #connect to a local machine for debugging - #url = "http://bitcoinrpc:DP6DvqZtqXarpeNWyN3LZTFchCCyCUuHwNF7E8pX99x1@%s:%d" % ('127.0.0.1', 18332) - #proxy = AuthServiceProxy(url) - #proxy.url = url # store URL on proxy for info - #self.nodes.append(proxy) - - connect_nodes_bi(self.nodes,0,1) - connect_nodes_bi(self.nodes,1,2) + super().setup_network() connect_nodes_bi(self.nodes,0,2) - self.is_network_split=False - self.sync_all() - def run_test(self): #prepare some coins for multiple *rawtransaction commands @@ -61,13 +49,8 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[2].createrawtransaction(inputs, outputs) rawtx = self.nodes[2].signrawtransaction(rawtx) - try: - rawtx = self.nodes[2].sendrawtransaction(rawtx['hex']) - except JSONRPCException as e: - assert("Missing inputs" in e.error['message']) - else: - assert(False) - + # This will raise an exception since there are missing inputs + assert_raises_jsonrpc(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) ######################### # RAW TX MULTISIG TESTS # @@ -109,7 +92,6 @@ class RawTransactionsTest(BitcoinTestFramework): txId = self.nodes[0].sendtoaddress(mSigObj, 2.2) decTx = self.nodes[0].gettransaction(txId) rawTx = self.nodes[0].decoderawtransaction(decTx['hex']) - sPK = rawTx['vout'][0]['scriptPubKey']['hex'] self.sync_all() self.nodes[0].generate(1) self.sync_all() @@ -161,13 +143,13 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(self.nodes[0].getrawtransaction(txHash, True)["hex"], rawTxSigned['hex']) # 6. invalid parameters - supply txid and string "Flase" - assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, txHash, "Flase") + assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, "Flase") # 7. invalid parameters - supply txid and empty array - assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, txHash, []) + assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, []) # 8. invalid parameters - supply txid and empty dict - assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, txHash, {}) + assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, {}) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] outputs = { self.nodes[0].getnewaddress() : 1 } @@ -175,13 +157,15 @@ class RawTransactionsTest(BitcoinTestFramework): decrawtx= self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['vin'][0]['sequence'], 1000) + # 9. invalid parameters - sequence number out of range inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : -1}] outputs = { self.nodes[0].getnewaddress() : 1 } - assert_raises(JSONRPCException, self.nodes[0].createrawtransaction, inputs, outputs) + assert_raises_jsonrpc(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) + # 10. invalid parameters - sequence number out of range inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967296}] outputs = { self.nodes[0].getnewaddress() : 1 } - assert_raises(JSONRPCException, self.nodes[0].createrawtransaction, inputs, outputs) + assert_raises_jsonrpc(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967294}] outputs = { self.nodes[0].getnewaddress() : 1 } diff --git a/qa/rpc-tests/receivedby.py b/test/functional/receivedby.py index 248bcdbd68..2cad6269ac 100755 --- a/qa/rpc-tests/receivedby.py +++ b/test/functional/receivedby.py @@ -32,7 +32,7 @@ class ReceivedByTest(BitcoinTestFramework): def setup_nodes(self): #This test requires mocktime enable_mocktime() - return start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): ''' diff --git a/qa/rpc-tests/reindex.py b/test/functional/reindex.py index 1b547a920f..b446baa04d 100755 --- a/qa/rpc-tests/reindex.py +++ b/test/functional/reindex.py @@ -10,11 +10,7 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - start_nodes, - stop_nodes, - assert_equal, -) +from test_framework.util import assert_equal import time class ReindexTest(BitcoinTestFramework): @@ -24,19 +20,16 @@ class ReindexTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - def reindex(self, justchainstate=False): self.nodes[0].generate(3) blockcount = self.nodes[0].getblockcount() - stop_nodes(self.nodes) - extra_args = [["-debug", "-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"]] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.stop_nodes() + extra_args = [["-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"]] + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) while self.nodes[0].getblockcount() < blockcount: time.sleep(0.1) assert_equal(self.nodes[0].getblockcount(), blockcount) - print("Success") + self.log.info("Success") def run_test(self): self.reindex(False) diff --git a/qa/rpc-tests/replace-by-fee.py b/test/functional/replace-by-fee.py index a3c1deddf6..077348e3b2 100755 --- a/qa/rpc-tests/replace-by-fee.py +++ b/test/functional/replace-by-fee.py @@ -25,19 +25,15 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): fee = 1*COIN while node.getbalance() < satoshi_round((amount + fee)/COIN): node.generate(100) - #print (node.getbalance(), amount, fee) new_addr = node.getnewaddress() - #print new_addr txid = node.sendtoaddress(new_addr, satoshi_round((amount+fee)/COIN)) tx1 = node.getrawtransaction(txid, 1) txid = int(txid, 16) i = None for i, txout in enumerate(tx1['vout']): - #print i, txout['scriptPubKey']['addresses'] if txout['scriptPubKey']['addresses'] == [new_addr]: - #print i break assert i is not None @@ -69,49 +65,44 @@ class ReplaceByFeeTest(BitcoinTestFramework): super().__init__() self.num_nodes = 1 self.setup_clean_chain = False - - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-debug", - "-whitelist=127.0.0.1", - "-limitancestorcount=50", - "-limitancestorsize=101", - "-limitdescendantcount=200", - "-limitdescendantsize=101" - ])) - self.is_network_split = False + self.extra_args= [["-maxorphantx=1000", + "-whitelist=127.0.0.1", + "-limitancestorcount=50", + "-limitancestorsize=101", + "-limitdescendantcount=200", + "-limitdescendantsize=101"]] def run_test(self): make_utxo(self.nodes[0], 1*COIN) - print("Running test simple doublespend...") + self.log.info("Running test simple doublespend...") self.test_simple_doublespend() - print("Running test doublespend chain...") + self.log.info("Running test doublespend chain...") self.test_doublespend_chain() - print("Running test doublespend tree...") + self.log.info("Running test doublespend tree...") self.test_doublespend_tree() - print("Running test replacement feeperkb...") + self.log.info("Running test replacement feeperkb...") self.test_replacement_feeperkb() - print("Running test spends of conflicting outputs...") + self.log.info("Running test spends of conflicting outputs...") self.test_spends_of_conflicting_outputs() - print("Running test new unconfirmed inputs...") + self.log.info("Running test new unconfirmed inputs...") self.test_new_unconfirmed_inputs() - print("Running test too many replacements...") + self.log.info("Running test too many replacements...") self.test_too_many_replacements() - print("Running test opt-in...") + self.log.info("Running test opt-in...") self.test_opt_in() - print("Running test prioritised transactions...") + self.log.info("Running test prioritised transactions...") self.test_prioritised_transactions() - print("Passed\n") + self.log.info("Passed") def test_simple_doublespend(self): """Simple doublespend""" @@ -129,12 +120,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b.vout = [CTxOut(1*COIN, CScript([b'b']))] tx1b_hex = txToHex(tx1b) - try: - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) # insufficient fee - else: - assert(False) + # This will raise an exception due to insufficient fee + assert_raises_jsonrpc(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) # Extra 0.1 BTC fee tx1b = CTransaction() @@ -176,12 +163,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vout = [CTxOut(initial_nValue - 30*COIN, CScript([1]))] dbl_tx_hex = txToHex(dbl_tx) - try: - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) # insufficient fee - else: - assert(False) # transaction mistakenly accepted! + # This will raise an exception due to insufficient fee + assert_raises_jsonrpc(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) # Accepted with sufficient fee dbl_tx = CTransaction() @@ -241,12 +224,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] dbl_tx.vout = [CTxOut(initial_nValue - fee*n, CScript([1]))] dbl_tx_hex = txToHex(dbl_tx) - try: - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) # insufficient fee - else: - assert(False) + # This will raise an exception due to insufficient fee + assert_raises_jsonrpc(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) # 1 BTC fee is enough dbl_tx = CTransaction() @@ -273,13 +252,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] dbl_tx.vout = [CTxOut(initial_nValue - 2*fee*n, CScript([1]))] dbl_tx_hex = txToHex(dbl_tx) - try: - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - assert_equal("too many potential replacements" in exp.error['message'], True) - else: - assert(False) + # This will raise an exception + assert_raises_jsonrpc(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) for tx in tree_txs: tx.rehash() @@ -302,12 +276,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b.vout = [CTxOut(int(0.001*COIN), CScript([b'a'*999000]))] tx1b_hex = txToHex(tx1b) - try: - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) # insufficient fee - else: - assert(False) + # This will raise an exception due to insufficient fee + assert_raises_jsonrpc(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) def test_spends_of_conflicting_outputs(self): """Replacements that spend conflicting tx outputs are rejected""" @@ -329,12 +299,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2.vout = tx1a.vout tx2_hex = txToHex(tx2) - try: - tx2_txid = self.nodes[0].sendrawtransaction(tx2_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - else: - assert(False) + # This will raise an exception + assert_raises_jsonrpc(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, True) # Spend tx1a's output to test the indirect case. tx1b = CTransaction() @@ -350,12 +316,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2.vout = tx1a.vout tx2_hex = txToHex(tx2) - try: - tx2_txid = self.nodes[0].sendrawtransaction(tx2_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - else: - assert(False) + # This will raise an exception + assert_raises_jsonrpc(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, True) def test_new_unconfirmed_inputs(self): """Replacements that add new unconfirmed inputs are rejected""" @@ -373,12 +335,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2.vout = tx1.vout tx2_hex = txToHex(tx2) - try: - tx2_txid = self.nodes[0].sendrawtransaction(tx2_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - else: - assert(False) + # This will raise an exception + assert_raises_jsonrpc(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, True) def test_too_many_replacements(self): """Replacements that evict too many transactions are rejected""" @@ -423,13 +381,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] double_tx_hex = txToHex(double_tx) - try: - self.nodes[0].sendrawtransaction(double_tx_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - assert_equal("too many potential replacements" in exp.error['message'], True) - else: - assert(False) + # This will raise an exception + assert_raises_jsonrpc(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, True) # If we remove an input, it should pass double_tx = CTransaction() @@ -455,13 +408,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b.vout = [CTxOut(int(0.9*COIN), CScript([b'b']))] tx1b_hex = txToHex(tx1b) - try: - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - else: - print(tx1b_txid) - assert(False) + # This will raise an exception + assert_raises_jsonrpc(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, True) tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) @@ -478,12 +426,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2b.vout = [CTxOut(int(0.9*COIN), CScript([b'b']))] tx2b_hex = txToHex(tx2b) - try: - tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - else: - assert(False) + # This will raise an exception + assert_raises_jsonrpc(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, True) # Now create a new transaction that spends from tx1a and tx2a # opt-in on one of the inputs @@ -535,15 +479,10 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # Verify tx1b cannot replace tx1a. - try: - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - else: - assert(False) + assert_raises_jsonrpc(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) # Use prioritisetransaction to set tx1a's fee to 0. - self.nodes[0].prioritisetransaction(tx1a_txid, 0, int(-0.1*COIN)) + self.nodes[0].prioritisetransaction(txid=tx1a_txid, fee_delta=int(-0.1*COIN)) # Now tx1b should be able to replace tx1a tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) @@ -567,15 +506,10 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2b_hex = txToHex(tx2b) # Verify tx2b cannot replace tx2a. - try: - tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, True) - except JSONRPCException as exp: - assert_equal(exp.error['code'], -26) - else: - assert(False) + assert_raises_jsonrpc(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, True) # Now prioritise tx2b to have a higher modified fee - self.nodes[0].prioritisetransaction(tx2b.hash, 0, int(0.1*COIN)) + self.nodes[0].prioritisetransaction(txid=tx2b.hash, fee_delta=int(0.1*COIN)) # tx2b should now be accepted tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, True) diff --git a/qa/rpc-tests/rest.py b/test/functional/rest.py index 9ca1cc187a..fbcceba0fa 100755 --- a/qa/rpc-tests/rest.py +++ b/test/functional/rest.py @@ -49,16 +49,12 @@ class RESTTest (BitcoinTestFramework): self.num_nodes = 3 def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes,0,1) - connect_nodes_bi(self.nodes,1,2) - connect_nodes_bi(self.nodes,0,2) - self.is_network_split=False - self.sync_all() + super().setup_network() + connect_nodes_bi(self.nodes, 0, 2) def run_test(self): url = urllib.parse.urlparse(self.nodes[0].url) - print("Mining blocks...") + self.log.info("Mining blocks...") self.nodes[0].generate(1) self.sync_all() diff --git a/qa/rpc-tests/rpcbind_test.py b/test/functional/rpcbind_test.py index 499fe33679..5336cf2ec8 100755 --- a/qa/rpc-tests/rpcbind_test.py +++ b/test/functional/rpcbind_test.py @@ -4,7 +4,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with the -rpcbind and -rpcallowip options.""" -from test_framework.test_framework import BitcoinTestFramework +import socket +import sys + +from test_framework.test_framework import BitcoinTestFramework, SkipTest from test_framework.util import * from test_framework.netutil import * @@ -33,10 +36,10 @@ class RPCBindTest(BitcoinTestFramework): if allow_ips: base_args += ['-rpcallowip=' + x for x in allow_ips] binds = ['-rpcbind='+addr for addr in addresses] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to) pid = bitcoind_processes[0].pid assert_equal(set(get_bind_addrs(pid)), set(expected)) - stop_nodes(self.nodes) + self.stop_nodes() def run_allowip_test(self, allow_ips, rpchost, rpcport): ''' @@ -44,15 +47,16 @@ class RPCBindTest(BitcoinTestFramework): at a non-localhost IP. ''' base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [base_args]) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args]) # connect to node through non-loopback interface node = get_rpc_proxy(rpc_url(0, "%s:%d" % (rpchost, rpcport)), 0) node.getnetworkinfo() - stop_nodes(self.nodes) + self.stop_nodes() def run_test(self): # due to OS-specific network stats queries, this test works only on Linux - assert(sys.platform.startswith('linux')) + if not sys.platform.startswith('linux'): + raise SkipTest("This test can only be run on linux.") # find the first non-loopback interface for testing non_loopback_ip = None for name,ip in all_interfaces(): @@ -60,8 +64,15 @@ class RPCBindTest(BitcoinTestFramework): non_loopback_ip = ip break if non_loopback_ip is None: - assert(not 'This test requires at least one non-loopback IPv4 interface') - print("Using interface %s for testing" % non_loopback_ip) + raise SkipTest("This test requires at least one non-loopback IPv4 interface.") + try: + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s.connect(("::1",1)) + s.close + except OSError: + raise SkipTest("This test requires IPv6 support.") + + self.log.info("Using interface %s for testing" % non_loopback_ip) defaultport = rpc_port(0) @@ -92,11 +103,7 @@ class RPCBindTest(BitcoinTestFramework): # Check that with invalid rpcallowip, we are denied self.run_allowip_test([non_loopback_ip], non_loopback_ip, defaultport) - try: - self.run_allowip_test(['1.1.1.1'], non_loopback_ip, defaultport) - assert(not 'Connection not denied by rpcallowip as expected') - except JSONRPCException: - pass + assert_raises_jsonrpc(-342, "non-JSON HTTP response with '403 Forbidden' from server", self.run_allowip_test, ['1.1.1.1'], non_loopback_ip, defaultport) if __name__ == '__main__': RPCBindTest().main() diff --git a/qa/rpc-tests/rpcnamedargs.py b/test/functional/rpcnamedargs.py index f6175c8ca7..3b286000a1 100755 --- a/qa/rpc-tests/rpcnamedargs.py +++ b/test/functional/rpcnamedargs.py @@ -8,7 +8,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_jsonrpc, - start_nodes, ) @@ -22,11 +21,6 @@ class NamedArgumentTest(BitcoinTestFramework): self.setup_clean_chain = False self.num_nodes = 1 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - self.is_network_split = False - self.sync_all() - def run_test(self): node = self.nodes[0] h = node.help(command='getinfo') diff --git a/qa/rpc-tests/segwit.py b/test/functional/segwit.py index f475427842..ac95d66466 100755 --- a/qa/rpc-tests/segwit.py +++ b/test/functional/segwit.py @@ -6,11 +6,10 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -from test_framework.mininode import sha256, ripemd160, CTransaction, CTxIn, COutPoint, CTxOut +from test_framework.mininode import sha256, CTransaction, CTxIn, COutPoint, CTxOut, COIN, ToHex, FromHex from test_framework.address import script_to_p2sh, key_to_p2pkh -from test_framework.script import CScript, OP_HASH160, OP_CHECKSIG, OP_0, hash160, OP_EQUAL, OP_DUP, OP_EQUALVERIFY, OP_1, OP_2, OP_CHECKMULTISIG +from test_framework.script import CScript, OP_HASH160, OP_CHECKSIG, OP_0, hash160, OP_EQUAL, OP_DUP, OP_EQUALVERIFY, OP_1, OP_2, OP_CHECKMULTISIG, OP_TRUE from io import BytesIO -from test_framework.mininode import FromHex NODE_0 = 0 NODE_1 = 1 @@ -18,47 +17,49 @@ NODE_2 = 2 WIT_V0 = 0 WIT_V1 = 1 -def witness_script(version, pubkey): - if (version == 0): - pubkeyhash = bytes_to_hex_str(ripemd160(sha256(hex_str_to_bytes(pubkey)))) - pkscript = "0014" + pubkeyhash - elif (version == 1): - # 1-of-1 multisig - scripthash = bytes_to_hex_str(sha256(hex_str_to_bytes("5121" + pubkey + "51ae"))) - pkscript = "0020" + scripthash +# Create a scriptPubKey corresponding to either a P2WPKH output for the +# given pubkey, or a P2WSH output of a 1-of-1 multisig for the given +# pubkey. Returns the hex encoding of the scriptPubKey. +def witness_script(use_p2wsh, pubkey): + if (use_p2wsh == False): + # P2WPKH instead + pubkeyhash = hash160(hex_str_to_bytes(pubkey)) + pkscript = CScript([OP_0, pubkeyhash]) else: - assert("Wrong version" == "0 or 1") - return pkscript - -def addlength(script): - scriptlen = format(len(script)//2, 'x') - assert(len(scriptlen) == 2) - return scriptlen + script - -def create_witnessprogram(version, node, utxo, pubkey, encode_p2sh, amount): - pkscript = witness_script(version, pubkey) + # 1-of-1 multisig + witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) + scripthash = sha256(witness_program) + pkscript = CScript([OP_0, scripthash]) + return bytes_to_hex_str(pkscript) + +# Return a transaction (in hex) that spends the given utxo to a segwit output, +# optionally wrapping the segwit output using P2SH. +def create_witnessprogram(use_p2wsh, utxo, pubkey, encode_p2sh, amount): + pkscript = hex_str_to_bytes(witness_script(use_p2wsh, pubkey)) if (encode_p2sh): - p2sh_hash = bytes_to_hex_str(ripemd160(sha256(hex_str_to_bytes(pkscript)))) - pkscript = "a914"+p2sh_hash+"87" - inputs = [] - outputs = {} - inputs.append({ "txid" : utxo["txid"], "vout" : utxo["vout"]} ) - DUMMY_P2SH = "2MySexEGVzZpRgNQ1JdjdP5bRETznm3roQ2" # P2SH of "OP_1 OP_DROP" - outputs[DUMMY_P2SH] = amount - tx_to_witness = node.createrawtransaction(inputs,outputs) - #replace dummy output with our own - tx_to_witness = tx_to_witness[0:110] + addlength(pkscript) + tx_to_witness[-8:] - return tx_to_witness - -def send_to_witness(version, node, utxo, pubkey, encode_p2sh, amount, sign=True, insert_redeem_script=""): - tx_to_witness = create_witnessprogram(version, node, utxo, pubkey, encode_p2sh, amount) + p2sh_hash = hash160(pkscript) + pkscript = CScript([OP_HASH160, p2sh_hash, OP_EQUAL]) + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), b"")) + tx.vout.append(CTxOut(int(amount*COIN), pkscript)) + return ToHex(tx) + +# Create a transaction spending a given utxo to a segwit output corresponding +# to the given pubkey: use_p2wsh determines whether to use P2WPKH or P2WSH; +# encode_p2sh determines whether to wrap in P2SH. +# sign=True will have the given node sign the transaction. +# insert_redeem_script will be added to the scriptSig, if given. +def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=True, insert_redeem_script=""): + tx_to_witness = create_witnessprogram(use_p2wsh, utxo, pubkey, encode_p2sh, amount) if (sign): signed = node.signrawtransaction(tx_to_witness) assert("errors" not in signed or len(["errors"]) == 0) return node.sendrawtransaction(signed["hex"]) else: if (insert_redeem_script): - tx_to_witness = tx_to_witness[0:82] + addlength(insert_redeem_script) + tx_to_witness[84:] + tx = FromHex(CTransaction(), tx_to_witness) + tx.vin[0].scriptSig += CScript([hex_str_to_bytes(insert_redeem_script)]) + tx_to_witness = ToHex(tx) return node.sendrawtransaction(tx_to_witness) @@ -79,16 +80,13 @@ class SegWitTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [["-walletprematurewitness", "-rpcserialversion=0"], + ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1"], + ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness"]] def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-logtimemicros", "-debug", "-walletprematurewitness", "-rpcserialversion=0"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-logtimemicros", "-debug", "-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1"])) - self.nodes.append(start_node(2, self.options.tmpdir, ["-logtimemicros", "-debug", "-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness"])) - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 1) + super().setup_network() connect_nodes(self.nodes[0], 2) - self.is_network_split = False self.sync_all() def success_mine(self, node, txid, sign, redeem_script=""): @@ -103,28 +101,18 @@ class SegWitTest(BitcoinTestFramework): assert_equal(len(node.getblock(block[0])["tx"]), 1) sync_blocks(self.nodes) - def fail_accept(self, node, txid, sign, redeem_script=""): - try: - send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) - except JSONRPCException as exp: - assert(exp.error["code"] == -26) - else: - raise AssertionError("Tx should not have been accepted") + def fail_accept(self, node, error_msg, txid, sign, redeem_script=""): + assert_raises_jsonrpc(-26, error_msg, send_to_witness, 1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) def fail_mine(self, node, txid, sign, redeem_script=""): send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) - try: - node.generate(1) - except JSONRPCException as exp: - assert(exp.error["code"] == -1) - else: - raise AssertionError("Created valid block when TestBlockValidity should have failed") + assert_raises_jsonrpc(-1, "CreateNewBlock: TestBlockValidity failed", node.generate, 1) sync_blocks(self.nodes) def run_test(self): self.nodes[0].generate(161) #block 161 - print("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") + self.log.info("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({}) assert(tmpl['sizelimit'] == 1000000) @@ -173,22 +161,22 @@ class SegWitTest(BitcoinTestFramework): self.nodes[0].generate(260) #block 423 sync_blocks(self.nodes) - print("Verify default node can't accept any witness format txs before fork") + self.log.info("Verify default node can't accept any witness format txs before fork") # unsigned, no scriptsig - self.fail_accept(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], False) - self.fail_accept(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], False) - self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], False) - self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], False) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", wit_ids[NODE_0][WIT_V0][0], False) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", wit_ids[NODE_0][WIT_V1][0], False) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V0][0], False) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False) # unsigned with redeem script - self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], False, addlength(witness_script(0, self.pubkey[0]))) - self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], False, addlength(witness_script(1, self.pubkey[0]))) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V0][0], False, witness_script(False, self.pubkey[0])) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) # signed - self.fail_accept(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) - self.fail_accept(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], True) - self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True) - self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True) + self.fail_accept(self.nodes[0], "no-witness-yet", wit_ids[NODE_0][WIT_V0][0], True) + self.fail_accept(self.nodes[0], "no-witness-yet", wit_ids[NODE_0][WIT_V1][0], True) + self.fail_accept(self.nodes[0], "no-witness-yet", p2sh_ids[NODE_0][WIT_V0][0], True) + self.fail_accept(self.nodes[0], "no-witness-yet", p2sh_ids[NODE_0][WIT_V1][0], True) - print("Verify witness txs are skipped for mining before the fork") + self.log.info("Verify witness txs are skipped for mining before the fork") self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][0], True) #block 424 self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][0], True) #block 425 self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][0], True) #block 426 @@ -196,19 +184,19 @@ class SegWitTest(BitcoinTestFramework): # TODO: An old node would see these txs without witnesses and be able to mine them - print("Verify unsigned bare witness txs in versionbits-setting blocks are valid before the fork") + self.log.info("Verify unsigned bare witness txs in versionbits-setting blocks are valid before the fork") self.success_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][1], False) #block 428 self.success_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][1], False) #block 429 - print("Verify unsigned p2sh witness txs without a redeem script are invalid") - self.fail_accept(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][1], False) - self.fail_accept(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][1], False) + self.log.info("Verify unsigned p2sh witness txs without a redeem script are invalid") + self.fail_accept(self.nodes[2], "mandatory-script-verify-flag", p2sh_ids[NODE_2][WIT_V0][1], False) + self.fail_accept(self.nodes[2], "mandatory-script-verify-flag", p2sh_ids[NODE_2][WIT_V1][1], False) - print("Verify unsigned p2sh witness txs with a redeem script in versionbits-settings blocks are valid before the fork") - self.success_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][1], False, addlength(witness_script(0, self.pubkey[2]))) #block 430 - self.success_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][1], False, addlength(witness_script(1, self.pubkey[2]))) #block 431 + self.log.info("Verify unsigned p2sh witness txs with a redeem script in versionbits-settings blocks are valid before the fork") + self.success_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][1], False, witness_script(False, self.pubkey[2])) #block 430 + self.success_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][1], False, witness_script(True, self.pubkey[2])) #block 431 - print("Verify previous witness txs skipped for mining can now be mined") + self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) block = self.nodes[2].generate(1) #block 432 (first block with new rules; 432 = 144 * 3) sync_blocks(self.nodes) @@ -216,7 +204,7 @@ class SegWitTest(BitcoinTestFramework): segwit_tx_list = self.nodes[2].getblock(block[0])["tx"] assert_equal(len(segwit_tx_list), 5) - print("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") + self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") assert(self.nodes[2].getblock(block[0], False) != self.nodes[0].getblock(block[0], False)) assert(self.nodes[1].getblock(block[0], False) == self.nodes[2].getblock(block[0], False)) for i in range(len(segwit_tx_list)): @@ -227,19 +215,19 @@ class SegWitTest(BitcoinTestFramework): assert(self.nodes[1].getrawtransaction(segwit_tx_list[i]) == self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) == bytes_to_hex_str(tx.serialize_without_witness())) - print("Verify witness txs without witness data are invalid after the fork") + self.log.info("Verify witness txs without witness data are invalid after the fork") self.fail_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][2], False) self.fail_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][2], False) - self.fail_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][2], False, addlength(witness_script(0, self.pubkey[2]))) - self.fail_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][2], False, addlength(witness_script(1, self.pubkey[2]))) + self.fail_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][2], False, witness_script(False, self.pubkey[2])) + self.fail_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][2], False, witness_script(True, self.pubkey[2])) - print("Verify default node can now use witness txs") + self.log.info("Verify default node can now use witness txs") self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) #block 432 self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], True) #block 433 self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True) #block 434 self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True) #block 435 - print("Verify sigops are counted in GBT with BIP141 rules after the fork") + self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules':['segwit']}) assert(tmpl['sizelimit'] >= 3999577) # actual maximum size is lower due to minimum mandatory non-witness data @@ -248,22 +236,56 @@ class SegWitTest(BitcoinTestFramework): assert(tmpl['transactions'][0]['txid'] == txid) assert(tmpl['transactions'][0]['sigops'] == 8) - print("Verify non-segwit miners get a valid GBT response after the fork") - send_to_witness(1, self.nodes[0], find_unspent(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.998")) - try: - tmpl = self.nodes[0].getblocktemplate({}) - assert(len(tmpl['transactions']) == 1) # Doesn't include witness tx - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) - assert(('!segwit' in tmpl['rules']) or ('segwit' not in tmpl['rules'])) - except JSONRPCException: - # This is an acceptable outcome - pass - - print("Verify behaviour of importaddress, addwitnessaddress and listunspent") + self.nodes[0].generate(1) # Mine a block to clear the gbt cache + + self.log.info("Non-segwit miners are able to use GBT response after activation.") + # Create a 3-tx chain: tx1 (non-segwit input, paying to a segwit output) -> + # tx2 (segwit input, paying to a non-segwit output) -> + # tx3 (non-segwit input, paying to a non-segwit output). + # tx1 is allowed to appear in the block, but no others. + txid1 = send_to_witness(1, self.nodes[0], find_unspent(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.996")) + hex_tx = self.nodes[0].gettransaction(txid)['hex'] + tx = FromHex(CTransaction(), hex_tx) + assert(tx.wit.is_null()) # This should not be a segwit input + assert(txid1 in self.nodes[0].getrawmempool()) + + # Now create tx2, which will spend from txid1. + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(int(txid1, 16), 0), b'')) + tx.vout.append(CTxOut(int(49.99*COIN), CScript([OP_TRUE]))) + tx2_hex = self.nodes[0].signrawtransaction(ToHex(tx))['hex'] + txid2 = self.nodes[0].sendrawtransaction(tx2_hex) + tx = FromHex(CTransaction(), tx2_hex) + assert(not tx.wit.is_null()) + + # Now create tx3, which will spend from txid2 + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(int(txid2, 16), 0), b"")) + tx.vout.append(CTxOut(int(49.95*COIN), CScript([OP_TRUE]))) # Huge fee + tx.calc_sha256() + txid3 = self.nodes[0].sendrawtransaction(ToHex(tx)) + assert(tx.wit.is_null()) + assert(txid3 in self.nodes[0].getrawmempool()) + + # Now try calling getblocktemplate() without segwit support. + template = self.nodes[0].getblocktemplate() + + # Check that tx1 is the only transaction of the 3 in the template. + template_txids = [ t['txid'] for t in template['transactions'] ] + assert(txid2 not in template_txids and txid3 not in template_txids) + assert(txid1 in template_txids) + + # Check that running with segwit support results in all 3 being included. + template = self.nodes[0].getblocktemplate({"rules": ["segwit"]}) + template_txids = [ t['txid'] for t in template['transactions'] ] + assert(txid1 in template_txids) + assert(txid2 in template_txids) + assert(txid3 in template_txids) + + # Mine a block to clear the gbt cache again. + self.nodes[0].generate(1) + + self.log.info("Verify behaviour of importaddress, addwitnessaddress and listunspent") # Some public keys to be used later pubkeys = [ @@ -421,10 +443,13 @@ class SegWitTest(BitcoinTestFramework): importlist.append(bytes_to_hex_str(p2wshop1)) for i in importlist: + # import all generated addresses. The wallet already has the private keys for some of these, so catch JSON RPC + # exceptions and continue. try: self.nodes[0].importaddress(i,"",False,True) except JSONRPCException as exp: assert_equal(exp.error["message"], "The wallet already contains the private key for this address or script") + assert_equal(exp.error["code"], -4) self.nodes[0].importaddress(script_to_p2sh(op0)) # import OP_0 as address only self.nodes[0].importaddress(multisig_without_privkey_address) # Test multisig_without_privkey @@ -439,12 +464,7 @@ class SegWitTest(BitcoinTestFramework): # note that no witness address should be returned by unsolvable addresses # the multisig_without_privkey_address will fail because its keys were not added with importpubkey for i in uncompressed_spendable_address + uncompressed_solvable_address + unknown_address + unsolvable_address + [multisig_without_privkey_address]: - try: - self.nodes[0].addwitnessaddress(i) - except JSONRPCException as exp: - assert_equal(exp.error["message"], "Public key or redeemscript not known to wallet, or the key is uncompressed") - else: - assert(False) + assert_raises_jsonrpc(-4, "Public key or redeemscript not known to wallet, or the key is uncompressed", self.nodes[0].addwitnessaddress, i) for i in compressed_spendable_address + compressed_solvable_address: witaddress = self.nodes[0].addwitnessaddress(i) @@ -523,12 +543,8 @@ class SegWitTest(BitcoinTestFramework): # note that a multisig address returned by addmultisigaddress is not solvable until it is added with importaddress # premature_witaddress are not accepted until the script is added with addwitnessaddress first for i in uncompressed_spendable_address + uncompressed_solvable_address + premature_witaddress + [compressed_solvable_address[1]]: - try: - self.nodes[0].addwitnessaddress(i) - except JSONRPCException as exp: - assert_equal(exp.error["message"], "Public key or redeemscript not known to wallet, or the key is uncompressed") - else: - assert(False) + # This will raise an exception + assert_raises_jsonrpc(-4, "Public key or redeemscript not known to wallet, or the key is uncompressed", self.nodes[0].addwitnessaddress, i) # after importaddress it should pass addwitnessaddress v = self.nodes[0].validateaddress(compressed_solvable_address[1]) diff --git a/qa/rpc-tests/sendheaders.py b/test/functional/sendheaders.py index 16ca4a4913..44c357c6db 100755 --- a/qa/rpc-tests/sendheaders.py +++ b/test/functional/sendheaders.py @@ -81,23 +81,17 @@ from test_framework.blocktools import create_block, create_coinbase direct_fetch_response_time = 0.05 -class BaseNode(SingleNodeConnCB): +class TestNode(NodeConnCB): def __init__(self): - SingleNodeConnCB.__init__(self) - self.last_inv = None - self.last_headers = None - self.last_block = None - self.last_getdata = None + super().__init__() self.block_announced = False - self.last_getheaders = None - self.disconnected = False self.last_blockhash_announced = None def clear_last_announcement(self): with mininode_lock: self.block_announced = False - self.last_inv = None - self.last_headers = None + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) # Request data for a list of block hashes def get_data(self, block_hashes): @@ -118,29 +112,17 @@ class BaseNode(SingleNodeConnCB): self.connection.send_message(msg) def on_inv(self, conn, message): - self.last_inv = message self.block_announced = True self.last_blockhash_announced = message.inv[-1].hash def on_headers(self, conn, message): - self.last_headers = message if len(message.headers): self.block_announced = True message.headers[-1].calc_sha256() self.last_blockhash_announced = message.headers[-1].sha256 def on_block(self, conn, message): - self.last_block = message.block - self.last_block.calc_sha256() - - def on_getdata(self, conn, message): - self.last_getdata = message - - def on_getheaders(self, conn, message): - self.last_getheaders = message - - def on_close(self, conn): - self.disconnected = True + self.last_message["block"].calc_sha256() # Test whether the last announcement we received had the # right header or the right inv @@ -155,43 +137,27 @@ class BaseNode(SingleNodeConnCB): success = True compare_inv = [] - if self.last_inv != None: - compare_inv = [x.hash for x in self.last_inv.inv] + if "inv" in self.last_message: + compare_inv = [x.hash for x in self.last_message["inv"].inv] if compare_inv != expect_inv: success = False hash_headers = [] - if self.last_headers != None: + if "headers" in self.last_message: # treat headers as a list of block hashes - hash_headers = [ x.sha256 for x in self.last_headers.headers ] + hash_headers = [ x.sha256 for x in self.last_message["headers"].headers ] if hash_headers != expect_headers: success = False - self.last_inv = None - self.last_headers = None + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) return success - # Syncing helpers - def wait_for_block(self, blockhash, timeout=60): - test_function = lambda: self.last_block != None and self.last_block.sha256 == blockhash - assert(wait_until(test_function, timeout=timeout)) - return - - def wait_for_getheaders(self, timeout=60): - test_function = lambda: self.last_getheaders != None - assert(wait_until(test_function, timeout=timeout)) - return - def wait_for_getdata(self, hash_list, timeout=60): if hash_list == []: return - test_function = lambda: self.last_getdata != None and [x.hash for x in self.last_getdata.inv] == hash_list - assert(wait_until(test_function, timeout=timeout)) - return - - def wait_for_disconnect(self, timeout=60): - test_function = lambda: self.disconnected + test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list assert(wait_until(test_function, timeout=timeout)) return @@ -210,28 +176,12 @@ class BaseNode(SingleNodeConnCB): getblocks_message.locator.vHave = locator self.send_message(getblocks_message) -# InvNode: This peer should only ever receive inv's, because it doesn't ever send a -# "sendheaders" message. -class InvNode(BaseNode): - def __init__(self): - BaseNode.__init__(self) - -# TestNode: This peer is the one we use for most of the testing. -class TestNode(BaseNode): - def __init__(self): - BaseNode.__init__(self) - class SendHeadersTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 2 - def setup_network(self): - self.nodes = [] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1"]]*2) - connect_nodes(self.nodes[0], 1) - # mine count blocks and return the new tip def mine_blocks(self, count): # Clear out last block announcement from each p2p listener @@ -260,7 +210,7 @@ class SendHeadersTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. - inv_node = InvNode() + inv_node = TestNode() test_node = TestNode() self.p2p_connections = [inv_node, test_node] @@ -283,7 +233,7 @@ class SendHeadersTest(BitcoinTestFramework): # PART 1 # 1. Mine a block; expect inv announcements each time - print("Part 1: headers don't start before sendheaders message...") + self.log.info("Part 1: headers don't start before sendheaders message...") for i in range(4): old_tip = tip tip = self.mine_blocks(1) @@ -293,7 +243,7 @@ class SendHeadersTest(BitcoinTestFramework): if i == 0: # first request the block test_node.get_data([tip]) - test_node.wait_for_block(tip, timeout=5) + test_node.wait_for_block(tip) elif i == 1: # next try requesting header and block test_node.get_headers(locator=[old_tip], hashstop=tip) @@ -308,14 +258,14 @@ class SendHeadersTest(BitcoinTestFramework): new_block = create_block(tip, create_coinbase(height+1), block_time) new_block.solve() test_node.send_header_for_blocks([new_block]) - test_node.wait_for_getdata([new_block.sha256], timeout=5) + test_node.wait_for_getdata([new_block.sha256]) test_node.send_message(msg_block(new_block)) test_node.sync_with_ping() # make sure this block is processed inv_node.clear_last_announcement() test_node.clear_last_announcement() - print("Part 1: success!") - print("Part 2: announce blocks with headers after sendheaders message...") + self.log.info("Part 1: success!") + self.log.info("Part 2: announce blocks with headers after sendheaders message...") # PART 2 # 2. Send a sendheaders message and test that headers announcements # commence and keep working. @@ -347,18 +297,18 @@ class SendHeadersTest(BitcoinTestFramework): if j == 0: # Announce via inv test_node.send_block_inv(tip) - test_node.wait_for_getheaders(timeout=5) + test_node.wait_for_getheaders() # Should have received a getheaders now test_node.send_header_for_blocks(blocks) # Test that duplicate inv's won't result in duplicate # getdata requests, or duplicate headers announcements [ inv_node.send_block_inv(x.sha256) for x in blocks ] - test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=5) + test_node.wait_for_getdata([x.sha256 for x in blocks]) inv_node.sync_with_ping() else: # Announce via headers test_node.send_header_for_blocks(blocks) - test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=5) + test_node.wait_for_getdata([x.sha256 for x in blocks]) # Test that duplicate headers won't result in duplicate # getdata requests (the check is further down) inv_node.send_header_for_blocks(blocks) @@ -368,17 +318,17 @@ class SendHeadersTest(BitcoinTestFramework): inv_node.sync_with_ping() # This block should not be announced to the inv node (since it also # broadcast it) - assert_equal(inv_node.last_inv, None) - assert_equal(inv_node.last_headers, None) + assert "inv" not in inv_node.last_message + assert "headers" not in inv_node.last_message tip = self.mine_blocks(1) assert_equal(inv_node.check_last_announcement(inv=[tip]), True) assert_equal(test_node.check_last_announcement(headers=[tip]), True) height += 1 block_time += 1 - print("Part 2: success!") + self.log.info("Part 2: success!") - print("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...") + self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...") # PART 3. Headers announcements can stop after large reorg, and resume after # getheaders or inv from peer. @@ -440,9 +390,9 @@ class SendHeadersTest(BitcoinTestFramework): assert_equal(inv_node.check_last_announcement(inv=[tip]), True) assert_equal(test_node.check_last_announcement(headers=[tip]), True) - print("Part 3: success!") + self.log.info("Part 3: success!") - print("Part 4: Testing direct fetch behavior...") + self.log.info("Part 4: Testing direct fetch behavior...") tip = self.mine_blocks(1) height = self.nodes[0].getblockcount() + 1 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] @@ -459,12 +409,12 @@ class SendHeadersTest(BitcoinTestFramework): inv_node.send_message(msg_block(blocks[-1])) inv_node.sync_with_ping() # Make sure blocks are processed - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() # should not have received any getdata messages with mininode_lock: - assert_equal(test_node.last_getdata, None) + assert "getdata" not in test_node.last_message # This time, direct fetch should work blocks = [] @@ -498,11 +448,11 @@ class SendHeadersTest(BitcoinTestFramework): # Announcing one block on fork should not trigger direct fetch # (less work than tip) - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[0:1]) test_node.sync_with_ping() with mininode_lock: - assert_equal(test_node.last_getdata, None) + assert "getdata" not in test_node.last_message # Announcing one more block on fork should trigger direct fetch for # both blocks (same work as tip) @@ -517,22 +467,22 @@ class SendHeadersTest(BitcoinTestFramework): test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=direct_fetch_response_time) # Announcing 1 more header should not trigger any response - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[18:19]) test_node.sync_with_ping() with mininode_lock: - assert_equal(test_node.last_getdata, None) + assert "getdata" not in test_node.last_message - print("Part 4: success!") + self.log.info("Part 4: success!") # Now deliver all those blocks we announced. [ test_node.send_message(msg_block(x)) for x in blocks ] - print("Part 5: Testing handling of unconnecting headers") + self.log.info("Part 5: Testing handling of unconnecting headers") # First we test that receipt of an unconnecting header doesn't prevent # chain sync. for i in range(10): - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) blocks = [] # Create two more blocks. for j in range(2): @@ -543,9 +493,9 @@ class SendHeadersTest(BitcoinTestFramework): height += 1 # Send the header of the second block -> this won't connect. with mininode_lock: - test_node.last_getheaders = None + test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[1]]) - test_node.wait_for_getheaders(timeout=1) + test_node.wait_for_getheaders() test_node.send_header_for_blocks(blocks) test_node.wait_for_getdata([x.sha256 for x in blocks]) [ test_node.send_message(msg_block(x)) for x in blocks ] @@ -566,9 +516,9 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(1, MAX_UNCONNECTING_HEADERS): # Send a header that doesn't connect, check that we get a getheaders. with mininode_lock: - test_node.last_getheaders = None + test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i]]) - test_node.wait_for_getheaders(timeout=1) + test_node.wait_for_getheaders() # Next header will connect, should re-set our count: test_node.send_header_for_blocks([blocks[0]]) @@ -581,25 +531,21 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(5*MAX_UNCONNECTING_HEADERS - 1): # Send a header that doesn't connect, check that we get a getheaders. with mininode_lock: - test_node.last_getheaders = None + test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i%len(blocks)]]) - test_node.wait_for_getheaders(timeout=1) + test_node.wait_for_getheaders() # Eventually this stops working. - with mininode_lock: - self.last_getheaders = None test_node.send_header_for_blocks([blocks[-1]]) # Should get disconnected test_node.wait_for_disconnect() - with mininode_lock: - self.last_getheaders = True - print("Part 5: success!") + self.log.info("Part 5: success!") # Finally, check that the inv node never received a getdata request, # throughout the test - assert_equal(inv_node.last_getdata, None) + assert "getdata" not in inv_node.last_message if __name__ == '__main__': SendHeadersTest().main() diff --git a/qa/rpc-tests/signmessages.py b/test/functional/signmessages.py index 91f5abef5d..42f6a9daaf 100755 --- a/qa/rpc-tests/signmessages.py +++ b/test/functional/signmessages.py @@ -5,7 +5,6 @@ """Test RPC commands for signing and verifying messages.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * class SignMessagesTest(BitcoinTestFramework): @@ -14,10 +13,6 @@ class SignMessagesTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - self.is_network_split = False - def run_test(self): message = 'This is just a test message' diff --git a/qa/rpc-tests/signrawtransactions.py b/test/functional/signrawtransactions.py index b24162ab97..437905e764 100755 --- a/qa/rpc-tests/signrawtransactions.py +++ b/test/functional/signrawtransactions.py @@ -14,10 +14,6 @@ class SignRawTransactionsTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - self.is_network_split = False - def successful_signing_test(self): """Create and sign a valid raw transaction with one input. @@ -118,6 +114,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): # 5) Script verification errors have certain properties assert 'txid' in rawTxSigned['errors'][0] assert 'vout' in rawTxSigned['errors'][0] + assert 'witness' in rawTxSigned['errors'][0] assert 'scriptSig' in rawTxSigned['errors'][0] assert 'sequence' in rawTxSigned['errors'][0] assert 'error' in rawTxSigned['errors'][0] @@ -127,6 +124,32 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(rawTxSigned['errors'][0]['vout'], inputs[1]['vout']) assert_equal(rawTxSigned['errors'][1]['txid'], inputs[2]['txid']) assert_equal(rawTxSigned['errors'][1]['vout'], inputs[2]['vout']) + assert not rawTxSigned['errors'][0]['witness'] + + # Now test signing failure for transaction with input witnesses + p2wpkh_raw_tx = "01000000000102fff7f7881a8099afa6940d42d1e7f6362bec38171ea3edf433541db4e4ad969f00000000494830450221008b9d1dc26ba6a9cb62127b02742fa9d754cd3bebf337f7a55d114c8e5cdd30be022040529b194ba3f9281a99f2b1c0a19c0489bc22ede944ccf4ecbab4cc618ef3ed01eeffffffef51e1b804cc89d182d279655c3aa89e815b1b309fe287d9b2b55d57b90ec68a0100000000ffffffff02202cb206000000001976a9148280b37df378db99f66f85c95a783a76ac7a6d5988ac9093510d000000001976a9143bde42dbee7e4dbe6a21b2d50ce2f0167faa815988ac000247304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee0121025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee635711000000" + + rawTxSigned = self.nodes[0].signrawtransaction(p2wpkh_raw_tx) + + # 7) The transaction has no complete set of signatures + assert 'complete' in rawTxSigned + assert_equal(rawTxSigned['complete'], False) + + # 8) Two script verification errors occurred + assert 'errors' in rawTxSigned + assert_equal(len(rawTxSigned['errors']), 2) + + # 9) Script verification errors have certain properties + assert 'txid' in rawTxSigned['errors'][0] + assert 'vout' in rawTxSigned['errors'][0] + assert 'witness' in rawTxSigned['errors'][0] + assert 'scriptSig' in rawTxSigned['errors'][0] + assert 'sequence' in rawTxSigned['errors'][0] + assert 'error' in rawTxSigned['errors'][0] + + # Non-empty witness checked here + assert_equal(rawTxSigned['errors'][1]['witness'], ["304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee01", "025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee6357"]) + assert not rawTxSigned['errors'][0]['witness'] def run_test(self): self.successful_signing_test() diff --git a/qa/rpc-tests/smartfees.py b/test/functional/smartfees.py index bde454968f..482c77863f 100755 --- a/qa/rpc-tests/smartfees.py +++ b/test/functional/smartfees.py @@ -4,18 +4,23 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" -from collections import OrderedDict from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * +from test_framework.script import CScript, OP_1, OP_DROP, OP_2, OP_HASH160, OP_EQUAL, hash160, OP_TRUE +from test_framework.mininode import CTransaction, CTxIn, CTxOut, COutPoint, ToHex, COIN # Construct 2 trivial P2SH's and the ScriptSigs that spend them # So we can create many many transactions without needing to spend # time signing. -P2SH_1 = "2MySexEGVzZpRgNQ1JdjdP5bRETznm3roQ2" # P2SH of "OP_1 OP_DROP" -P2SH_2 = "2NBdpwq8Aoo1EEKEXPNrKvr5xQr3M9UfcZA" # P2SH of "OP_2 OP_DROP" +redeem_script_1 = CScript([OP_1, OP_DROP]) +redeem_script_2 = CScript([OP_2, OP_DROP]) +P2SH_1 = CScript([OP_HASH160, hash160(redeem_script_1), OP_EQUAL]) +P2SH_2 = CScript([OP_HASH160, hash160(redeem_script_2), OP_EQUAL]) + # Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2 -# 4 bytes of OP_TRUE and push 2-byte redeem script of "OP_1 OP_DROP" or "OP_2 OP_DROP" -SCRIPT_SIG = ["0451025175", "0451025275"] +SCRIPT_SIG = [CScript([OP_TRUE, redeem_script_1]), CScript([OP_TRUE, redeem_script_2])] + +global log def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment): """ @@ -33,60 +38,55 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee rand_fee = float(fee_increment)*(1.1892**random.randint(0,28)) # Total fee ranges from min_fee to min_fee + 127*fee_increment fee = min_fee - fee_increment + satoshi_round(rand_fee) - inputs = [] + tx = CTransaction() total_in = Decimal("0.00000000") while total_in <= (amount + fee) and len(conflist) > 0: t = conflist.pop(0) total_in += t["amount"] - inputs.append({ "txid" : t["txid"], "vout" : t["vout"]} ) + tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b"")) if total_in <= amount + fee: while total_in <= (amount + fee) and len(unconflist) > 0: t = unconflist.pop(0) total_in += t["amount"] - inputs.append({ "txid" : t["txid"], "vout" : t["vout"]} ) + tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b"")) if total_in <= amount + fee: raise RuntimeError("Insufficient funds: need %d, have %d"%(amount+fee, total_in)) - outputs = {} - outputs = OrderedDict([(P2SH_1, total_in - amount - fee), - (P2SH_2, amount)]) - rawtx = from_node.createrawtransaction(inputs, outputs) - # createrawtransaction constructs a transaction that is ready to be signed. - # These transactions don't need to be signed, but we still have to insert the ScriptSig - # that will satisfy the ScriptPubKey. - completetx = rawtx[0:10] - inputnum = 0 - for inp in inputs: - completetx += rawtx[10+82*inputnum:82+82*inputnum] - completetx += SCRIPT_SIG[inp["vout"]] - completetx += rawtx[84+82*inputnum:92+82*inputnum] - inputnum += 1 - completetx += rawtx[10+82*inputnum:] - txid = from_node.sendrawtransaction(completetx, True) + tx.vout.append(CTxOut(int((total_in - amount - fee)*COIN), P2SH_1)) + tx.vout.append(CTxOut(int(amount*COIN), P2SH_2)) + # These transactions don't need to be signed, but we still have to insert + # the ScriptSig that will satisfy the ScriptPubKey. + for inp in tx.vin: + inp.scriptSig = SCRIPT_SIG[inp.prevout.n] + txid = from_node.sendrawtransaction(ToHex(tx), True) unconflist.append({ "txid" : txid, "vout" : 0 , "amount" : total_in - amount - fee}) unconflist.append({ "txid" : txid, "vout" : 1 , "amount" : amount}) - return (completetx, fee) + return (ToHex(tx), fee) def split_inputs(from_node, txins, txouts, initial_split = False): """ - We need to generate a lot of very small inputs so we can generate a ton of transactions - and they will have low priority. + We need to generate a lot of inputs so we can generate a ton of transactions. This function takes an input from txins, and creates and sends a transaction which splits the value into 2 outputs which are appended to txouts. + Previously this was designed to be small inputs so they wouldn't have + a high coin age when the notion of priority still existed. """ prevtxout = txins.pop() - inputs = [] - inputs.append({ "txid" : prevtxout["txid"], "vout" : prevtxout["vout"] }) + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(int(prevtxout["txid"], 16), prevtxout["vout"]), b"")) + half_change = satoshi_round(prevtxout["amount"]/2) rem_change = prevtxout["amount"] - half_change - Decimal("0.00001000") - outputs = OrderedDict([(P2SH_1, half_change), (P2SH_2, rem_change)]) - rawtx = from_node.createrawtransaction(inputs, outputs) + tx.vout.append(CTxOut(int(half_change*COIN), P2SH_1)) + tx.vout.append(CTxOut(int(rem_change*COIN), P2SH_2)) + # If this is the initial split we actually need to sign the transaction - # Otherwise we just need to insert the property ScriptSig + # Otherwise we just need to insert the proper ScriptSig if (initial_split) : - completetx = from_node.signrawtransaction(rawtx)["hex"] + completetx = from_node.signrawtransaction(ToHex(tx))["hex"] else : - completetx = rawtx[0:82] + SCRIPT_SIG[prevtxout["vout"]] + rawtx[84:] + tx.vin[0].scriptSig = SCRIPT_SIG[prevtxout["vout"]] + completetx = ToHex(tx) txid = from_node.sendrawtransaction(completetx, True) txouts.append({ "txid" : txid, "vout" : 0 , "amount" : half_change}) txouts.append({ "txid" : txid, "vout" : 1 , "amount" : rem_change}) @@ -98,7 +98,7 @@ def check_estimates(node, fees_seen, max_invalid, print_estimates = True): """ all_estimates = [ node.estimatefee(i) for i in range(1,26) ] if print_estimates: - print([str(all_estimates[e-1]) for e in [1,2,3,6,15,25]]) + log.info([str(all_estimates[e-1]) for e in [1,2,3,6,15,25]]) delta = 1.0e-6 # account for rounding error last_e = max(fees_seen) for e in [x for x in all_estimates if x >= 0]: @@ -116,8 +116,8 @@ def check_estimates(node, fees_seen, max_invalid, print_estimates = True): for i,e in enumerate(all_estimates): # estimate is for i+1 if e >= 0: valid_estimate = True - # estimatesmartfee should return the same result - assert_equal(node.estimatesmartfee(i+1)["feerate"], e) + if i >= 13: # for n>=14 estimatesmartfee(n/2) should be at least as high as estimatefee(n) + assert(node.estimatesmartfee((i+1)//2)["feerate"] > float(e) - delta) else: invalid_estimates += 1 @@ -150,16 +150,16 @@ class EstimateFeeTest(BitcoinTestFramework): def setup_network(self): """ We'll setup the network to have 3 nodes that all mine with different parameters. - But first we need to use one node to create a lot of small low priority outputs + But first we need to use one node to create a lot of outputs which we will use to generate our transactions. """ self.nodes = [] # Use node0 to mine blocks for input splitting - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-whitelist=127.0.0.1"])) - print("This test is time consuming, please be patient") - print("Splitting inputs to small size so we can generate low priority tx's") + self.log.info("This test is time consuming, please be patient") + self.log.info("Splitting inputs so we can generate tx's") self.txouts = [] self.txouts2 = [] # Split a coinbase into two transaction puzzle outputs @@ -184,28 +184,25 @@ class EstimateFeeTest(BitcoinTestFramework): while (len(self.nodes[0].getrawmempool()) > 0): self.nodes[0].generate(1) reps += 1 - print("Finished splitting") + self.log.info("Finished splitting") # Now we can connect the other nodes, didn't want to connect them earlier # so the estimates would not be affected by the splitting transactions - # Node1 mines small blocks but that are bigger than the expected transaction rate, - # and allows free transactions. + # Node1 mines small blocks but that are bigger than the expected transaction rate. # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes, # (17k is room enough for 110 or so transactions) - self.nodes.append(start_node(1, self.options.tmpdir, - ["-blockprioritysize=1500", "-blockmaxsize=17000", - "-maxorphantx=1000", "-debug=estimatefee"])) + self.nodes.append(self.start_node(1, self.options.tmpdir, + ["-blockmaxsize=17000", "-maxorphantx=1000"])) connect_nodes(self.nodes[1], 0) # Node2 is a stingy miner, that # produces too small blocks (room for only 55 or so transactions) - node2args = ["-blockprioritysize=0", "-blockmaxsize=8000", "-maxorphantx=1000"] + node2args = ["-blockmaxsize=8000", "-maxorphantx=1000"] - self.nodes.append(start_node(2, self.options.tmpdir, node2args)) + self.nodes.append(self.start_node(2, self.options.tmpdir, node2args)) connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[2], 1) - self.is_network_split = False self.sync_all() def transact_and_mine(self, numblocks, mining_node): @@ -235,18 +232,21 @@ class EstimateFeeTest(BitcoinTestFramework): self.memutxo = newmem def run_test(self): + # Make log handler available to helper functions + global log + log = self.log self.fees_per_kb = [] self.memutxo = [] self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting - print("Will output estimates for 1/2/3/6/15/25 blocks") + self.log.info("Will output estimates for 1/2/3/6/15/25 blocks") for i in range(2): - print("Creating transactions and mining them with a block size that can't keep up") + self.log.info("Creating transactions and mining them with a block size that can't keep up") # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine self.transact_and_mine(10, self.nodes[2]) check_estimates(self.nodes[1], self.fees_per_kb, 14) - print("Creating transactions and mining them at a block size that is just big enough") + self.log.info("Creating transactions and mining them at a block size that is just big enough") # Generate transactions while mining 10 more blocks, this time with node1 # which mines blocks with capacity just above the rate that transactions are being created self.transact_and_mine(10, self.nodes[1]) @@ -257,7 +257,7 @@ class EstimateFeeTest(BitcoinTestFramework): self.nodes[1].generate(1) sync_blocks(self.nodes[0:3], wait=.1) - print("Final estimates after emptying mempools") + self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb, 2) if __name__ == '__main__': diff --git a/qa/rpc-tests/test_framework/__init__.py b/test/functional/test_framework/__init__.py index e69de29bb2..e69de29bb2 100644 --- a/qa/rpc-tests/test_framework/__init__.py +++ b/test/functional/test_framework/__init__.py diff --git a/qa/rpc-tests/test_framework/address.py b/test/functional/test_framework/address.py index 96bebe1ea1..96bebe1ea1 100644 --- a/qa/rpc-tests/test_framework/address.py +++ b/test/functional/test_framework/address.py diff --git a/qa/rpc-tests/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 9ab3094b06..dfcc524313 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -42,6 +42,7 @@ import decimal import json import logging import socket +import time try: import urllib.parse as urlparse except ImportError: @@ -163,6 +164,7 @@ class AuthServiceProxy(object): return self._request('POST', self.__url.path, postdata.encode('utf-8')) def _get_response(self): + req_start_time = time.time() try: http_response = self.__conn.getresponse() except socket.timeout as e: @@ -183,8 +185,9 @@ class AuthServiceProxy(object): responsedata = http_response.read().decode('utf8') response = json.loads(responsedata, parse_float=decimal.Decimal) + elapsed = time.time() - req_start_time if "error" in response and response["error"] is None: - log.debug("<-%s- %s"%(response["id"], json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("<-%s- [%.6f] %s"%(response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: - log.debug("<-- "+responsedata) + log.debug("<-- [%.6f] %s"%(elapsed,responsedata)) return response diff --git a/qa/rpc-tests/test_framework/bignum.py b/test/functional/test_framework/bignum.py index 024611da6e..024611da6e 100644 --- a/qa/rpc-tests/test_framework/bignum.py +++ b/test/functional/test_framework/bignum.py diff --git a/qa/rpc-tests/test_framework/blockstore.py b/test/functional/test_framework/blockstore.py index 5280d18cdc..4cfd682bb5 100644 --- a/qa/rpc-tests/test_framework/blockstore.py +++ b/test/functional/test_framework/blockstore.py @@ -8,6 +8,8 @@ from .mininode import * from io import BytesIO import dbm.dumb as dbmd +logger = logging.getLogger("TestFramework.blockstore") + class BlockStore(object): """BlockStore helper class. @@ -86,7 +88,7 @@ class BlockStore(object): try: self.blockDB[repr(block.sha256)] = bytes(block.serialize()) except TypeError as e: - print("Unexpected error: ", sys.exc_info()[0], e.args) + logger.exception("Unexpected error") self.currentBlock = block.sha256 self.headers_map[block.sha256] = CBlockHeader(block) @@ -156,7 +158,7 @@ class TxStore(object): try: self.txDB[repr(tx.sha256)] = bytes(tx.serialize()) except TypeError as e: - print("Unexpected error: ", sys.exc_info()[0], e.args) + logger.exception("Unexpected error") def get_transactions(self, inv): responses = [] diff --git a/qa/rpc-tests/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 2c9a0857df..5dcf516dc6 100644 --- a/qa/rpc-tests/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -25,6 +25,13 @@ def create_block(hashprev, coinbase, nTime=None): # From BIP141 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" + +def get_witness_script(witness_root, witness_nonce): + witness_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(witness_nonce))) + output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(witness_commitment) + return CScript([OP_RETURN, output_data]) + + # According to BIP141, blocks with witness rules active must commit to the # hash of all in-block transactions including witness. def add_witness_commitment(block, nonce=0): @@ -32,14 +39,12 @@ def add_witness_commitment(block, nonce=0): # transactions, with witnesses. witness_nonce = nonce witness_root = block.calc_witness_merkle_root() - witness_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(witness_nonce))) # witness_nonce should go to coinbase witness. block.vtx[0].wit.vtxinwit = [CTxInWitness()] block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(witness_nonce)] # witness commitment is the last OP_RETURN output in coinbase - output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(witness_commitment) - block.vtx[0].vout.append(CTxOut(0, CScript([OP_RETURN, output_data]))) + block.vtx[0].vout.append(CTxOut(0, get_witness_script(witness_root, witness_nonce))) block.vtx[0].rehash() block.hashMerkleRoot = block.calc_merkle_root() block.rehash() diff --git a/qa/rpc-tests/test_framework/comptool.py b/test/functional/test_framework/comptool.py index b617895cac..9f062865a3 100755 --- a/qa/rpc-tests/test_framework/comptool.py +++ b/test/functional/test_framework/comptool.py @@ -21,6 +21,10 @@ from .mininode import * from .blockstore import BlockStore, TxStore from .util import p2p_port +import logging + +logger=logging.getLogger("TestFramework.comptool") + global mininode_lock class RejectResult(object): @@ -38,7 +42,7 @@ class RejectResult(object): class TestNode(NodeConnCB): def __init__(self, block_store, tx_store): - NodeConnCB.__init__(self) + super().__init__() self.conn = None self.bestblockhash = None self.block_store = block_store @@ -188,9 +192,7 @@ class TestManager(object): return wait_until(disconnected, timeout=10) def wait_for_verack(self): - def veracked(): - return all(node.verack_received for node in self.test_nodes) - return wait_until(veracked, timeout=10) + return all(node.wait_for_verack() for node in self.test_nodes) def wait_for_pings(self, counter): def received_pongs(): @@ -209,7 +211,6 @@ class TestManager(object): # --> error if not requested if not wait_until(blocks_requested, attempts=20*num_blocks): - # print [ c.cb.block_request_map for c in self.connections ] raise AssertionError("Not all nodes requested block") # Send getheaders message @@ -231,7 +232,6 @@ class TestManager(object): # --> error if not requested if not wait_until(transaction_requested, attempts=20*num_events): - # print [ c.cb.tx_request_map for c in self.connections ] raise AssertionError("Not all nodes requested transaction") # Get the mempool @@ -258,13 +258,12 @@ class TestManager(object): if c.cb.bestblockhash == blockhash: return False if blockhash not in c.cb.block_reject_map: - print('Block not in reject map: %064x' % (blockhash)) + logger.error('Block not in reject map: %064x' % (blockhash)) return False if not outcome.match(c.cb.block_reject_map[blockhash]): - print('Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash)) + logger.error('Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash)) return False elif ((c.cb.bestblockhash == blockhash) != outcome): - # print c.cb.bestblockhash, blockhash, outcome return False return True @@ -280,19 +279,17 @@ class TestManager(object): if outcome is None: # Make sure the mempools agree with each other if c.cb.lastInv != self.connections[0].cb.lastInv: - # print c.rpc.getrawmempool() return False elif isinstance(outcome, RejectResult): # Check that tx was rejected w/ code if txhash in c.cb.lastInv: return False if txhash not in c.cb.tx_reject_map: - print('Tx not in reject map: %064x' % (txhash)) + logger.error('Tx not in reject map: %064x' % (txhash)) return False if not outcome.match(c.cb.tx_reject_map[txhash]): - print('Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash)) + logger.error('Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash)) return False elif ((txhash in c.cb.lastInv) != outcome): - # print c.rpc.getrawmempool(), c.cb.lastInv return False return True @@ -402,7 +399,7 @@ class TestManager(object): if (not self.check_mempool(tx.sha256, tx_outcome)): raise AssertionError("Mempool test failed at test %d" % test_number) - print("Test %d: PASS" % test_number, [ c.rpc.getblockcount() for c in self.connections ]) + logger.info("Test %d: PASS" % test_number) test_number += 1 [ c.disconnect_node() for c in self.connections ] diff --git a/qa/rpc-tests/test_framework/coverage.py b/test/functional/test_framework/coverage.py index 3f87ef91f6..3f87ef91f6 100644 --- a/qa/rpc-tests/test_framework/coverage.py +++ b/test/functional/test_framework/coverage.py diff --git a/qa/rpc-tests/test_framework/key.py b/test/functional/test_framework/key.py index 85a6158a2f..85a6158a2f 100644 --- a/qa/rpc-tests/test_framework/key.py +++ b/test/functional/test_framework/key.py diff --git a/qa/rpc-tests/test_framework/mininode.py b/test/functional/test_framework/mininode.py index b2292f6477..fb3ed1473a 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -20,21 +20,22 @@ msg_block, msg_tx, msg_headers, etc.: ser_*, deser_*: functions that handle serialization/deserialization """ -import struct -import socket import asyncore -import time -import sys -import random -from .util import hex_str_to_bytes, bytes_to_hex_str -from io import BytesIO from codecs import encode +from collections import defaultdict +import copy import hashlib -from threading import RLock -from threading import Thread +from io import BytesIO import logging -import copy +import random +import socket +import struct +import sys +import time +from threading import RLock, Thread + from test_framework.siphash import siphash256 +from test_framework.util import hex_str_to_bytes, bytes_to_hex_str BIP0031_VERSION = 60000 MY_VERSION = 70014 # past bip-31 for ping/pong @@ -51,6 +52,8 @@ NODE_GETUTXO = (1 << 1) NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) +logger = logging.getLogger("TestFramework.mininode") + # Keep our own socket map for asyncore, so that we can track disconnects # ourselves (to workaround an issue with closing an asyncore socket when # using select) @@ -608,7 +611,8 @@ class CBlock(CBlockHeader): return r # Calculate the merkle root given a vector of transaction hashes - def get_merkle_root(self, hashes): + @classmethod + def get_merkle_root(cls, hashes): while len(hashes) > 1: newhashes = [] for i in range(0, len(hashes), 2): @@ -1354,6 +1358,8 @@ class msg_reject(object): # Helper function def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf')): + if attempts == float('inf') and timeout == float('inf'): + timeout = 60 attempt = 0 elapsed = 0 @@ -1463,59 +1469,94 @@ class msg_witness_blocktxn(msg_blocktxn): r += self.block_transactions.serialize(with_witness=True) return r -# This is what a callback should look like for NodeConn -# Reimplement the on_* functions to provide handling for events class NodeConnCB(object): + """Callback and helper functions for P2P connection to a bitcoind node. + + Individual testcases should subclass this and override the on_* methods + if they want to alter message handling behaviour. + """ + def __init__(self): - self.verack_received = False + # Track whether we have a P2P connection open to the node + self.connected = False + self.connection = None + + # Track number of messages of each type received and the most recent + # message of each type + self.message_count = defaultdict(int) + self.last_message = {} + + # A count of the number of ping messages we've sent to the node + self.ping_counter = 1 + # deliver_sleep_time is helpful for debugging race conditions in p2p # tests; it causes message delivery to sleep for the specified time # before acquiring the global lock and delivering the next message. self.deliver_sleep_time = None + # Remember the services our peer has advertised self.peer_services = None - def set_deliver_sleep_time(self, value): - with mininode_lock: - self.deliver_sleep_time = value + # Message receiving methods - def get_deliver_sleep_time(self): - with mininode_lock: - return self.deliver_sleep_time + def deliver(self, conn, message): + """Receive message and dispatch message to appropriate callback. - # Spin until verack message is received from the node. - # Tests may want to use this as a signal that the test can begin. - # This can be called from the testing thread, so it needs to acquire the - # global lock. - def wait_for_verack(self): - while True: - with mininode_lock: - if self.verack_received: - return - time.sleep(0.05) + We keep a count of how many of each message type has been received + and the most recent message of each type. + + Optionally waits for deliver_sleep_time before dispatching message. + """ - def deliver(self, conn, message): deliver_sleep = self.get_deliver_sleep_time() if deliver_sleep is not None: time.sleep(deliver_sleep) with mininode_lock: try: - getattr(self, 'on_' + message.command.decode('ascii'))(conn, message) + command = message.command.decode('ascii') + self.message_count[command] += 1 + self.last_message[command] = message + getattr(self, 'on_' + command)(conn, message) except: print("ERROR delivering %s (%s)" % (repr(message), sys.exc_info()[0])) - def on_version(self, conn, message): - if message.nVersion >= 209: - conn.send_message(msg_verack()) - conn.ver_send = min(MY_VERSION, message.nVersion) - if message.nVersion < 209: - conn.ver_recv = conn.ver_send - conn.nServices = message.nServices + def set_deliver_sleep_time(self, value): + with mininode_lock: + self.deliver_sleep_time = value - def on_verack(self, conn, message): - conn.ver_recv = conn.ver_send - self.verack_received = True + def get_deliver_sleep_time(self): + with mininode_lock: + return self.deliver_sleep_time + + # Callback methods. Can be overridden by subclasses in individual test + # cases to provide custom message handling behaviour. + + def on_open(self, conn): + self.connected = True + + def on_close(self, conn): + self.connected = False + self.connection = None + + def on_addr(self, conn, message): pass + def on_alert(self, conn, message): pass + def on_block(self, conn, message): pass + def on_blocktxn(self, conn, message): pass + def on_cmpctblock(self, conn, message): pass + def on_feefilter(self, conn, message): pass + def on_getaddr(self, conn, message): pass + def on_getblocks(self, conn, message): pass + def on_getblocktxn(self, conn, message): pass + def on_getdata(self, conn, message): pass + def on_getheaders(self, conn, message): pass + def on_headers(self, conn, message): pass + def on_mempool(self, conn): pass + def on_pong(self, conn, message): pass + def on_reject(self, conn, message): pass + def on_sendcmpct(self, conn, message): pass + def on_sendheaders(self, conn, message): pass + def on_tx(self, conn, message): pass def on_inv(self, conn, message): want = msg_getdata() @@ -1525,60 +1566,77 @@ class NodeConnCB(object): if len(want.inv): conn.send_message(want) - def on_addr(self, conn, message): pass - def on_alert(self, conn, message): pass - def on_getdata(self, conn, message): pass - def on_getblocks(self, conn, message): pass - def on_tx(self, conn, message): pass - def on_block(self, conn, message): pass - def on_getaddr(self, conn, message): pass - def on_headers(self, conn, message): pass - def on_getheaders(self, conn, message): pass def on_ping(self, conn, message): if conn.ver_send > BIP0031_VERSION: conn.send_message(msg_pong(message.nonce)) - def on_reject(self, conn, message): pass - def on_open(self, conn): pass - def on_close(self, conn): pass - def on_mempool(self, conn): pass - def on_pong(self, conn, message): pass - def on_feefilter(self, conn, message): pass - def on_sendheaders(self, conn, message): pass - def on_sendcmpct(self, conn, message): pass - def on_cmpctblock(self, conn, message): pass - def on_getblocktxn(self, conn, message): pass - def on_blocktxn(self, conn, message): pass -# More useful callbacks and functions for NodeConnCB's which have a single NodeConn -class SingleNodeConnCB(NodeConnCB): - def __init__(self): - NodeConnCB.__init__(self) - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() + def on_verack(self, conn, message): + conn.ver_recv = conn.ver_send + self.verack_received = True + + def on_version(self, conn, message): + if message.nVersion >= 209: + conn.send_message(msg_verack()) + conn.ver_send = min(MY_VERSION, message.nVersion) + if message.nVersion < 209: + conn.ver_recv = conn.ver_send + conn.nServices = message.nServices + + # Connection helper methods def add_connection(self, conn): self.connection = conn - # Wrapper for the NodeConn's send_message function + def wait_for_disconnect(self, timeout=60): + test_function = lambda: not self.connected + assert wait_until(test_function, timeout=timeout) + + # Message receiving helper methods + + def wait_for_block(self, blockhash, timeout=60): + test_function = lambda: self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash + assert wait_until(test_function, timeout=timeout) + + def wait_for_getdata(self, timeout=60): + test_function = lambda: self.last_message.get("getdata") + assert wait_until(test_function, timeout=timeout) + + def wait_for_getheaders(self, timeout=60): + test_function = lambda: self.last_message.get("getheaders") + assert wait_until(test_function, timeout=timeout) + + def wait_for_inv(self, expected_inv, timeout=60): + """Waits for an INV message and checks that the first inv object in the message was as expected.""" + if len(expected_inv) > 1: + raise NotImplementedError("wait_for_inv() will only verify the first inv object") + test_function = lambda: self.last_message.get("inv") and \ + self.last_message["inv"].inv[0].type == expected_inv[0].type and \ + self.last_message["inv"].inv[0].hash == expected_inv[0].hash + assert wait_until(test_function, timeout=timeout) + + def wait_for_verack(self, timeout=60): + test_function = lambda: self.message_count["verack"] + assert wait_until(test_function, timeout=timeout) + + # Message sending helper functions + def send_message(self, message): - self.connection.send_message(message) + if self.connection: + self.connection.send_message(message) + else: + logger.error("Cannot send message. No connection to node!") def send_and_ping(self, message): self.send_message(message) self.sync_with_ping() - def on_pong(self, conn, message): - self.last_pong = message - # Sync up with the node - def sync_with_ping(self, timeout=30): - def received_pong(): - return (self.last_pong.nonce == self.ping_counter) + def sync_with_ping(self, timeout=60): self.send_message(msg_ping(nonce=self.ping_counter)) - success = wait_until(received_pong, timeout=timeout) + test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter + assert wait_until(test_function, timeout=timeout) self.ping_counter += 1 - return success + return True # The actual NodeConn class # This class provides an interface for a p2p connection to a specified node @@ -1615,7 +1673,6 @@ class NodeConn(asyncore.dispatcher): def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK, send_version=True): asyncore.dispatcher.__init__(self, map=mininode_socket_map) - self.log = logging.getLogger("NodeConn(%s:%d)" % (dstaddr, dstport)) self.dstaddr = dstaddr self.dstport = dstport self.create_socket(socket.AF_INET, socket.SOCK_STREAM) @@ -1640,8 +1697,7 @@ class NodeConn(asyncore.dispatcher): vt.addrFrom.port = 0 self.send_message(vt, True) - print('MiniNode: Connecting to Bitcoin Node IP # ' + dstaddr + ':' \ - + str(dstport)) + logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) try: self.connect((dstaddr, dstport)) @@ -1649,18 +1705,14 @@ class NodeConn(asyncore.dispatcher): self.handle_close() self.rpc = rpc - def show_debug_msg(self, msg): - self.log.debug(msg) - def handle_connect(self): if self.state != "connected": - self.show_debug_msg("MiniNode: Connected & Listening: \n") + logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) self.state = "connected" self.cb.on_open(self) def handle_close(self): - self.show_debug_msg("MiniNode: Closing Connection to %s:%d... " - % (self.dstaddr, self.dstport)) + logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport)) self.state = "closed" self.recvbuf = b"" self.sendbuf = b"" @@ -1742,17 +1794,14 @@ class NodeConn(asyncore.dispatcher): t.deserialize(f) self.got_message(t) else: - self.show_debug_msg("Unknown command: '" + command + "' " + - repr(msg)) + logger.warning("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) except Exception as e: - print('got_data:', repr(e)) - # import traceback - # traceback.print_tb(sys.exc_info()[2]) + logger.exception('got_data:', repr(e)) def send_message(self, message, pushbuf=False): if self.state != "connected" and not pushbuf: raise IOError('Not connected, no pushbuf') - self.show_debug_msg("Send %s" % repr(message)) + self._log_message("send", message) command = message.command data = message.serialize() tmsg = self.MAGIC_BYTES[self.network] @@ -1774,9 +1823,19 @@ class NodeConn(asyncore.dispatcher): self.messagemap[b'ping'] = msg_ping_prebip31 if self.last_sent + 30 * 60 < time.time(): self.send_message(self.messagemap[b'ping']()) - self.show_debug_msg("Recv %s" % repr(message)) + self._log_message("receive", message) self.cb.deliver(self, message) + def _log_message(self, direction, msg): + if direction == "send": + log_message = "Send message to " + elif direction == "receive": + log_message = "Received message from " + log_message += "%s:%d: %s" % (self.dstaddr, self.dstport, repr(msg)[:500]) + if len(log_message) > 500: + log_message += "... (msg truncated)" + logger.debug(log_message) + def disconnect_node(self): self.disconnect = True diff --git a/qa/rpc-tests/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 45d8e22d22..45d8e22d22 100644 --- a/qa/rpc-tests/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py diff --git a/qa/rpc-tests/test_framework/script.py b/test/functional/test_framework/script.py index 3d9572788e..3d9572788e 100644 --- a/qa/rpc-tests/test_framework/script.py +++ b/test/functional/test_framework/script.py diff --git a/qa/rpc-tests/test_framework/siphash.py b/test/functional/test_framework/siphash.py index f68ecad36b..f68ecad36b 100644 --- a/qa/rpc-tests/test_framework/siphash.py +++ b/test/functional/test_framework/siphash.py diff --git a/qa/rpc-tests/test_framework/socks5.py b/test/functional/test_framework/socks5.py index 450bf3775e..a08b03ed24 100644 --- a/qa/rpc-tests/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -5,7 +5,9 @@ """Dummy Socks5 server for testing.""" import socket, threading, queue -import traceback, sys +import logging + +logger = logging.getLogger("TestFramework.socks5") ### Protocol constants class Command: @@ -112,10 +114,10 @@ class Socks5Connection(object): cmdin = Socks5Command(cmd, atyp, addr, port, username, password) self.serv.queue.put(cmdin) - print('Proxy: ', cmdin) + logger.info('Proxy: %s', cmdin) # Fall through to disconnect except Exception as e: - traceback.print_exc(file=sys.stderr) + logger.exception("socks5 request handling failed.") self.serv.queue.put(e) finally: self.conn.close() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py new file mode 100755 index 0000000000..67abf35687 --- /dev/null +++ b/test/functional/test_framework/test_framework.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Base class for RPC testing.""" + +from collections import deque +from enum import Enum +import logging +import optparse +import os +import shutil +import subprocess +import sys +import tempfile +import time + +from .util import ( + PortSeed, + MAX_NODES, + bitcoind_processes, + check_json_precision, + connect_nodes_bi, + disable_mocktime, + disconnect_nodes, + enable_coverage, + enable_mocktime, + get_mocktime, + get_rpc_proxy, + initialize_datadir, + log_filename, + p2p_port, + rpc_url, + set_node_times, + _start_node, + _start_nodes, + _stop_node, + _stop_nodes, + sync_blocks, + sync_mempools, + wait_for_bitcoind_start, +) +from .authproxy import JSONRPCException + +class TestStatus(Enum): + PASSED = 1 + FAILED = 2 + SKIPPED = 3 + +TEST_EXIT_PASSED = 0 +TEST_EXIT_FAILED = 1 +TEST_EXIT_SKIPPED = 77 + +class BitcoinTestFramework(object): + """Base class for a bitcoin test script. + + Individual bitcoin test scripts should subclass this class and override the following methods: + + - __init__() + - add_options() + - setup_chain() + - setup_network() + - run_test() + + The main() method should not be overridden. + + This class also contains various public and private helper methods.""" + + # Methods to override in subclass test scripts. + def __init__(self): + self.num_nodes = 4 + self.setup_clean_chain = False + self.nodes = None + + def add_options(self, parser): + pass + + def setup_chain(self): + self.log.info("Initializing test directory "+self.options.tmpdir) + if self.setup_clean_chain: + self._initialize_chain_clean(self.options.tmpdir, self.num_nodes) + else: + self._initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir) + + def setup_network(self): + self.setup_nodes() + + # Connect the nodes as a "chain". This allows us + # to split the network between nodes 1 and 2 to get + # two halves that can work on competing chains. + for i in range(self.num_nodes - 1): + connect_nodes_bi(self.nodes, i, i + 1) + self.sync_all() + + def setup_nodes(self): + extra_args = None + if hasattr(self, "extra_args"): + extra_args = self.extra_args + self.nodes = _start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + + def run_test(self): + raise NotImplementedError + + # Main function. This should not be overridden by the subclass test scripts. + + def main(self): + + parser = optparse.OptionParser(usage="%prog [options]") + parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true", + help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_option("--noshutdown", dest="noshutdown", default=False, action="store_true", + help="Don't stop bitcoinds after the test execution") + parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"), + help="Source directory containing bitcoind/bitcoin-cli (default: %default)") + parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"), + help="Directory for caching pregenerated datadirs") + parser.add_option("--tmpdir", dest="tmpdir", help="Root directory for datadirs") + parser.add_option("-l", "--loglevel", dest="loglevel", default="INFO", + help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") + parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true", + help="Print out all RPC calls as they are made") + parser.add_option("--portseed", dest="port_seed", default=os.getpid(), type='int', + help="The seed to use for assigning port numbers (default: current process id)") + parser.add_option("--coveragedir", dest="coveragedir", + help="Write tested RPC commands into this directory") + parser.add_option("--configfile", dest="configfile", + help="Location of the test framework config file") + self.add_options(parser) + (self.options, self.args) = parser.parse_args() + + if self.options.coveragedir: + enable_coverage(self.options.coveragedir) + + PortSeed.n = self.options.port_seed + + os.environ['PATH'] = self.options.srcdir+":"+self.options.srcdir+"/qt:"+os.environ['PATH'] + + check_json_precision() + + # Set up temp directory and start logging + if self.options.tmpdir: + os.makedirs(self.options.tmpdir, exist_ok=False) + else: + self.options.tmpdir = tempfile.mkdtemp(prefix="test") + self._start_logging() + + success = TestStatus.FAILED + + try: + self.setup_chain() + self.setup_network() + self.run_test() + success = TestStatus.PASSED + except JSONRPCException as e: + self.log.exception("JSONRPC error") + except SkipTest as e: + self.log.warning("Test Skipped: %s" % e.message) + success = TestStatus.SKIPPED + except AssertionError as e: + self.log.exception("Assertion failed") + except KeyError as e: + self.log.exception("Key error") + except Exception as e: + self.log.exception("Unexpected exception caught during testing") + except KeyboardInterrupt as e: + self.log.warning("Exiting after keyboard interrupt") + + if not self.options.noshutdown: + self.log.info("Stopping nodes") + if self.nodes: + self.stop_nodes() + else: + self.log.info("Note: bitcoinds were not stopped and may still be running") + + if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED: + self.log.info("Cleaning up") + shutil.rmtree(self.options.tmpdir) + else: + self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) + if os.getenv("PYTHON_DEBUG", ""): + # Dump the end of the debug logs, to aid in debugging rare + # travis failures. + import glob + filenames = [self.options.tmpdir + "/test_framework.log"] + filenames += glob.glob(self.options.tmpdir + "/node*/regtest/debug.log") + MAX_LINES_TO_PRINT = 1000 + for fn in filenames: + try: + with open(fn, 'r') as f: + print("From" , fn, ":") + print("".join(deque(f, MAX_LINES_TO_PRINT))) + except OSError: + print("Opening file %s failed." % fn) + traceback.print_exc() + + if success == TestStatus.PASSED: + self.log.info("Tests successful") + sys.exit(TEST_EXIT_PASSED) + elif success == TestStatus.SKIPPED: + self.log.info("Test skipped") + sys.exit(TEST_EXIT_SKIPPED) + else: + self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) + logging.shutdown() + sys.exit(TEST_EXIT_FAILED) + + # Public helper methods. These can be accessed by the subclass test scripts. + + def start_node(self, i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): + return _start_node(i, dirname, extra_args, rpchost, timewait, binary, stderr) + + def start_nodes(self, num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): + return _start_nodes(num_nodes, dirname, extra_args, rpchost, timewait, binary) + + def stop_node(self, num_node): + _stop_node(self.nodes[num_node], num_node) + + def stop_nodes(self): + _stop_nodes(self.nodes) + + def split_network(self): + """ + Split the network of four nodes into nodes 0/1 and 2/3. + """ + disconnect_nodes(self.nodes[1], 2) + disconnect_nodes(self.nodes[2], 1) + self.sync_all([self.nodes[:2], self.nodes[2:]]) + + def join_network(self): + """ + Join the (previously split) network halves together. + """ + connect_nodes_bi(self.nodes, 1, 2) + self.sync_all() + + def sync_all(self, node_groups=None): + if not node_groups: + node_groups = [self.nodes] + + for group in node_groups: + sync_blocks(group) + sync_mempools(group) + + # Private helper methods. These should not be accessed by the subclass test scripts. + + def _start_logging(self): + # Add logger and logging handlers + self.log = logging.getLogger('TestFramework') + self.log.setLevel(logging.DEBUG) + # Create file handler to log all messages + fh = logging.FileHandler(self.options.tmpdir + '/test_framework.log') + fh.setLevel(logging.DEBUG) + # Create console handler to log messages to stderr. By default this logs only error messages, but can be configured with --loglevel. + ch = logging.StreamHandler(sys.stdout) + # User can provide log level as a number or string (eg DEBUG). loglevel was caught as a string, so try to convert it to an int + ll = int(self.options.loglevel) if self.options.loglevel.isdigit() else self.options.loglevel.upper() + ch.setLevel(ll) + # Format logs the same as bitcoind's debug.log with microprecision (so log files can be concatenated and sorted) + formatter = logging.Formatter(fmt = '%(asctime)s.%(msecs)03d000 %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + formatter.converter = time.gmtime + fh.setFormatter(formatter) + ch.setFormatter(formatter) + # add the handlers to the logger + self.log.addHandler(fh) + self.log.addHandler(ch) + + if self.options.trace_rpc: + rpc_logger = logging.getLogger("BitcoinRPC") + rpc_logger.setLevel(logging.DEBUG) + rpc_handler = logging.StreamHandler(sys.stdout) + rpc_handler.setLevel(logging.DEBUG) + rpc_logger.addHandler(rpc_handler) + + def _initialize_chain(self, test_dir, num_nodes, cachedir): + """Initialize a pre-mined blockchain for use by the test. + + Create a cache of a 200-block-long chain (with wallet) for MAX_NODES + Afterward, create num_nodes copies from the cache.""" + + assert num_nodes <= MAX_NODES + create_cache = False + for i in range(MAX_NODES): + if not os.path.isdir(os.path.join(cachedir, 'node' + str(i))): + create_cache = True + break + + if create_cache: + self.log.debug("Creating data directories from cached datadir") + + # find and delete old cache directories if any exist + for i in range(MAX_NODES): + if os.path.isdir(os.path.join(cachedir, "node" + str(i))): + shutil.rmtree(os.path.join(cachedir, "node" + str(i))) + + # Create cache directories, run bitcoinds: + for i in range(MAX_NODES): + datadir = initialize_datadir(cachedir, i) + args = [os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"] + if i > 0: + args.append("-connect=127.0.0.1:" + str(p2p_port(0))) + bitcoind_processes[i] = subprocess.Popen(args) + self.log.debug("initialize_chain: bitcoind started, waiting for RPC to come up") + wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i) + self.log.debug("initialize_chain: RPC successfully started") + + self.nodes = [] + for i in range(MAX_NODES): + try: + self.nodes.append(get_rpc_proxy(rpc_url(i), i)) + except: + self.log.exception("Error connecting to node %d" % i) + sys.exit(1) + + # Create a 200-block-long chain; each of the 4 first nodes + # gets 25 mature blocks and 25 immature. + # Note: To preserve compatibility with older versions of + # initialize_chain, only 4 nodes will generate coins. + # + # blocks are created with timestamps 10 minutes apart + # starting from 2010 minutes in the past + enable_mocktime() + block_time = get_mocktime() - (201 * 10 * 60) + for i in range(2): + for peer in range(4): + for j in range(25): + set_node_times(self.nodes, block_time) + self.nodes[peer].generate(1) + block_time += 10 * 60 + # Must sync before next peer starts generating blocks + sync_blocks(self.nodes) + + # Shut them down, and clean up cache directories: + self.stop_nodes() + self.nodes = [] + disable_mocktime() + for i in range(MAX_NODES): + os.remove(log_filename(cachedir, i, "debug.log")) + os.remove(log_filename(cachedir, i, "db.log")) + os.remove(log_filename(cachedir, i, "peers.dat")) + os.remove(log_filename(cachedir, i, "fee_estimates.dat")) + + for i in range(num_nodes): + from_dir = os.path.join(cachedir, "node" + str(i)) + to_dir = os.path.join(test_dir, "node" + str(i)) + shutil.copytree(from_dir, to_dir) + initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf + + def _initialize_chain_clean(self, test_dir, num_nodes): + """Initialize empty blockchain for use by the test. + + Create an empty blockchain and num_nodes wallets. + Useful if a test case wants complete control over initialization.""" + for i in range(num_nodes): + initialize_datadir(test_dir, i) + +# Test framework for doing p2p comparison testing, which sets up some bitcoind +# binaries: +# 1 binary: test binary +# 2 binaries: 1 test binary, 1 ref binary +# n>2 binaries: 1 test binary, n-1 ref binaries + +class SkipTest(Exception): + """This exception is raised to skip a test""" + def __init__(self, message): + self.message = message + +class ComparisonTestFramework(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = True + + def add_options(self, parser): + parser.add_option("--testbinary", dest="testbinary", + default=os.getenv("BITCOIND", "bitcoind"), + help="bitcoind binary to test") + parser.add_option("--refbinary", dest="refbinary", + default=os.getenv("BITCOIND", "bitcoind"), + help="bitcoind binary to use for reference nodes (if any)") + + def setup_network(self): + self.nodes = self.start_nodes( + self.num_nodes, self.options.tmpdir, + extra_args=[['-whitelist=127.0.0.1']] * self.num_nodes, + binary=[self.options.testbinary] + + [self.options.refbinary]*(self.num_nodes-1)) diff --git a/qa/rpc-tests/test_framework/util.py b/test/functional/test_framework/util.py index e838a40582..2b0f32c2b6 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -15,15 +15,19 @@ import http.client import random import shutil import subprocess +import tempfile import time import re import errno +import logging from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException COVERAGE_DIR = None +logger = logging.getLogger("TestFramework.utils") + # The maximum number of nodes a single test can spawn MAX_NODES = 8 # Don't assign rpc or p2p ports lower than this @@ -222,125 +226,22 @@ def wait_for_bitcoind_start(process, url, i): raise # unknown JSON RPC exception time.sleep(0.25) -def initialize_chain(test_dir, num_nodes, cachedir): - """ - Create a cache of a 200-block-long chain (with wallet) for MAX_NODES - Afterward, create num_nodes copies from the cache - """ - - assert num_nodes <= MAX_NODES - create_cache = False - for i in range(MAX_NODES): - if not os.path.isdir(os.path.join(cachedir, 'node'+str(i))): - create_cache = True - break - - if create_cache: - - #find and delete old cache directories if any exist - for i in range(MAX_NODES): - if os.path.isdir(os.path.join(cachedir,"node"+str(i))): - shutil.rmtree(os.path.join(cachedir,"node"+str(i))) - - # Create cache directories, run bitcoinds: - for i in range(MAX_NODES): - datadir=initialize_datadir(cachedir, i) - args = [ os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir="+datadir, "-discover=0" ] - if i > 0: - args.append("-connect=127.0.0.1:"+str(p2p_port(0))) - bitcoind_processes[i] = subprocess.Popen(args) - if os.getenv("PYTHON_DEBUG", ""): - print("initialize_chain: bitcoind started, waiting for RPC to come up") - wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i) - if os.getenv("PYTHON_DEBUG", ""): - print("initialize_chain: RPC successfully started") - - rpcs = [] - for i in range(MAX_NODES): - try: - rpcs.append(get_rpc_proxy(rpc_url(i), i)) - except: - sys.stderr.write("Error connecting to "+url+"\n") - sys.exit(1) - - # Create a 200-block-long chain; each of the 4 first nodes - # gets 25 mature blocks and 25 immature. - # Note: To preserve compatibility with older versions of - # initialize_chain, only 4 nodes will generate coins. - # - # blocks are created with timestamps 10 minutes apart - # starting from 2010 minutes in the past - enable_mocktime() - block_time = get_mocktime() - (201 * 10 * 60) - for i in range(2): - for peer in range(4): - for j in range(25): - set_node_times(rpcs, block_time) - rpcs[peer].generate(1) - block_time += 10*60 - # Must sync before next peer starts generating blocks - sync_blocks(rpcs) - - # Shut them down, and clean up cache directories: - stop_nodes(rpcs) - disable_mocktime() - for i in range(MAX_NODES): - os.remove(log_filename(cachedir, i, "debug.log")) - os.remove(log_filename(cachedir, i, "db.log")) - os.remove(log_filename(cachedir, i, "peers.dat")) - os.remove(log_filename(cachedir, i, "fee_estimates.dat")) - - for i in range(num_nodes): - from_dir = os.path.join(cachedir, "node"+str(i)) - to_dir = os.path.join(test_dir, "node"+str(i)) - shutil.copytree(from_dir, to_dir) - initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf - -def initialize_chain_clean(test_dir, num_nodes): - """ - Create an empty blockchain and num_nodes wallets. - Useful if a test case wants complete control over initialization. - """ - for i in range(num_nodes): - datadir=initialize_datadir(test_dir, i) - - -def _rpchost_to_args(rpchost): - '''Convert optional IP:port spec to rpcconnect/rpcport args''' - if rpchost is None: - return [] - - match = re.match('(\[[0-9a-fA-f:]+\]|[^:]+)(?::([0-9]+))?$', rpchost) - if not match: - raise ValueError('Invalid RPC host spec ' + rpchost) - - rpcconnect = match.group(1) - rpcport = match.group(2) - if rpcconnect.startswith('['): # remove IPv6 [...] wrapping - rpcconnect = rpcconnect[1:-1] +def _start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): + """Start a bitcoind and return RPC connection to it - rv = ['-rpcconnect=' + rpcconnect] - if rpcport: - rv += ['-rpcport=' + rpcport] - return rv + This function should only be called from within test_framework, not by individual test scripts.""" -def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): - """ - Start a bitcoind and return RPC connection to it - """ datadir = os.path.join(dirname, "node"+str(i)) if binary is None: binary = os.getenv("BITCOIND", "bitcoind") - args = [ binary, "-datadir="+datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-mocktime="+str(get_mocktime()) ] + args = [binary, "-datadir=" + datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(get_mocktime()), "-uacomment=testnode%d" % i] if extra_args is not None: args.extend(extra_args) - bitcoind_processes[i] = subprocess.Popen(args) - if os.getenv("PYTHON_DEBUG", ""): - print("start_node: bitcoind started, waiting for RPC to come up") + bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr) + logger.debug("initialize_chain: bitcoind started, waiting for RPC to come up") url = rpc_url(i, rpchost) wait_for_bitcoind_start(bitcoind_processes[i], url, i) - if os.getenv("PYTHON_DEBUG", ""): - print("start_node: RPC successfully started") + logger.debug("initialize_chain: RPC successfully started") proxy = get_rpc_proxy(url, i, timeout=timewait) if COVERAGE_DIR: @@ -348,42 +249,84 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= return proxy -def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): - """ - Start multiple bitcoinds, return RPC connections to them - """ +def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=None): + with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: + try: + node = _start_node(i, dirname, extra_args, stderr=log_stderr) + _stop_node(node, i) + except Exception as e: + assert 'bitcoind exited' in str(e) #node must have shutdown + if expected_msg is not None: + log_stderr.seek(0) + stderr = log_stderr.read().decode('utf-8') + if expected_msg not in stderr: + raise AssertionError("Expected error \"" + expected_msg + "\" not found in:\n" + stderr) + else: + if expected_msg is None: + assert_msg = "bitcoind should have exited with an error" + else: + assert_msg = "bitcoind should have exited with expected error " + expected_msg + raise AssertionError(assert_msg) + +def _start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): + """Start multiple bitcoinds, return RPC connections to them + + This function should only be called from within test_framework, not by individual test scripts.""" + if extra_args is None: extra_args = [ None for _ in range(num_nodes) ] if binary is None: binary = [ None for _ in range(num_nodes) ] + assert_equal(len(extra_args), num_nodes) + assert_equal(len(binary), num_nodes) rpcs = [] try: for i in range(num_nodes): - rpcs.append(start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) + rpcs.append(_start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) except: # If one node failed to start, stop the others - stop_nodes(rpcs) + _stop_nodes(rpcs) raise return rpcs def log_filename(dirname, n_node, logname): return os.path.join(dirname, "node"+str(n_node), "regtest", logname) -def stop_node(node, i): +def _stop_node(node, i): + """Stop a bitcoind test node + + This function should only be called from within test_framework, not by individual test scripts.""" + + logger.debug("Stopping node %d" % i) try: node.stop() except http.client.CannotSendRequest as e: - print("WARN: Unable to stop node: " + repr(e)) + logger.exception("Unable to stop node") return_code = bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) assert_equal(return_code, 0) del bitcoind_processes[i] -def stop_nodes(nodes): +def _stop_nodes(nodes): + """Stop multiple bitcoind test nodes + + This function should only be called from within test_framework, not by individual test scripts.""" + for i, node in enumerate(nodes): - stop_node(node, i) + _stop_node(node, i) assert not bitcoind_processes.values() # All connections must be gone now def set_node_times(nodes, t): for node in nodes: node.setmocktime(t) +def disconnect_nodes(from_connection, node_num): + for peer_id in [peer['id'] for peer in from_connection.getpeerinfo() if "testnode%d" % node_num in peer['subver']]: + from_connection.disconnectnode(nodeid=peer_id) + + for _ in range(50): + if [peer['id'] for peer in from_connection.getpeerinfo() if "testnode%d" % node_num in peer['subver']] == []: + break + time.sleep(0.1) + else: + raise AssertionError("timed out waiting for disconnect") + def connect_nodes(from_connection, node_num): ip_port = "127.0.0.1:"+str(p2p_port(node_num)) from_connection.addnode(ip_port, "onetry") @@ -442,47 +385,6 @@ def make_change(from_node, amount_in, amount_out, fee): outputs[from_node.getnewaddress()] = change return outputs -def send_zeropri_transaction(from_node, to_node, amount, fee): - """ - Create&broadcast a zero-priority transaction. - Returns (txid, hex-encoded-txdata) - Ensures transaction is zero-priority by first creating a send-to-self, - then using its output - """ - - # Create a send-to-self with confirmed inputs: - self_address = from_node.getnewaddress() - (total_in, inputs) = gather_inputs(from_node, amount+fee*2) - outputs = make_change(from_node, total_in, amount+fee, fee) - outputs[self_address] = float(amount+fee) - - self_rawtx = from_node.createrawtransaction(inputs, outputs) - self_signresult = from_node.signrawtransaction(self_rawtx) - self_txid = from_node.sendrawtransaction(self_signresult["hex"], True) - - vout = find_output(from_node, self_txid, amount+fee) - # Now immediately spend the output to create a 1-input, 1-output - # zero-priority transaction: - inputs = [ { "txid" : self_txid, "vout" : vout } ] - outputs = { to_node.getnewaddress() : float(amount) } - - rawtx = from_node.createrawtransaction(inputs, outputs) - signresult = from_node.signrawtransaction(rawtx) - txid = from_node.sendrawtransaction(signresult["hex"], True) - - return (txid, signresult["hex"]) - -def random_zeropri_transaction(nodes, amount, min_fee, fee_increment, fee_variants): - """ - Create a random zero-priority transaction. - Returns (txid, hex-encoded-transaction-data, fee) - """ - from_node = random.choice(nodes) - to_node = random.choice(nodes) - fee = min_fee + fee_increment*random.randint(0,fee_variants) - (txid, txhex) = send_zeropri_transaction(from_node, to_node, amount, fee) - return (txid, txhex, fee) - def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): """ Create a random transaction. diff --git a/qa/pull-tester/rpc-tests.py b/test/functional/test_runner.py index 68e11b4c2b..b2aee7c739 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/test/functional/test_runner.py @@ -2,21 +2,21 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -""" -rpc-tests.py - run regression test suite +"""Run regression test suite. This module calls down into individual test cases via subprocess. It will forward all unrecognized arguments onto the individual test scripts. -RPC tests are disabled on Windows by default. Use --force to run them anyway. +Functional tests are disabled on Windows by default. Use --force to run them anyway. For a description of arguments recognized by test scripts, see -`qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`. +`test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`. """ import argparse import configparser +import datetime import os import time import shutil @@ -24,6 +24,31 @@ import sys import subprocess import tempfile import re +import logging + +# Formatting. Default colors to empty strings. +BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") +try: + # Make sure python thinks it can write unicode to its stdout + "\u2713".encode("utf_8").decode(sys.stdout.encoding) + TICK = "✓ " + CROSS = "✖ " + CIRCLE = "○ " +except UnicodeDecodeError: + TICK = "P " + CROSS = "x " + CIRCLE = "o " + +if os.name == 'posix': + # primitive formatting on supported + # terminal via ANSI escape sequences: + BOLD = ('\033[0m', '\033[1m') + BLUE = ('\033[0m', '\033[0;34m') + RED = ('\033[0m', '\033[0;31m') + GREY = ('\033[0m', '\033[1;30m') + +TEST_EXIT_PASSED = 0 +TEST_EXIT_SKIPPED = 77 BASE_SCRIPTS= [ # Scripts that are run by the travis build process. @@ -53,6 +78,7 @@ BASE_SCRIPTS= [ 'rawtransactions.py', 'reindex.py', # vv Tests less than 30s vv + "zmq_test.py", 'mempool_resurrect_test.py', 'txn_doublespend.py --mineblock', 'txn_clone.py', @@ -60,14 +86,16 @@ BASE_SCRIPTS= [ 'rest.py', 'mempool_spendcoinbase.py', 'mempool_reorg.py', + 'mempool_persist.py', 'httpbasics.py', 'multi_rpc.py', 'proxy_test.py', 'signrawtransactions.py', - 'nodehandling.py', + 'disconnect_ban.py', 'decodescript.py', 'blockchain.py', 'disablewallet.py', + 'net.py', 'keypool.py', 'p2p-mempool.py', 'prioritise_transaction.py', @@ -85,11 +113,6 @@ BASE_SCRIPTS= [ 'p2p-leaktests.py', ] -ZMQ_SCRIPTS = [ - # ZMQ test can only be run if bitcoin was built with zmq-enabled. - # call rpc_tests.py with -nozmq to explicitly exclude these tests. - "zmq_test.py"] - EXTENDED_SCRIPTS = [ # These tests are not run by the travis build process. # Longest test should go first, to favor running tests in parallel @@ -108,6 +131,7 @@ EXTENDED_SCRIPTS = [ 'p2p-feefilter.py', 'rpcbind_test.py', # vv Tests less than 30s vv + 'assumevalid.py', 'bip65-cltv.py', 'bip65-cltv-p2p.py', 'bipdersig-p2p.py', @@ -117,42 +141,63 @@ EXTENDED_SCRIPTS = [ 'txn_clone.py --mineblock', 'forknotify.py', 'invalidateblock.py', - 'maxblocksinflight.py', 'p2p-acceptblock.py', 'replace-by-fee.py', ] -ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS +# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests +ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS + +NON_SCRIPTS = [ + # These are python files that live in the functional tests directory, but are not test scripts. + "combine_logs.py", + "create_cache.py", + "test_runner.py", +] def main(): # Parse arguments and pass through unrecognised args parser = argparse.ArgumentParser(add_help=False, - usage='%(prog)s [rpc-test.py options] [script options] [scripts]', + usage='%(prog)s [test_runner.py options] [script options] [scripts]', description=__doc__, epilog=''' Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') - parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude. Do not include the .py extension in the name.') + parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') - parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') + parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') + parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') + parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") args, unknown_args = parser.parse_known_args() - # Create a set to store arguments and create the passon string - tests = set(arg for arg in unknown_args if arg[:2] != "--") + # args to be passed on always start with two dashes; tests are the remaining unknown args + tests = [arg for arg in unknown_args if arg[:2] != "--"] passon_args = [arg for arg in unknown_args if arg[:2] == "--"] # Read config generated by configure. config = configparser.ConfigParser() - config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) + configfile = os.path.abspath(os.path.dirname(__file__)) + "/config.ini" + config.read_file(open(configfile)) + + passon_args.append("--configfile=%s" % configfile) + + # Set up logging + logging_level = logging.INFO if args.quiet else logging.DEBUG + logging.basicConfig(format='%(message)s', level=logging_level) + + # Create base test directory + tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) + os.makedirs(tmpdir) + + logging.debug("Temporary test directory at %s" % tmpdir) enable_wallet = config["components"].getboolean("ENABLE_WALLET") enable_utils = config["components"].getboolean("ENABLE_UTILS") enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") - enable_zmq = config["components"].getboolean("ENABLE_ZMQ") and not args.nozmq if config["environment"]["EXEEXT"] == ".exe" and not args.force: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 @@ -161,125 +206,154 @@ def main(): sys.exit(0) if not (enable_wallet and enable_utils and enable_bitcoind): - print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") + print("No functional tests to run. Wallet, utils, and bitcoind must all be enabled") print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") sys.exit(0) - # python3-zmq may not be installed. Handle this gracefully and with some helpful info - if enable_zmq: - try: - import zmq - except ImportError: - print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." - "To run zmq tests, see dependency info in /qa/README.md.") - raise - # Build list of tests if tests: # Individual tests have been specified. Run specified tests that exist # in the ALL_SCRIPTS list. Accept the name with or without .py extension. - test_list = [t for t in ALL_SCRIPTS if - (t in tests or re.sub(".py$", "", t) in tests)] + tests = [re.sub("\.py$", "", t) + ".py" for t in tests] + test_list = [] + for t in tests: + if t in ALL_SCRIPTS: + test_list.append(t) + else: + print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t)) else: - # No individual tests have been specified. Run base tests, and - # optionally ZMQ tests and extended tests. + # No individual tests have been specified. + # Run all base tests, and optionally run extended tests. test_list = BASE_SCRIPTS - if enable_zmq: - test_list += ZMQ_SCRIPTS if args.extended: - test_list += EXTENDED_SCRIPTS - # TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime - # (for parallel running efficiency). This combined list will is no - # longer sorted. + # place the EXTENDED_SCRIPTS first since the three longest ones + # are there and the list is shorter + test_list = EXTENDED_SCRIPTS + test_list # Remove the test cases that the user has explicitly asked to exclude. if args.exclude: - for exclude_test in args.exclude.split(','): - if exclude_test + ".py" in test_list: - test_list.remove(exclude_test + ".py") + tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')] + for exclude_test in tests_excl: + if exclude_test in test_list: + test_list.remove(exclude_test) + else: + print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test)) if not test_list: print("No valid test scripts specified. Check that your test is in one " - "of the test lists in rpc-tests.py, or run rpc-tests.py with no arguments to run all tests") + "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests") sys.exit(0) if args.help: - # Print help for rpc-tests.py, then print help of the first script and exit. + # Print help for test_runner.py, then print help of the first script (with args removed) and exit. parser.print_help() - subprocess.check_call((config["environment"]["SRCDIR"] + '/qa/rpc-tests/' + test_list[0]).split() + ['-h']) + subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h']) sys.exit(0) - run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) + check_script_list(config["environment"]["SRCDIR"]) + + if not args.keepcache: + shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) + + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args) + +def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]): + # Warn if bitcoind is already running (unix only) + try: + if subprocess.check_output(["pidof", "bitcoind"]) is not None: + print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) + except (OSError, subprocess.SubprocessError): + pass -def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): - BOLD = ("","") - if os.name == 'posix': - # primitive formatting on supported - # terminal via ANSI escape sequences: - BOLD = ('\033[0m', '\033[1m') + # Warn if there is a cache directory + cache_dir = "%s/test/cache" % build_dir + if os.path.isdir(cache_dir): + print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir)) #Set env vars if "BITCOIND" not in os.environ: os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext - tests_dir = src_dir + '/qa/rpc-tests/' + tests_dir = src_dir + '/test/functional/' flags = ["--srcdir={}/src".format(build_dir)] + args - flags.append("--cachedir=%s/qa/cache" % build_dir) + flags.append("--cachedir=%s" % cache_dir) if enable_coverage: coverage = RPCCoverage() flags.append(coverage.flag) - print("Initializing coverage directory at %s\n" % coverage.dir) + logging.debug("Initializing coverage directory at %s" % coverage.dir) else: coverage = None if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([tests_dir + 'create_cache.py'] + flags) + subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) #Run Tests - all_passed = True - time_sum = 0 + job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) time0 = time.time() - - job_queue = RPCTestHandler(jobs, tests_dir, test_list, flags) + test_results = [] max_len_name = len(max(test_list, key=len)) - results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] - for _ in range(len(test_list)): - (name, stdout, stderr, passed, duration) = job_queue.get_next() - all_passed = all_passed and passed - time_sum += duration - print('\n' + BOLD[1] + name + BOLD[0] + ":") - print('' if passed else stdout + '\n', end='') - print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='') - print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) + for _ in range(len(test_list)): + test_result, stdout, stderr = job_queue.get_next() + test_results.append(test_result) - results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) + if test_result.status == "Passed": + logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) + elif test_result.status == "Skipped": + logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0])) + else: + print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) + print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') + print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') - results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] - print(results) - print("\nRuntime: %s s" % (int(time.time() - time0))) + print_results(test_results, max_len_name, (int(time.time() - time0))) if coverage: coverage.report_rpc_coverage() - print("Cleaning up coverage data") + logging.debug("Cleaning up coverage data") coverage.cleanup() + # Clear up the temp directory if all subdirectories are gone + if not os.listdir(tmpdir): + os.rmdir(tmpdir) + + all_passed = all(map(lambda test_result: test_result.was_successful, test_results)) + sys.exit(not all_passed) -class RPCTestHandler: +def print_results(test_results, max_len_name, runtime): + results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] + + test_results.sort(key=lambda result: result.name.lower()) + all_passed = True + time_sum = 0 + + for test_result in test_results: + all_passed = all_passed and test_result.was_successful + time_sum += test_result.time + test_result.padding = max_len_name + results += str(test_result) + + status = TICK + "Passed" if all_passed else CROSS + "Failed" + results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0] + results += "Runtime: %s s\n" % (runtime) + print(results) + +class TestHandler: """ Trigger the testscrips passed in via the list. """ - def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None): + def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None): assert(num_tests_parallel >= 1) self.num_jobs = num_tests_parallel self.tests_dir = tests_dir + self.tmpdir = tmpdir self.test_list = test_list self.flags = flags self.num_running = 0 @@ -294,12 +368,15 @@ class RPCTestHandler: # Add tests self.num_running += 1 t = self.test_list.pop(0) - port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)] + portseed = len(self.test_list) + self.portseed_offset + portseed_arg = ["--portseed={}".format(portseed)] log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) + test_argv = t.split() + tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)] self.jobs.append((t, time.time(), - subprocess.Popen((self.tests_dir + t).split() + self.flags + port_seed, + subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir, universal_newlines=True, stdout=log_stdout, stderr=log_stderr), @@ -316,16 +393,60 @@ class RPCTestHandler: log_out.seek(0), log_err.seek(0) [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)] log_out.close(), log_err.close() - passed = stderr == "" and proc.returncode == 0 + if proc.returncode == TEST_EXIT_PASSED and stderr == "": + status = "Passed" + elif proc.returncode == TEST_EXIT_SKIPPED: + status = "Skipped" + else: + status = "Failed" self.num_running -= 1 self.jobs.remove(j) - return name, stdout, stderr, passed, int(time.time() - time0) + + return TestResult(name, status, int(time.time() - time0)), stdout, stderr print('.', end='', flush=True) +class TestResult(): + def __init__(self, name, status, time): + self.name = name + self.status = status + self.time = time + self.padding = 0 + + def __repr__(self): + if self.status == "Passed": + color = BLUE + glyph = TICK + elif self.status == "Failed": + color = RED + glyph = CROSS + elif self.status == "Skipped": + color = GREY + glyph = CIRCLE + + return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] + + @property + def was_successful(self): + return self.status != "Failed" + + +def check_script_list(src_dir): + """Check scripts directory. + + Check that there are no scripts in the functional tests directory which are + not being run by pull-tester.py.""" + script_dir = src_dir + '/test/functional/' + python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"]) + missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) + if len(missed_tests) != 0: + print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) + if os.getenv('TRAVIS') == 'true': + # On travis this warning is an error to prevent merging incomplete commits into master + sys.exit(1) class RPCCoverage(object): """ - Coverage reporting utilities for pull-tester. + Coverage reporting utilities for test_runner. Coverage calculation works by having each test script subprocess write coverage files into a particular directory. These files contain the RPC @@ -335,7 +456,7 @@ class RPCCoverage(object): After all tests complete, the commands run are combined and diff'd against the complete list to calculate uncovered RPC commands. - See also: qa/rpc-tests/test_framework/coverage.py + See also: test/functional/test_framework/coverage.py """ def __init__(self): @@ -363,7 +484,7 @@ class RPCCoverage(object): Return a set of currently untested RPC commands. """ - # This is shared from `qa/rpc-tests/test-framework/coverage.py` + # This is shared from `test/functional/test-framework/coverage.py` reference_filename = 'rpc_interface.txt' coverage_file_prefix = 'coverage.' diff --git a/qa/rpc-tests/txn_clone.py b/test/functional/txn_clone.py index 7a3b8d3474..9b81af96cf 100755 --- a/qa/rpc-tests/txn_clone.py +++ b/test/functional/txn_clone.py @@ -20,7 +20,9 @@ class TxnMallTest(BitcoinTestFramework): def setup_network(self): # Start with split network: - return super(TxnMallTest, self).setup_network(True) + super(TxnMallTest, self).setup_network() + disconnect_nodes(self.nodes[1], 2) + disconnect_nodes(self.nodes[2], 1) def run_test(self): # All nodes should start with 1,250 BTC: diff --git a/qa/rpc-tests/txn_doublespend.py b/test/functional/txn_doublespend.py index 5b12cf4c29..1bd3b3271c 100755 --- a/qa/rpc-tests/txn_doublespend.py +++ b/test/functional/txn_doublespend.py @@ -20,7 +20,9 @@ class TxnMallTest(BitcoinTestFramework): def setup_network(self): # Start with split network: - return super(TxnMallTest, self).setup_network(True) + super().setup_network() + disconnect_nodes(self.nodes[1], 2) + disconnect_nodes(self.nodes[2], 1) def run_test(self): # All nodes should start with 1,250 BTC: diff --git a/qa/rpc-tests/wallet-accounts.py b/test/functional/wallet-accounts.py index ea12d4ec22..158aa9ae89 100755 --- a/qa/rpc-tests/wallet-accounts.py +++ b/test/functional/wallet-accounts.py @@ -7,16 +7,14 @@ RPCs tested are: - getaccountaddress - getaddressesbyaccount + - listaddressgroupings - setaccount - sendfrom (with account arguments) - move (with account arguments) """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - start_nodes, - assert_equal, -) +from test_framework.util import assert_equal class WalletAccountsTest(BitcoinTestFramework): @@ -24,22 +22,59 @@ class WalletAccountsTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 - self.node_args = [[]] + self.extra_args = [[]] - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.node_args) - self.is_network_split = False - - def run_test (self): + def run_test(self): node = self.nodes[0] # Check that there's no UTXO on any of the nodes assert_equal(len(node.listunspent()), 0) - + + # Note each time we call generate, all generated coins go into + # the same address, so we call twice to get two addresses w/50 each + node.generate(1) node.generate(101) - - assert_equal(node.getbalance(), 50) - - accounts = ["a","b","c","d","e"] + assert_equal(node.getbalance(), 100) + + # there should be 2 address groups + # each with 1 address with a balance of 50 Bitcoins + address_groups = node.listaddressgroupings() + assert_equal(len(address_groups), 2) + # the addresses aren't linked now, but will be after we send to the + # common address + linked_addresses = set() + for address_group in address_groups: + assert_equal(len(address_group), 1) + assert_equal(len(address_group[0]), 2) + assert_equal(address_group[0][1], 50) + linked_addresses.add(address_group[0][0]) + + # send 50 from each address to a third address not in this wallet + # There's some fee that will come back to us when the miner reward + # matures. + common_address = "msf4WtN1YQKXvNtvdFYt9JBnUD2FB41kjr" + txid = node.sendmany( + fromaccount="", + amounts={common_address: 100}, + subtractfeefrom=[common_address], + minconf=1, + ) + tx_details = node.gettransaction(txid) + fee = -tx_details['details'][0]['fee'] + # there should be 1 address group, with the previously + # unlinked addresses now linked (they both have 0 balance) + address_groups = node.listaddressgroupings() + assert_equal(len(address_groups), 1) + assert_equal(len(address_groups[0]), 2) + assert_equal(set([a[0] for a in address_groups[0]]), linked_addresses) + assert_equal([a[1] for a in address_groups[0]], [0, 0]) + + node.generate(1) + + # we want to reset so that the "" account has what's expected. + # otherwise we're off by exactly the fee amount as that's mined + # and matures in the next 100 blocks + node.sendfrom("", common_address, fee) + accounts = ["a", "b", "c", "d", "e"] amount_to_send = 1.0 account_addresses = dict() for account in accounts: @@ -56,7 +91,7 @@ class WalletAccountsTest(BitcoinTestFramework): for i in range(len(accounts)): from_account = accounts[i] - to_account = accounts[(i+1)%len(accounts)] + to_account = accounts[(i+1) % len(accounts)] to_address = account_addresses[to_account] node.sendfrom(from_account, to_address, amount_to_send) @@ -67,7 +102,7 @@ class WalletAccountsTest(BitcoinTestFramework): assert(address != account_addresses[account]) assert_equal(node.getreceivedbyaccount(account), 2) node.move(account, "", node.getbalance(account)) - + node.generate(101) expected_account_balances = {"": 5200} @@ -97,4 +132,4 @@ class WalletAccountsTest(BitcoinTestFramework): assert_equal(node.getbalance(account), 50) if __name__ == '__main__': - WalletAccountsTest().main () + WalletAccountsTest().main() diff --git a/qa/rpc-tests/wallet-dump.py b/test/functional/wallet-dump.py index b819b72b75..9cb32d4650 100755 --- a/qa/rpc-tests/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -4,8 +4,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" +import os + from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_nodes, start_node, assert_equal, bitcoind_processes) +from test_framework.util import (assert_equal, bitcoind_processes) def read_dump(file_name, addrs, hd_master_addr_old): @@ -66,7 +68,7 @@ class WalletDumpTest(BitcoinTestFramework): # longer than the default 30 seconds due to an expensive # CWallet::TopUpKeyPool call, and the encryptwallet RPC made later in # the test often takes even longer. - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args, timewait=60) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args, timewait=60) def run_test (self): tmpdir = self.options.tmpdir @@ -82,18 +84,19 @@ class WalletDumpTest(BitcoinTestFramework): self.nodes[0].keypoolrefill() # dump unencrypted wallet - self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump") + result = self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump") + assert_equal(result['filename'], os.path.abspath(tmpdir + "/node0/wallet.unencrypted.dump")) found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, None) assert_equal(found_addr, test_addr_count) # all keys must be in the dump assert_equal(found_addr_chg, 50) # 50 blocks where mined - assert_equal(found_addr_rsv, 90 + 1) # keypool size (TODO: fix off-by-one) + assert_equal(found_addr_rsv, 90*2) # 90 keys plus 100% internal keys #encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') bitcoind_processes[0].wait() - self.nodes[0] = start_node(0, self.options.tmpdir, self.extra_args[0]) + self.nodes[0] = self.start_node(0, self.options.tmpdir, self.extra_args[0]) self.nodes[0].walletpassphrase('test', 10) # Should be a no-op: self.nodes[0].keypoolrefill() @@ -102,8 +105,8 @@ class WalletDumpTest(BitcoinTestFramework): found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_enc = \ read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, hd_master_addr_unenc) assert_equal(found_addr, test_addr_count) - assert_equal(found_addr_chg, 90 + 1 + 50) # old reserve keys are marked as change now - assert_equal(found_addr_rsv, 90 + 1) # keypool size (TODO: fix off-by-one) + assert_equal(found_addr_chg, 90*2 + 50) # old reserve keys are marked as change now + assert_equal(found_addr_rsv, 90*2) if __name__ == '__main__': WalletDumpTest().main () diff --git a/qa/rpc-tests/wallet-hd.py b/test/functional/wallet-hd.py index bf07590098..e7ec72a248 100755 --- a/qa/rpc-tests/wallet-hd.py +++ b/test/functional/wallet-hd.py @@ -6,10 +6,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - start_nodes, - start_node, assert_equal, connect_nodes_bi, + assert_start_raises_init_error ) import os import shutil @@ -21,20 +20,26 @@ class WalletHDTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 2 - self.node_args = [['-usehd=0'], ['-usehd=1', '-keypool=0']] - - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.node_args) - self.is_network_split = False - connect_nodes_bi(self.nodes, 0, 1) + self.extra_args = [['-usehd=0'], ['-usehd=1', '-keypool=0']] def run_test (self): tmpdir = self.options.tmpdir + # Make sure can't switch off usehd after wallet creation + self.stop_node(1) + assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet') + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1]) + connect_nodes_bi(self.nodes, 0, 1) + # Make sure we use hd, keep masterkeyid masterkeyid = self.nodes[1].getwalletinfo()['hdmasterkeyid'] assert_equal(len(masterkeyid), 40) + # create an internal key + change_addr = self.nodes[1].getrawchangeaddress() + change_addrV= self.nodes[1].validateaddress(change_addr) + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key + # Import a non-HD private key in the HD wallet non_hd_add = self.nodes[0].getnewaddress() self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(non_hd_add)) @@ -58,14 +63,19 @@ class WalletHDTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(non_hd_add, 1) self.nodes[0].generate(1) + # create an internal key (again) + change_addr = self.nodes[1].getrawchangeaddress() + change_addrV= self.nodes[1].validateaddress(change_addr) + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key + self.sync_all() assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) - print("Restore backup ...") + self.log.info("Restore backup ...") self.stop_node(1) os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat") shutil.copyfile(tmpdir + "/hd.bak", tmpdir + "/node1/regtest/wallet.dat") - self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[1]) + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1]) #connect_nodes_bi(self.nodes, 0, 1) # Assert that derivation is deterministic @@ -79,10 +89,19 @@ class WalletHDTest(BitcoinTestFramework): # Needs rescan self.stop_node(1) - self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[1] + ['-rescan']) + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1] + ['-rescan']) #connect_nodes_bi(self.nodes, 0, 1) assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) + # send a tx and make sure its using the internal chain for the changeoutput + txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) + outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout'] + keypath = "" + for out in outs: + if out['value'] != 1: + keypath = self.nodes[1].validateaddress(out['scriptPubKey']['addresses'][0])['hdkeypath'] + + assert_equal(keypath[0:7], "m/0'/1'") if __name__ == '__main__': WalletHDTest().main () diff --git a/qa/rpc-tests/wallet.py b/test/functional/wallet.py index ddeac657e5..3e3e8fcddb 100755 --- a/qa/rpc-tests/wallet.py +++ b/test/functional/wallet.py @@ -6,7 +6,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class WalletTest (BitcoinTestFramework): +class WalletTest(BitcoinTestFramework): def check_fee_amount(self, curr_balance, balance_with_fee, fee_per_byte, tx_size): """Return curr_balance after asserting the fee was in range""" @@ -20,22 +20,21 @@ class WalletTest (BitcoinTestFramework): self.num_nodes = 4 self.extra_args = [['-usehd={:d}'.format(i%2==0)] for i in range(4)] - def setup_network(self, split=False): - self.nodes = start_nodes(3, self.options.tmpdir, self.extra_args[:3]) + def setup_network(self): + self.nodes = self.start_nodes(3, self.options.tmpdir, self.extra_args[:3]) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) - self.is_network_split=False self.sync_all() - def run_test (self): + def run_test(self): # Check that there's no UTXO on none of the nodes assert_equal(len(self.nodes[0].listunspent()), 0) assert_equal(len(self.nodes[1].listunspent()), 0) assert_equal(len(self.nodes[2].listunspent()), 0) - print("Mining blocks...") + self.log.info("Mining blocks...") self.nodes[0].generate(1) @@ -52,14 +51,38 @@ class WalletTest (BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 0) # Check that only first and second nodes have UTXOs - assert_equal(len(self.nodes[0].listunspent()), 1) + utxos = self.nodes[0].listunspent() + assert_equal(len(utxos), 1) assert_equal(len(self.nodes[1].listunspent()), 1) assert_equal(len(self.nodes[2].listunspent()), 0) # Send 21 BTC from 0 to 2 using sendtoaddress call. + # Locked memory should use at least 32 bytes to sign each transaction + self.log.info("test getmemoryinfo") + memory_before = self.nodes[0].getmemoryinfo() self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) - + mempool_txid = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) + memory_after = self.nodes[0].getmemoryinfo() + assert(memory_before['locked']['used'] + 64 <= memory_after['locked']['used']) + + self.log.info("test gettxout") + # utxo spent in mempool should be visible if you exclude mempool + # but invisible if you include mempool + confirmed_txid, confirmed_index = utxos[0]["txid"], utxos[0]["vout"] + txout = self.nodes[0].gettxout(confirmed_txid, confirmed_index, False) + assert_equal(txout['value'], 50) + txout = self.nodes[0].gettxout(confirmed_txid, confirmed_index, True) + assert txout is None + # new utxo from mempool should be invisible if you exclude mempool + # but visible if you include mempool + txout = self.nodes[0].gettxout(mempool_txid, 0, False) + assert txout is None + txout1 = self.nodes[0].gettxout(mempool_txid, 0, True) + txout2 = self.nodes[0].gettxout(mempool_txid, 1, True) + # note the mempool tx will have randomly assigned indices + # but 10 will go to node2 and the rest will go to node0 + balance = self.nodes[0].getbalance() + assert_equal(set([txout1['value'], txout2['value']]), set([10, balance])) walletinfo = self.nodes[0].getwalletinfo() assert_equal(walletinfo['immature_balance'], 0) @@ -71,7 +94,7 @@ class WalletTest (BitcoinTestFramework): unspent_0 = self.nodes[2].listunspent()[0] unspent_0 = {"txid": unspent_0["txid"], "vout": unspent_0["vout"]} self.nodes[2].lockunspent(False, [unspent_0]) - assert_raises_message(JSONRPCException, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) + assert_raises_jsonrpc(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) assert_equal([unspent_0], self.nodes[2].listlockunspent()) self.nodes[2].lockunspent(True, [unspent_0]) assert_equal(len(self.nodes[2].listlockunspent()), 0) @@ -155,7 +178,7 @@ class WalletTest (BitcoinTestFramework): txid2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) sync_mempools(self.nodes) - self.nodes.append(start_node(3, self.options.tmpdir, self.extra_args[3])) + self.nodes.append(self.start_node(3, self.options.tmpdir, self.extra_args[3])) connect_nodes_bi(self.nodes, 0, 3) sync_blocks(self.nodes) @@ -198,8 +221,8 @@ class WalletTest (BitcoinTestFramework): assert(found) #do some -walletbroadcast tests - stop_nodes(self.nodes) - self.nodes = start_nodes(3, self.options.tmpdir, [["-walletbroadcast=0"],["-walletbroadcast=0"],["-walletbroadcast=0"]]) + self.stop_nodes() + self.nodes = self.start_nodes(3, self.options.tmpdir, [["-walletbroadcast=0"],["-walletbroadcast=0"],["-walletbroadcast=0"]]) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -223,8 +246,8 @@ class WalletTest (BitcoinTestFramework): txIdNotBroadcasted = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 2) #restart the nodes with -walletbroadcast=1 - stop_nodes(self.nodes) - self.nodes = start_nodes(3, self.options.tmpdir) + self.stop_nodes() + self.nodes = self.start_nodes(3, self.options.tmpdir) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -251,19 +274,11 @@ class WalletTest (BitcoinTestFramework): txObj = self.nodes[0].gettransaction(txId) assert_equal(txObj['amount'], Decimal('-0.0001')) - try: - txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "1f-4") - except JSONRPCException as e: - assert("Invalid amount" in e.error['message']) - else: - raise AssertionError("Must not parse invalid amounts") - + # This will raise an exception because the amount type is wrong + assert_raises_jsonrpc(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") - try: - self.nodes[0].generate("2") - raise AssertionError("Must not accept strings as numeric") - except JSONRPCException as e: - assert("not an integer" in e.error['message']) + # This will raise an exception since generate does not accept a string + assert_raises_jsonrpc(-1, "not an integer", self.nodes[0].generate, "2") # Import address and private key to check correct behavior of spendable unspents # 1. Send some coins to generate new UTXO @@ -332,10 +347,10 @@ class WalletTest (BitcoinTestFramework): ] chainlimit = 6 for m in maintenance: - print("check " + m) - stop_nodes(self.nodes) + self.log.info("check " + m) + self.stop_nodes() # set lower ancestor limit for later - self.nodes = start_nodes(3, self.options.tmpdir, [[m, "-limitancestorcount="+str(chainlimit)]] * 3) + self.nodes = self.start_nodes(3, self.options.tmpdir, [[m, "-limitancestorcount="+str(chainlimit)]] * 3) while m == '-reindex' and [block_count] * 3 != [self.nodes[i].getblockcount() for i in range(3)]: # reindex will leave rpc warm up "early"; Wait for it to finish time.sleep(0.1) @@ -382,8 +397,8 @@ class WalletTest (BitcoinTestFramework): # Try with walletrejectlongchains # Double chain limit but require combining inputs, so we pass SelectCoinsMinConf - stop_node(self.nodes[0],0) - self.nodes[0] = start_node(0, self.options.tmpdir, ["-walletrejectlongchains", "-limitancestorcount="+str(2*chainlimit)]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-walletrejectlongchains", "-limitancestorcount="+str(2*chainlimit)]) # wait for loadmempool timeout = 10 @@ -394,7 +409,7 @@ class WalletTest (BitcoinTestFramework): node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. - assert_raises_message(JSONRPCException, "mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) + assert_raises_jsonrpc(-4, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) # Verify nothing new in wallet assert_equal(total_txs, len(self.nodes[0].listtransactions("*",99999))) diff --git a/qa/rpc-tests/walletbackup.py b/test/functional/walletbackup.py index ea249096cf..a4507182a2 100755 --- a/qa/rpc-tests/walletbackup.py +++ b/test/functional/walletbackup.py @@ -34,8 +34,6 @@ and confirm again balances are correct. from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from random import randint -import logging -logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO, stream=sys.stdout) class WalletBackupTest(BitcoinTestFramework): @@ -46,14 +44,12 @@ class WalletBackupTest(BitcoinTestFramework): # nodes 1, 2,3 are spenders, let's give them a keypool=100 self.extra_args = [["-keypool=100"], ["-keypool=100"], ["-keypool=100"], []] - # This mirrors how the network was setup in the bash test def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.setup_nodes() connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) connect_nodes(self.nodes[2], 3) connect_nodes(self.nodes[2], 0) - self.is_network_split=False self.sync_all() def one_send(self, from_node, to_address): @@ -81,18 +77,18 @@ class WalletBackupTest(BitcoinTestFramework): # As above, this mirrors the original bash test. def start_three(self): - self.nodes[0] = start_node(0, self.options.tmpdir) - self.nodes[1] = start_node(1, self.options.tmpdir) - self.nodes[2] = start_node(2, self.options.tmpdir) + self.nodes[0] = self.start_node(0, self.options.tmpdir) + self.nodes[1] = self.start_node(1, self.options.tmpdir) + self.nodes[2] = self.start_node(2, self.options.tmpdir) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) connect_nodes(self.nodes[2], 3) connect_nodes(self.nodes[2], 0) def stop_three(self): - stop_node(self.nodes[0], 0) - stop_node(self.nodes[1], 1) - stop_node(self.nodes[2], 2) + self.stop_node(0) + self.stop_node(1) + self.stop_node(2) def erase_three(self): os.remove(self.options.tmpdir + "/node0/regtest/wallet.dat") @@ -100,7 +96,7 @@ class WalletBackupTest(BitcoinTestFramework): os.remove(self.options.tmpdir + "/node2/regtest/wallet.dat") def run_test(self): - logging.info("Generating initial blockchain") + self.log.info("Generating initial blockchain") self.nodes[0].generate(1) sync_blocks(self.nodes) self.nodes[1].generate(1) @@ -115,12 +111,12 @@ class WalletBackupTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 50) assert_equal(self.nodes[3].getbalance(), 0) - logging.info("Creating transactions") + self.log.info("Creating transactions") # Five rounds of sending each other transactions. for i in range(5): self.do_one_round() - logging.info("Backing up") + self.log.info("Backing up") tmpdir = self.options.tmpdir self.nodes[0].backupwallet(tmpdir + "/node0/wallet.bak") self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.dump") @@ -129,7 +125,7 @@ class WalletBackupTest(BitcoinTestFramework): self.nodes[2].backupwallet(tmpdir + "/node2/wallet.bak") self.nodes[2].dumpwallet(tmpdir + "/node2/wallet.dump") - logging.info("More transactions") + self.log.info("More transactions") for i in range(5): self.do_one_round() @@ -150,7 +146,7 @@ class WalletBackupTest(BitcoinTestFramework): ## # Test restoring spender wallets from backups ## - logging.info("Restoring using wallet.dat") + self.log.info("Restoring using wallet.dat") self.stop_three() self.erase_three() @@ -163,7 +159,7 @@ class WalletBackupTest(BitcoinTestFramework): shutil.copyfile(tmpdir + "/node1/wallet.bak", tmpdir + "/node1/regtest/wallet.dat") shutil.copyfile(tmpdir + "/node2/wallet.bak", tmpdir + "/node2/regtest/wallet.dat") - logging.info("Re-starting nodes") + self.log.info("Re-starting nodes") self.start_three() sync_blocks(self.nodes) @@ -171,7 +167,7 @@ class WalletBackupTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), balance1) assert_equal(self.nodes[2].getbalance(), balance2) - logging.info("Restoring using dumped wallet") + self.log.info("Restoring using dumped wallet") self.stop_three() self.erase_three() diff --git a/qa/rpc-tests/zapwallettxes.py b/test/functional/zapwallettxes.py index 9597d05f3a..a8600e82f6 100755 --- a/qa/rpc-tests/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -23,16 +23,12 @@ class ZapWalletTXesTest (BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes,0,1) - connect_nodes_bi(self.nodes,1,2) + def setup_network(self): + super().setup_network() connect_nodes_bi(self.nodes,0,2) - self.is_network_split=False - self.sync_all() def run_test (self): - print("Mining blocks...") + self.log.info("Mining blocks...") self.nodes[0].generate(1) self.sync_all() self.nodes[1].generate(101) @@ -62,18 +58,16 @@ class ZapWalletTXesTest (BitcoinTestFramework): assert_equal(tx3['txid'], txid3) #tx3 must be available (unconfirmed) #restart bitcoind - self.nodes[0].stop() - bitcoind_processes[0].wait() - self.nodes[0] = start_node(0,self.options.tmpdir) + self.stop_node(0) + self.nodes[0] = self.start_node(0,self.options.tmpdir) tx3 = self.nodes[0].gettransaction(txid3) assert_equal(tx3['txid'], txid3) #tx must be available (unconfirmed) - self.nodes[0].stop() - bitcoind_processes[0].wait() + self.stop_node(0) #restart bitcoind with zapwallettxes - self.nodes[0] = start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) + self.nodes[0] = self.start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) assert_raises(JSONRPCException, self.nodes[0].gettransaction, [txid3]) #there must be a expection because the unconfirmed wallettx0 must be gone by now diff --git a/qa/rpc-tests/zmq_test.py b/test/functional/zmq_test.py index 1e2f06bd54..ce39cfefdc 100755 --- a/qa/rpc-tests/zmq_test.py +++ b/test/functional/zmq_test.py @@ -3,11 +3,12 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ API.""" +import configparser +import os +import struct -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import BitcoinTestFramework, SkipTest from test_framework.util import * -import zmq -import struct class ZMQTest (BitcoinTestFramework): @@ -18,12 +19,27 @@ class ZMQTest (BitcoinTestFramework): port = 28332 def setup_nodes(self): + # Try to import python3-zmq. Skip this test if the import fails. + try: + import zmq + except ImportError: + raise SkipTest("python3-zmq module not available.") + + # Check that bitcoin has been built with ZMQ enabled + config = configparser.ConfigParser() + if not self.options.configfile: + self.options.configfile = os.path.dirname(__file__) + "/config.ini" + config.read_file(open(self.options.configfile)) + + if not config["components"].getboolean("ENABLE_ZMQ"): + raise SkipTest("bitcoind has not been built with zmq enabled.") + self.zmqContext = zmq.Context() self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock") self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashtx") self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port) - return start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[ + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[ ['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)], [], [], @@ -36,12 +52,11 @@ class ZMQTest (BitcoinTestFramework): genhashes = self.nodes[0].generate(1) self.sync_all() - print("listen...") + self.log.info("listen...") msg = self.zmqSubSocket.recv_multipart() topic = msg[0] assert_equal(topic, b"hashtx") body = msg[1] - nseq = msg[2] msgSequence = struct.unpack('<I', msg[-1])[-1] assert_equal(msgSequence, 0) #must be sequence 0 on hashtx diff --git a/src/test/bctest.py b/test/util/bctest.py index adc5d0e418..b17cf77ae3 100644 --- a/src/test/bctest.py +++ b/test/util/bctest.py @@ -10,6 +10,7 @@ import sys import binascii import difflib import logging +import pprint def parse_output(a, fmt): """Parse the output according to specified format. @@ -22,14 +23,14 @@ def parse_output(a, fmt): else: raise NotImplementedError("Don't know how to compare %s" % fmt) -def bctest(testDir, testObj, exeext): +def bctest(testDir, testObj, buildenv): """Runs a single test, comparing output and RC to expected output and RC. Raises an error if input can't be read, executable fails, or output/RC are not as expected. Error is caught by bctester() and reported. """ # Get the exec names and arguments - execprog = testObj['exec'] + exeext + execprog = buildenv.BUILDDIR + "/src/" + testObj['exec'] + buildenv.exeext execargs = testObj['args'] execrun = [execprog] + execargs @@ -65,6 +66,7 @@ def bctest(testDir, testObj, exeext): raise if outputData: + data_mismatch, formatting_mismatch = False, False # Parse command output and expected output try: a_parsed = parse_output(outs[0], outputType) @@ -79,7 +81,7 @@ def bctest(testDir, testObj, exeext): # Compare data if a_parsed != b_parsed: logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") - raise Exception + data_mismatch = True # Compare formatting if outs[0] != outputData: error_message = "Output formatting mismatch for " + outputFn + ":\n" @@ -88,7 +90,9 @@ def bctest(testDir, testObj, exeext): fromfile=outputFn, tofile="returned")) logging.error(error_message) - raise Exception + formatting_mismatch = True + + assert not data_mismatch and not formatting_mismatch # Compare the return code to the expected return code wantRC = 0 @@ -98,6 +102,18 @@ def bctest(testDir, testObj, exeext): logging.error("Return code mismatch for " + outputFn) raise Exception + if "error_txt" in testObj: + want_error = testObj["error_txt"] + # Compare error text + # TODO: ideally, we'd compare the strings exactly and also assert + # That stderr is empty if no errors are expected. However, bitcoin-tx + # emits DISPLAY errors when running as a windows application on + # linux through wine. Just assert that the expected error text appears + # somewhere in stderr. + if want_error not in outs[1]: + logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) + raise Exception + def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" input_filename = testDir + "/" + input_basename @@ -108,14 +124,16 @@ def bctester(testDir, input_basename, buildenv): for testObj in input_data: try: - bctest(testDir, testObj, buildenv.exeext) + bctest(testDir, testObj, buildenv) logging.info("PASSED: " + testObj["description"]) except: logging.info("FAILED: " + testObj["description"]) failed_testcases.append(testObj["description"]) if failed_testcases: - logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]") + error_message = "FAILED_TESTCASES:\n" + error_message += pprint.pformat(failed_testcases, width=400) + logging.error(error_message) sys.exit(1) else: sys.exit(0) diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py new file mode 100755 index 0000000000..e09a25159d --- /dev/null +++ b/test/util/bitcoin-util-test.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Copyright 2014 BitPay Inc. +# Copyright 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +from __future__ import division,print_function,unicode_literals +import os +import sys +import argparse +import logging + +help_text="""Test framework for bitcoin utils. + +Runs automatically during `make check`. + +Can also be run manually.""" + +if __name__ == '__main__': + sys.path.append(os.path.dirname(os.path.abspath(__file__))) + import buildenv + import bctest + + parser = argparse.ArgumentParser(description=help_text) + parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args() + verbose = args.verbose + + if verbose: + level = logging.DEBUG + else: + level = logging.ERROR + formatter = '%(asctime)s - %(levelname)s - %(message)s' + # Add the format/level to the logger + logging.basicConfig(format = formatter, level=level) + + bctest.bctester(buildenv.SRCDIR + "/test/util/data", "bitcoin-util-test.json", buildenv) diff --git a/test/util/buildenv.py.in b/test/util/buildenv.py.in new file mode 100644 index 0000000000..33030b0348 --- /dev/null +++ b/test/util/buildenv.py.in @@ -0,0 +1,4 @@ +#!/usr/bin/env python +exeext="@EXEEXT@" +SRCDIR="@abs_top_srcdir@" +BUILDDIR="@abs_top_builddir@" diff --git a/src/test/data/bitcoin-util-test.json b/test/util/data/bitcoin-util-test.json index a80ab51901..b61a4f7f8f 100644 --- a/src/test/data/bitcoin-util-test.json +++ b/test/util/data/bitcoin-util-test.json @@ -42,6 +42,7 @@ "args": ["-", "delin=31"], "input": "tx394b54bb.hex", "return_code": 1, + "error_txt": "error: Invalid TX input index '31'", "description": "Attempts to delete an input with a bad index from a transaction. Expected to fail." }, { "exec": "./bitcoin-tx", @@ -60,6 +61,7 @@ "args": ["-", "delout=2"], "input": "tx394b54bb.hex", "return_code": 1, + "error_txt": "error: Invalid TX output index '2'", "description": "Attempts to delete an output with a bad index from a transaction. Expected to fail." }, { "exec": "./bitcoin-tx", @@ -77,6 +79,38 @@ { "exec": "./bitcoin-tx", "args": ["-create", + "outaddr=1"], + "return_code": 1, + "error_txt": "error: TX output missing or too many separators", + "description": "Malformed outaddr argument (no address specified). Expected to fail." + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", + "outaddr=1:13tuJJDR2RgArmgfv6JScSdreahzgc4T6o:garbage"], + "return_code": 1, + "error_txt": "error: TX output missing or too many separators", + "description": "Malformed outaddr argument (too many separators). Expected to fail." + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", + "outpubkey=0"], + "return_code": 1, + "error_txt": "error: TX output missing or too many separators", + "description": "Malformed outpubkey argument (no pubkey specified). Expected to fail." + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", + "outpubkey=0:02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397:W:non53nse"], + "return_code": 1, + "error_txt": "error: TX output missing or too many separators", + "description": "Malformed outpubkey argument (too many separators). Expected to fail." + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", "in=bf829c6bcf84579331337659d31f89dfd138f7f7785802d5501c92333145ca7c:18", "in=22a6f904655d53ae2ff70e701a0bbd90aa3975c0f40bfc6cc996a9049e31cdfc:1", @@ -233,6 +267,7 @@ "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", "outdata=4:badhexdata"], "return_code": 1, + "error_txt": "error: invalid TX output data", "description": "Attempts to create a new transaction with one input and an output with malformed hex data. Expected to fail" }, { "exec": "./bitcoin-tx", @@ -241,6 +276,7 @@ "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", "outdata=badhexdata"], "return_code": 1, + "error_txt": "error: invalid TX output data", "description": "Attempts to create a new transaction with one input and an output with no value and malformed hex data. Expected to fail" }, { "exec": "./bitcoin-tx", diff --git a/src/test/data/blanktxv1.hex b/test/util/data/blanktxv1.hex index 36b6f00fb6..36b6f00fb6 100644 --- a/src/test/data/blanktxv1.hex +++ b/test/util/data/blanktxv1.hex diff --git a/src/test/data/blanktxv1.json b/test/util/data/blanktxv1.json index 51c25a5a98..9fe2de649b 100644 --- a/src/test/data/blanktxv1.json +++ b/test/util/data/blanktxv1.json @@ -2,6 +2,8 @@ "txid": "d21633ba23f70118185227be58a63527675641ad37967e2aa461559f577aec43", "hash": "d21633ba23f70118185227be58a63527675641ad37967e2aa461559f577aec43", "version": 1, + "size": 10, + "vsize": 10, "locktime": 0, "vin": [ ], diff --git a/src/test/data/blanktxv2.hex b/test/util/data/blanktxv2.hex index 22d830eda1..22d830eda1 100644 --- a/src/test/data/blanktxv2.hex +++ b/test/util/data/blanktxv2.hex diff --git a/src/test/data/blanktxv2.json b/test/util/data/blanktxv2.json index 266919f445..e97626e421 100644 --- a/src/test/data/blanktxv2.json +++ b/test/util/data/blanktxv2.json @@ -2,6 +2,8 @@ "txid": "4ebd325a4b394cff8c57e8317ccf5a8d0e2bdf1b8526f8aad6c8e43d8240621a", "hash": "4ebd325a4b394cff8c57e8317ccf5a8d0e2bdf1b8526f8aad6c8e43d8240621a", "version": 2, + "size": 10, + "vsize": 10, "locktime": 0, "vin": [ ], diff --git a/src/test/data/tt-delin1-out.hex b/test/util/data/tt-delin1-out.hex index 42ad840f43..42ad840f43 100644 --- a/src/test/data/tt-delin1-out.hex +++ b/test/util/data/tt-delin1-out.hex diff --git a/src/test/data/tt-delin1-out.json b/test/util/data/tt-delin1-out.json index 712a2c27f8..f6dfbb51cc 100644 --- a/src/test/data/tt-delin1-out.json +++ b/test/util/data/tt-delin1-out.json @@ -2,6 +2,8 @@ "txid": "81b2035be1da1abe745c6141174a73d151009ec17b3d5ebffa2e177408c50dfd", "hash": "81b2035be1da1abe745c6141174a73d151009ec17b3d5ebffa2e177408c50dfd", "version": 1, + "size": 3040, + "vsize": 3040, "locktime": 0, "vin": [ { diff --git a/src/test/data/tt-delout1-out.hex b/test/util/data/tt-delout1-out.hex index cc60c3fac6..cc60c3fac6 100644 --- a/src/test/data/tt-delout1-out.hex +++ b/test/util/data/tt-delout1-out.hex diff --git a/src/test/data/tt-delout1-out.json b/test/util/data/tt-delout1-out.json index afc4e95762..6769ed79ff 100644 --- a/src/test/data/tt-delout1-out.json +++ b/test/util/data/tt-delout1-out.json @@ -2,6 +2,8 @@ "txid": "c46ccd75b5050e942b2e86a3648f843f525fe6fc000bf0534ba5973063354493", "hash": "c46ccd75b5050e942b2e86a3648f843f525fe6fc000bf0534ba5973063354493", "version": 1, + "size": 3155, + "vsize": 3155, "locktime": 0, "vin": [ { diff --git a/src/test/data/tt-locktime317000-out.hex b/test/util/data/tt-locktime317000-out.hex index 287f420a40..287f420a40 100644 --- a/src/test/data/tt-locktime317000-out.hex +++ b/test/util/data/tt-locktime317000-out.hex diff --git a/src/test/data/tt-locktime317000-out.json b/test/util/data/tt-locktime317000-out.json index 2b9075f8ac..82b64df075 100644 --- a/src/test/data/tt-locktime317000-out.json +++ b/test/util/data/tt-locktime317000-out.json @@ -2,6 +2,8 @@ "txid": "aded538f642c17e15f4d3306b8be7e1a4d1ae0c4616d641ab51ea09ba65e5cb5", "hash": "aded538f642c17e15f4d3306b8be7e1a4d1ae0c4616d641ab51ea09ba65e5cb5", "version": 1, + "size": 3189, + "vsize": 3189, "locktime": 317000, "vin": [ { diff --git a/src/test/data/tx394b54bb.hex b/test/util/data/tx394b54bb.hex index 33f26cb4d6..33f26cb4d6 100644 --- a/src/test/data/tx394b54bb.hex +++ b/test/util/data/tx394b54bb.hex diff --git a/src/test/data/txcreate1.hex b/test/util/data/txcreate1.hex index 9ec6ee3531..9ec6ee3531 100644 --- a/src/test/data/txcreate1.hex +++ b/test/util/data/txcreate1.hex diff --git a/src/test/data/txcreate1.json b/test/util/data/txcreate1.json index f83e036f33..36741044c9 100644 --- a/src/test/data/txcreate1.json +++ b/test/util/data/txcreate1.json @@ -2,6 +2,8 @@ "txid": "fe7d174f42dce0cffa7a527e9bc8368956057619ec817648f6138b98f2533e8f", "hash": "fe7d174f42dce0cffa7a527e9bc8368956057619ec817648f6138b98f2533e8f", "version": 2, + "size": 201, + "vsize": 201, "locktime": 0, "vin": [ { diff --git a/src/test/data/txcreate2.hex b/test/util/data/txcreate2.hex index 38bb7b1046..38bb7b1046 100644 --- a/src/test/data/txcreate2.hex +++ b/test/util/data/txcreate2.hex diff --git a/src/test/data/txcreate2.json b/test/util/data/txcreate2.json index fb5e177db7..23fe7ace67 100644 --- a/src/test/data/txcreate2.json +++ b/test/util/data/txcreate2.json @@ -2,6 +2,8 @@ "txid": "0481afb29931341d0d7861d8a2f6f26456fa042abf54a23e96440ed7946e0715", "hash": "0481afb29931341d0d7861d8a2f6f26456fa042abf54a23e96440ed7946e0715", "version": 2, + "size": 19, + "vsize": 19, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatedata1.hex b/test/util/data/txcreatedata1.hex index cefd1a05a6..cefd1a05a6 100644 --- a/src/test/data/txcreatedata1.hex +++ b/test/util/data/txcreatedata1.hex diff --git a/src/test/data/txcreatedata1.json b/test/util/data/txcreatedata1.json index 760518d30a..e65a1859eb 100644 --- a/src/test/data/txcreatedata1.json +++ b/test/util/data/txcreatedata1.json @@ -2,6 +2,8 @@ "txid": "07894b4d12fe7853dd911402db1620920d261b9627c447f931417d330c25f06e", "hash": "07894b4d12fe7853dd911402db1620920d261b9627c447f931417d330c25f06e", "version": 1, + "size": 176, + "vsize": 176, "locktime": 0, "vin": [ { diff --git a/src/test/data/txcreatedata2.hex b/test/util/data/txcreatedata2.hex index d69cf58ba1..d69cf58ba1 100644 --- a/src/test/data/txcreatedata2.hex +++ b/test/util/data/txcreatedata2.hex diff --git a/src/test/data/txcreatedata2.json b/test/util/data/txcreatedata2.json index 3c6da40f90..8f1544e1c0 100644 --- a/src/test/data/txcreatedata2.json +++ b/test/util/data/txcreatedata2.json @@ -2,6 +2,8 @@ "txid": "c14b007fa3a6c1e7765919c1d14c1cfc2b8642c3a5d3be4b1fa8c4ccfec98bb0", "hash": "c14b007fa3a6c1e7765919c1d14c1cfc2b8642c3a5d3be4b1fa8c4ccfec98bb0", "version": 2, + "size": 176, + "vsize": 176, "locktime": 0, "vin": [ { diff --git a/src/test/data/txcreatedata_seq0.hex b/test/util/data/txcreatedata_seq0.hex index 54b89d2381..54b89d2381 100644 --- a/src/test/data/txcreatedata_seq0.hex +++ b/test/util/data/txcreatedata_seq0.hex diff --git a/src/test/data/txcreatedata_seq0.json b/test/util/data/txcreatedata_seq0.json index d272a4c447..e52401f418 100644 --- a/src/test/data/txcreatedata_seq0.json +++ b/test/util/data/txcreatedata_seq0.json @@ -2,6 +2,8 @@ "txid": "8df6ed527472542dd5e137c242a7c5a9f337ac34f7b257ae4af886aeaebb51b0", "hash": "8df6ed527472542dd5e137c242a7c5a9f337ac34f7b257ae4af886aeaebb51b0", "version": 2, + "size": 85, + "vsize": 85, "locktime": 0, "vin": [ { diff --git a/src/test/data/txcreatedata_seq1.hex b/test/util/data/txcreatedata_seq1.hex index 4cedcd975c..4cedcd975c 100644 --- a/src/test/data/txcreatedata_seq1.hex +++ b/test/util/data/txcreatedata_seq1.hex diff --git a/src/test/data/txcreatedata_seq1.json b/test/util/data/txcreatedata_seq1.json index d323255418..093ff4a56b 100644 --- a/src/test/data/txcreatedata_seq1.json +++ b/test/util/data/txcreatedata_seq1.json @@ -2,6 +2,8 @@ "txid": "c4dea671b0d7b48f8ab10bc46650e8329d3c5766931f548f513847a19f5ba75b", "hash": "c4dea671b0d7b48f8ab10bc46650e8329d3c5766931f548f513847a19f5ba75b", "version": 1, + "size": 126, + "vsize": 126, "locktime": 0, "vin": [ { diff --git a/src/test/data/txcreatemultisig1.hex b/test/util/data/txcreatemultisig1.hex index 9c00004d38..9c00004d38 100644 --- a/src/test/data/txcreatemultisig1.hex +++ b/test/util/data/txcreatemultisig1.hex diff --git a/src/test/data/txcreatemultisig1.json b/test/util/data/txcreatemultisig1.json index f6ce43c202..0cc530836a 100644 --- a/src/test/data/txcreatemultisig1.json +++ b/test/util/data/txcreatemultisig1.json @@ -2,6 +2,8 @@ "txid": "0d1d4edfc217d9db3ab6a9298f26a52eae3c52f55a6cb8ccbc14f7c727572894", "hash": "0d1d4edfc217d9db3ab6a9298f26a52eae3c52f55a6cb8ccbc14f7c727572894", "version": 1, + "size": 124, + "vsize": 124, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatemultisig2.hex b/test/util/data/txcreatemultisig2.hex index 07835c54d3..07835c54d3 100644 --- a/src/test/data/txcreatemultisig2.hex +++ b/test/util/data/txcreatemultisig2.hex diff --git a/src/test/data/txcreatemultisig2.json b/test/util/data/txcreatemultisig2.json index e09d22060f..8ad2ffdc65 100644 --- a/src/test/data/txcreatemultisig2.json +++ b/test/util/data/txcreatemultisig2.json @@ -2,6 +2,8 @@ "txid": "0d861f278a3b7bce7cb5a88d71e6e6a903336f95ad5a2c29b295b63835b6eee3", "hash": "0d861f278a3b7bce7cb5a88d71e6e6a903336f95ad5a2c29b295b63835b6eee3", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatemultisig3.hex b/test/util/data/txcreatemultisig3.hex index 8d34f28f87..8d34f28f87 100644 --- a/src/test/data/txcreatemultisig3.hex +++ b/test/util/data/txcreatemultisig3.hex diff --git a/src/test/data/txcreatemultisig3.json b/test/util/data/txcreatemultisig3.json index 88e32bd310..086bf44b8a 100644 --- a/src/test/data/txcreatemultisig3.json +++ b/test/util/data/txcreatemultisig3.json @@ -2,6 +2,8 @@ "txid": "ccc552220b46a3b5140048b03395987ce4f0fa1ddf8c635bba1fa44e0f8c1d7f", "hash": "ccc552220b46a3b5140048b03395987ce4f0fa1ddf8c635bba1fa44e0f8c1d7f", "version": 1, + "size": 53, + "vsize": 53, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatemultisig4.hex b/test/util/data/txcreatemultisig4.hex index 7da54366c7..7da54366c7 100644 --- a/src/test/data/txcreatemultisig4.hex +++ b/test/util/data/txcreatemultisig4.hex diff --git a/src/test/data/txcreatemultisig4.json b/test/util/data/txcreatemultisig4.json index fc69c7269c..d23ccc045e 100644 --- a/src/test/data/txcreatemultisig4.json +++ b/test/util/data/txcreatemultisig4.json @@ -2,6 +2,8 @@ "txid": "5e8b1cc73234e208d4b7ca9075f136b908c34101be7a048df4ba9ac758b61567", "hash": "5e8b1cc73234e208d4b7ca9075f136b908c34101be7a048df4ba9ac758b61567", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreateoutpubkey1.hex b/test/util/data/txcreateoutpubkey1.hex index 4a08244b2f..4a08244b2f 100644 --- a/src/test/data/txcreateoutpubkey1.hex +++ b/test/util/data/txcreateoutpubkey1.hex diff --git a/src/test/data/txcreateoutpubkey1.json b/test/util/data/txcreateoutpubkey1.json index 6019fa2dcd..f10aaecf7a 100644 --- a/src/test/data/txcreateoutpubkey1.json +++ b/test/util/data/txcreateoutpubkey1.json @@ -2,6 +2,8 @@ "txid": "f42b38ac12e3fafc96ba1a9ba70cbfe326744aef75df5fb9db5d6e2855ca415f", "hash": "f42b38ac12e3fafc96ba1a9ba70cbfe326744aef75df5fb9db5d6e2855ca415f", "version": 1, + "size": 54, + "vsize": 54, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreateoutpubkey2.hex b/test/util/data/txcreateoutpubkey2.hex index 8283c722ab..8283c722ab 100644 --- a/src/test/data/txcreateoutpubkey2.hex +++ b/test/util/data/txcreateoutpubkey2.hex diff --git a/src/test/data/txcreateoutpubkey2.json b/test/util/data/txcreateoutpubkey2.json index 6fc3d57527..5a473b76c3 100644 --- a/src/test/data/txcreateoutpubkey2.json +++ b/test/util/data/txcreateoutpubkey2.json @@ -2,6 +2,8 @@ "txid": "70f2a088cde460e677415fa1fb71895e90c231e6ed38ed203a35b6f848e9cc73", "hash": "70f2a088cde460e677415fa1fb71895e90c231e6ed38ed203a35b6f848e9cc73", "version": 1, + "size": 41, + "vsize": 41, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreateoutpubkey3.hex b/test/util/data/txcreateoutpubkey3.hex index 84adff4d89..84adff4d89 100644 --- a/src/test/data/txcreateoutpubkey3.hex +++ b/test/util/data/txcreateoutpubkey3.hex diff --git a/src/test/data/txcreateoutpubkey3.json b/test/util/data/txcreateoutpubkey3.json index a1a25fc834..b8389b8f7e 100644 --- a/src/test/data/txcreateoutpubkey3.json +++ b/test/util/data/txcreateoutpubkey3.json @@ -2,6 +2,8 @@ "txid": "bfc7e898ee9f6a9652d7b8cca147e2da134502e2ada0f279ed634fc8cf833f8c", "hash": "bfc7e898ee9f6a9652d7b8cca147e2da134502e2ada0f279ed634fc8cf833f8c", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatescript1.hex b/test/util/data/txcreatescript1.hex index 0adce270fb..0adce270fb 100644 --- a/src/test/data/txcreatescript1.hex +++ b/test/util/data/txcreatescript1.hex diff --git a/src/test/data/txcreatescript1.json b/test/util/data/txcreatescript1.json index 8ffecba411..823168e9fb 100644 --- a/src/test/data/txcreatescript1.json +++ b/test/util/data/txcreatescript1.json @@ -2,6 +2,8 @@ "txid": "f0851b68202f736b792649cfc960259c2374badcb644ab20cac726b5f72f61c9", "hash": "f0851b68202f736b792649cfc960259c2374badcb644ab20cac726b5f72f61c9", "version": 1, + "size": 20, + "vsize": 20, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatescript2.hex b/test/util/data/txcreatescript2.hex index 5afe8786e3..5afe8786e3 100644 --- a/src/test/data/txcreatescript2.hex +++ b/test/util/data/txcreatescript2.hex diff --git a/src/test/data/txcreatescript2.json b/test/util/data/txcreatescript2.json index 41eb69f1af..d4c7e10c78 100644 --- a/src/test/data/txcreatescript2.json +++ b/test/util/data/txcreatescript2.json @@ -2,6 +2,8 @@ "txid": "6e07a7cc075e0703f32ee8c4e5373fe654bfbc315148fda364e1be286ff290d0", "hash": "6e07a7cc075e0703f32ee8c4e5373fe654bfbc315148fda364e1be286ff290d0", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatescript3.hex b/test/util/data/txcreatescript3.hex index 8a2b973bf0..8a2b973bf0 100644 --- a/src/test/data/txcreatescript3.hex +++ b/test/util/data/txcreatescript3.hex diff --git a/src/test/data/txcreatescript3.json b/test/util/data/txcreatescript3.json index 90e7e27f9f..001e69511f 100644 --- a/src/test/data/txcreatescript3.json +++ b/test/util/data/txcreatescript3.json @@ -2,6 +2,8 @@ "txid": "8a234037b088e987c877030efc83374a07441c321bf9dc6dd2f206bc26507df8", "hash": "8a234037b088e987c877030efc83374a07441c321bf9dc6dd2f206bc26507df8", "version": 1, + "size": 53, + "vsize": 53, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatescript4.hex b/test/util/data/txcreatescript4.hex index b4cfe58f42..b4cfe58f42 100644 --- a/src/test/data/txcreatescript4.hex +++ b/test/util/data/txcreatescript4.hex diff --git a/src/test/data/txcreatescript4.json b/test/util/data/txcreatescript4.json index 11783751a4..20094bcd44 100644 --- a/src/test/data/txcreatescript4.json +++ b/test/util/data/txcreatescript4.json @@ -2,6 +2,8 @@ "txid": "24225cf5e9391100d6b218134b9f03383ca4c880a1f634ac12990cf28b66adbc", "hash": "24225cf5e9391100d6b218134b9f03383ca4c880a1f634ac12990cf28b66adbc", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/src/test/data/txcreatesignv1.hex b/test/util/data/txcreatesignv1.hex index a46fcc88cb..a46fcc88cb 100644 --- a/src/test/data/txcreatesignv1.hex +++ b/test/util/data/txcreatesignv1.hex diff --git a/src/test/data/txcreatesignv1.json b/test/util/data/txcreatesignv1.json index ff39e71b40..519d3ab066 100644 --- a/src/test/data/txcreatesignv1.json +++ b/test/util/data/txcreatesignv1.json @@ -2,6 +2,8 @@ "txid": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af", "hash": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af", "version": 1, + "size": 224, + "vsize": 224, "locktime": 0, "vin": [ { diff --git a/src/test/data/txcreatesignv2.hex b/test/util/data/txcreatesignv2.hex index ee425cd98c..ee425cd98c 100644 --- a/src/test/data/txcreatesignv2.hex +++ b/test/util/data/txcreatesignv2.hex |