diff options
Diffstat (limited to 'src')
218 files changed, 8040 insertions, 5168 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index 69ae02f582..c7124c3ef9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -61,6 +61,7 @@ EXTRA_LIBRARIES += \ lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) bin_PROGRAMS = +noinst_PROGRAMS = TESTS = BENCHMARKS = @@ -104,11 +105,11 @@ BITCOIN_CORE_H = \ keystore.h \ dbwrapper.h \ limitedmap.h \ - main.h \ memusage.h \ merkleblock.h \ miner.h \ net.h \ + net_processing.h \ netaddress.h \ netbase.h \ netmessagemaker.h \ @@ -145,6 +146,7 @@ BITCOIN_CORE_H = \ util.h \ utilmoneystr.h \ utiltime.h \ + validation.h \ validationinterface.h \ versionbits.h \ wallet/coincontrol.h \ @@ -153,6 +155,7 @@ BITCOIN_CORE_H = \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ + warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ zmq/zmqnotificationinterface.h \ @@ -179,10 +182,10 @@ libbitcoin_server_a_SOURCES = \ httpserver.cpp \ init.cpp \ dbwrapper.cpp \ - main.cpp \ merkleblock.cpp \ miner.cpp \ net.cpp \ + net_processing.cpp \ noui.cpp \ policy/fees.cpp \ policy/policy.cpp \ @@ -201,6 +204,7 @@ libbitcoin_server_a_SOURCES = \ txdb.cpp \ txmempool.cpp \ ui_interface.cpp \ + validation.cpp \ validationinterface.cpp \ versionbits.cpp \ $(BITCOIN_CORE_H) @@ -303,6 +307,7 @@ libbitcoin_common_a_SOURCES = \ scheduler.cpp \ script/sign.cpp \ script/standard.cpp \ + warnings.cpp \ $(BITCOIN_CORE_H) # util: shared between all executables. diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index cb310d5a1b..039f8ac547 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -6,6 +6,7 @@ bin_PROGRAMS += qt/test/test_bitcoin-qt TESTS += qt/test/test_bitcoin-qt TEST_QT_MOC_CPP = \ + qt/test/moc_compattests.cpp \ qt/test/moc_rpcnestedtests.cpp \ qt/test/moc_uritests.cpp @@ -14,6 +15,7 @@ TEST_QT_MOC_CPP += qt/test/moc_paymentservertests.cpp endif TEST_QT_H = \ + qt/test/compattests.h \ qt/test/rpcnestedtests.h \ qt/test/uritests.h \ qt/test/paymentrequestdata.h \ @@ -23,6 +25,7 @@ qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_ $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(PROTOBUF_CFLAGS) qt_test_test_bitcoin_qt_SOURCES = \ + qt/test/compattests.cpp \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ qt/test/uritests.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index a14adc7876..5969de087e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -4,6 +4,7 @@ TESTS += test/test_bitcoin bin_PROGRAMS += test/test_bitcoin +noinst_PROGRAMS += test/test_bitcoin_fuzzy TEST_SRCDIR = test TEST_BINARY=test/test_bitcoin$(EXEEXT) @@ -38,6 +39,7 @@ RAW_TEST_FILES = GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) +# test_bitcoin binary # BITCOIN_TESTS =\ test/arith_uint256_tests.cpp \ test/scriptnum10.h \ @@ -50,9 +52,11 @@ BITCOIN_TESTS =\ test/bip32_tests.cpp \ test/blockencodings_tests.cpp \ test/bloom_tests.cpp \ + test/bswap_tests.cpp \ test/coins_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ + test/cuckoocache_tests.cpp \ test/DoS_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ @@ -119,6 +123,25 @@ test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -s if ENABLE_ZMQ test_test_bitcoin_LDADD += $(ZMQ_LIBS) endif +# + +# test_bitcoin_fuzzy binary # +test_test_bitcoin_fuzzy_SOURCES = test/test_bitcoin_fuzzy.cpp +test_test_bitcoin_fuzzy_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_test_bitcoin_fuzzy_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_test_bitcoin_fuzzy_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + +test_test_bitcoin_fuzzy_LDADD = \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBSECP256K1) + +test_test_bitcoin_fuzzy_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +# nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/bench/Examples.cpp b/src/bench/Examples.cpp index b6b020a971..9f35a1ea04 100644 --- a/src/bench/Examples.cpp +++ b/src/bench/Examples.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "bench.h" -#include "main.h" +#include "validation.h" #include "utiltime.h" // Sanity test: this should loop ten times, and diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index a791b5b7fa..3319c179bf 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -4,7 +4,7 @@ #include "bench.h" -#include "main.h" +#include "validation.h" #include "base58.h" #include <vector> diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index db1402216d..bd768180c6 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -5,7 +5,7 @@ #include "bench.h" #include "key.h" -#include "main.h" +#include "validation.h" #include "util.h" int diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 4a564d3fc8..230e4ca773 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -5,7 +5,8 @@ #include "bench.h" #include "chainparams.h" -#include "main.h" +#include "validation.h" +#include "streams.h" #include "consensus/validation.h" namespace block_bench { diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index ce1ce511e3..cacead4a59 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -19,7 +19,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, vector<COutput tx.nLockTime = nextLockTime++; // so all transactions get different hashes tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - CWalletTx* wtx = new CWalletTx(&wallet, tx); + CWalletTx* wtx = new CWalletTx(&wallet, MakeTransactionRef(std::move(tx))); int nAge = 6 * 24; COutput output(wtx, nInput, nAge, true, true); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 168006154f..737d3572ae 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -22,7 +22,7 @@ static void RIPEMD160(benchmark::State& state) uint8_t hash[CRIPEMD160::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); while (state.KeepRunning()) - CRIPEMD160().Write(begin_ptr(in), in.size()).Finalize(hash); + CRIPEMD160().Write(in.data(), in.size()).Finalize(hash); } static void SHA1(benchmark::State& state) @@ -30,7 +30,7 @@ static void SHA1(benchmark::State& state) uint8_t hash[CSHA1::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); while (state.KeepRunning()) - CSHA1().Write(begin_ptr(in), in.size()).Finalize(hash); + CSHA1().Write(in.data(), in.size()).Finalize(hash); } static void SHA256(benchmark::State& state) @@ -38,7 +38,7 @@ static void SHA256(benchmark::State& state) uint8_t hash[CSHA256::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); while (state.KeepRunning()) - CSHA256().Write(begin_ptr(in), in.size()).Finalize(hash); + CSHA256().Write(in.data(), in.size()).Finalize(hash); } static void SHA256_32b(benchmark::State& state) @@ -46,7 +46,7 @@ static void SHA256_32b(benchmark::State& state) std::vector<uint8_t> in(32,0); while (state.KeepRunning()) { for (int i = 0; i < 1000000; i++) { - CSHA256().Write(begin_ptr(in), in.size()).Finalize(&in[0]); + CSHA256().Write(in.data(), in.size()).Finalize(&in[0]); } } } @@ -56,7 +56,7 @@ static void SHA512(benchmark::State& state) uint8_t hash[CSHA512::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); while (state.KeepRunning()) - CSHA512().Write(begin_ptr(in), in.size()).Finalize(hash); + CSHA512().Write(in.data(), in.size()).Finalize(hash); } static void SipHash_32b(benchmark::State& state) diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index dc3940cdbd..91b8c44472 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -91,7 +91,7 @@ static void VerifyScriptBench(benchmark::State& state) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << txSpend; int csuccess = bitcoinconsensus_verify_script_with_amount( - begin_ptr(txCredit.vout[0].scriptPubKey), + txCredit.vout[0].scriptPubKey.data(), txCredit.vout[0].scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, flags, nullptr); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index b272640819..596cc8eada 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -247,7 +247,7 @@ UniValue CallRPC(const std::string& strMethod, const UniValue& params) event_base_free(base); if (response.status == 0) - throw CConnectionFailed(strprintf("couldn't connect to server\n(make sure server is running and you are connecting to the correct RPC port: %d %s)", response.error, http_errorstring(response.error))); + throw CConnectionFailed(strprintf("couldn't connect to server: %s (code %d)\n(make sure server is running and you are connecting to the correct RPC port)", http_errorstring(response.error), response.error)); else if (response.status == HTTP_UNAUTHORIZED) throw std::runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 346d8e180d..1ed1449f03 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -623,7 +623,7 @@ static int CommandLineRawTx(int argc, char* argv[]) argv++; } - CTransaction txDecodeTmp; + CMutableTransaction tx; int startArg; if (!fCreateBlank) { @@ -636,15 +636,13 @@ static int CommandLineRawTx(int argc, char* argv[]) if (strHexTx == "-") // "-" implies standard input strHexTx = readStdin(); - if (!DecodeHexTx(txDecodeTmp, strHexTx, true)) + if (!DecodeHexTx(tx, strHexTx, true)) throw std::runtime_error("invalid transaction encoding"); startArg = 2; } else startArg = 1; - CMutableTransaction tx(txDecodeTmp); - for (int i = startArg; i < argc; i++) { std::string arg = argv[i]; std::string key, value; diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 2efeda9cc6..914af0c670 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -10,7 +10,7 @@ #include "random.h" #include "streams.h" #include "txmempool.h" -#include "main.h" +#include "validation.h" #include "util.h" #include <unordered_map> diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index d22c188c16..a487fb1dc4 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -6,7 +6,7 @@ #include "chain.h" #include "chainparams.h" -#include "main.h" +#include "validation.h" #include "uint256.h" #include <stdint.h> diff --git a/src/coins.h b/src/coins.h index d295b3c940..dd6ef6cc3a 100644 --- a/src/coins.h +++ b/src/coins.h @@ -467,7 +467,6 @@ public: friend class CCoinsModifier; private: - CCoinsMap::iterator FetchCoins(const uint256 &txid); CCoinsMap::const_iterator FetchCoins(const uint256 &txid) const; /** diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index 07ca535728..56f4240e49 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -15,6 +15,23 @@ #include <byteswap.h> #endif +#if defined(__APPLE__) + +#if !defined(bswap_16) + +// Mac OS X / Darwin features; we include a check for bswap_16 because if it is already defined, protobuf has +// defined these macros for us already; if it isn't, we do it ourselves. In either case, we get the exact same +// result regardless which path was taken +#include <libkern/OSByteOrder.h> +#define bswap_16(x) OSSwapInt16(x) +#define bswap_32(x) OSSwapInt32(x) +#define bswap_64(x) OSSwapInt64(x) + +#endif // !defined(bswap_16) + +#else +// Non-Mac OS X / non-Darwin + #if HAVE_DECL_BSWAP_16 == 0 inline uint16_t bswap_16(uint16_t x) { @@ -44,4 +61,6 @@ inline uint64_t bswap_64(uint64_t x) } #endif // HAVE_DECL_BSWAP64 +#endif // defined(__APPLE__) + #endif // BITCOIN_COMPAT_BYTESWAP_H diff --git a/src/core_io.h b/src/core_io.h index 5aecbc4489..7642bc6d6e 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -11,13 +11,14 @@ class CBlock; class CScript; class CTransaction; +struct CMutableTransaction; class uint256; class UniValue; // core_read.cpp CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); -bool DecodeHexTx(CTransaction& tx, const std::string& strHexTx, bool fTryNoWitness = false); +bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx, bool fTryNoWitness = false); bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); uint256 ParseHashUV(const UniValue& v, const std::string& strName); uint256 ParseHashStr(const std::string&, const std::string& strName); @@ -25,7 +26,7 @@ std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strN // core_write.cpp std::string FormatScript(const CScript& script); -std::string EncodeHexTx(const CTransaction& tx); +std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry); diff --git a/src/core_read.cpp b/src/core_read.cpp index 7cfda6dd6d..b6d0285459 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -90,7 +90,7 @@ CScript ParseScript(const std::string& s) return result; } -bool DecodeHexTx(CTransaction& tx, const std::string& strHexTx, bool fTryNoWitness) +bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx, bool fTryNoWitness) { if (!IsHex(strHexTx)) return false; diff --git a/src/core_write.cpp b/src/core_write.cpp index ea01ddc10d..9f859ba9e1 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -116,9 +116,9 @@ string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode) return str; } -string EncodeHexTx(const CTransaction& tx) +string EncodeHexTx(const CTransaction& tx, const int serialFlags) { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serialFlags); ssTx << tx; return HexStr(ssTx.begin(), ssTx.end()); } diff --git a/src/crypto/ctaes/ctaes.c b/src/crypto/ctaes/ctaes.c index 2389fc0bb2..55962bf252 100644 --- a/src/crypto/ctaes/ctaes.c +++ b/src/crypto/ctaes/ctaes.c @@ -134,7 +134,7 @@ static void SubBytes(AES_state *s, int inv) { D = U7; } - /* Non-linear transformation (identical to the code in SubBytes) */ + /* Non-linear transformation (shared between the forward and backward case) */ M1 = T13 & T6; M6 = T3 & T16; M11 = T1 & T15; @@ -469,9 +469,9 @@ static void AES_encrypt(const AES_state* rounds, int nrounds, unsigned char* cip static void AES_decrypt(const AES_state* rounds, int nrounds, unsigned char* plain16, const unsigned char* cipher16) { /* Most AES decryption implementations use the alternate scheme - * (the Equivalent Inverse Cipher), which looks more like encryption, but - * needs different round constants. We can't reuse any code here anyway, so - * don't bother. */ + * (the Equivalent Inverse Cipher), which allows for more code reuse between + * the encryption and decryption code, but requires separate setup for both. + */ AES_state s = {{0}}; int round; diff --git a/src/cuckoocache.h b/src/cuckoocache.h new file mode 100644 index 0000000000..efd6a820b5 --- /dev/null +++ b/src/cuckoocache.h @@ -0,0 +1,457 @@ +// Copyright (c) 2016 Jeremy Rubin +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef _BITCOIN_CUCKOOCACHE_H_ +#define _BITCOIN_CUCKOOCACHE_H_ + +#include <array> +#include <algorithm> +#include <atomic> +#include <cstring> +#include <cmath> +#include <memory> +#include <vector> + + +/** namespace CuckooCache provides high performance cache primitives + * + * Summary: + * + * 1) bit_packed_atomic_flags is bit-packed atomic flags for garbage collection + * + * 2) cache is a cache which is performant in memory usage and lookup speed. It + * is lockfree for erase operations. Elements are lazily erased on the next + * insert. + */ +namespace CuckooCache +{ +/** bit_packed_atomic_flags implements a container for garbage collection flags + * that is only thread unsafe on calls to setup. This class bit-packs collection + * flags for memory efficiency. + * + * All operations are std::memory_order_relaxed so external mechanisms must + * ensure that writes and reads are properly synchronized. + * + * On setup(n), all bits up to n are marked as collected. + * + * Under the hood, because it is an 8-bit type, it makes sense to use a multiple + * of 8 for setup, but it will be safe if that is not the case as well. + * + */ +class bit_packed_atomic_flags +{ + std::unique_ptr<std::atomic<uint8_t>[]> mem; + +public: + /** No default constructor as there must be some size */ + bit_packed_atomic_flags() = delete; + + /** + * bit_packed_atomic_flags constructor creates memory to sufficiently + * keep track of garbage collection information for size entries. + * + * @param size the number of elements to allocate space for + * + * @post bit_set, bit_unset, and bit_is_set function properly forall x. x < + * size + * @post All calls to bit_is_set (without subsequent bit_unset) will return + * true. + */ + bit_packed_atomic_flags(uint32_t size) + { + // pad out the size if needed + size = (size + 7) / 8; + mem.reset(new std::atomic<uint8_t>[size]); + for (uint32_t i = 0; i < size; ++i) + mem[i].store(0xFF); + }; + + /** setup marks all entries and ensures that bit_packed_atomic_flags can store + * at least size entries + * + * @param b the number of elements to allocate space for + * @post bit_set, bit_unset, and bit_is_set function properly forall x. x < + * b + * @post All calls to bit_is_set (without subsequent bit_unset) will return + * true. + */ + inline void setup(uint32_t b) + { + bit_packed_atomic_flags d(b); + std::swap(mem, d.mem); + } + + /** bit_set sets an entry as discardable. + * + * @param s the index of the entry to bit_set. + * @post immediately subsequent call (assuming proper external memory + * ordering) to bit_is_set(s) == true. + * + */ + inline void bit_set(uint32_t s) + { + mem[s >> 3].fetch_or(1 << (s & 7), std::memory_order_relaxed); + } + + /** bit_unset marks an entry as something that should not be overwritten + * + * @param s the index of the entry to bit_unset. + * @post immediately subsequent call (assuming proper external memory + * ordering) to bit_is_set(s) == false. + */ + inline void bit_unset(uint32_t s) + { + mem[s >> 3].fetch_and(~(1 << (s & 7)), std::memory_order_relaxed); + } + + /** bit_is_set queries the table for discardability at s + * + * @param s the index of the entry to read. + * @returns if the bit at index s was set. + * */ + inline bool bit_is_set(uint32_t s) const + { + return (1 << (s & 7)) & mem[s >> 3].load(std::memory_order_relaxed); + } +}; + +/** cache implements a cache with properties similar to a cuckoo-set + * + * The cache is able to hold up to (~(uint32_t)0) - 1 elements. + * + * Read Operations: + * - contains(*, false) + * + * Read+Erase Operations: + * - contains(*, true) + * + * Erase Operations: + * - allow_erase() + * + * Write Operations: + * - setup() + * - setup_bytes() + * - insert() + * - please_keep() + * + * Synchronization Free Operations: + * - invalid() + * - compute_hashes() + * + * User Must Guarantee: + * + * 1) Write Requires synchronized access (e.g., a lock) + * 2) Read Requires no concurrent Write, synchronized with the last insert. + * 3) Erase requires no concurrent Write, synchronized with last insert. + * 4) An Erase caller must release all memory before allowing a new Writer. + * + * + * Note on function names: + * - The name "allow_erase" is used because the real discard happens later. + * - The name "please_keep" is used because elements may be erased anyways on insert. + * + * @tparam Element should be a movable and copyable type + * @tparam Hash should be a function/callable which takes a template parameter + * hash_select and an Element and extracts a hash from it. Should return + * high-entropy hashes for `Hash h; h<0>(e) ... h<7>(e)`. + */ +template <typename Element, typename Hash> +class cache +{ +private: + /** table stores all the elements */ + std::vector<Element> table; + + /** size stores the total available slots in the hash table */ + uint32_t size; + + /** The bit_packed_atomic_flags array is marked mutable because we want + * garbage collection to be allowed to occur from const methods */ + mutable bit_packed_atomic_flags collection_flags; + + /** epoch_flags tracks how recently an element was inserted into + * the cache. true denotes recent, false denotes not-recent. See insert() + * method for full semantics. + */ + mutable std::vector<bool> epoch_flags; + + /** epoch_heuristic_counter is used to determine when a epoch might be aged + * & an expensive scan should be done. epoch_heuristic_counter is + * decremented on insert and reset to the new number of inserts which would + * cause the epoch to reach epoch_size when it reaches zero. + */ + uint32_t epoch_heuristic_counter; + + /** epoch_size is set to be the number of elements supposed to be in a + * epoch. When the number of non-erased elements in a epoch + * exceeds epoch_size, a new epoch should be started and all + * current entries demoted. epoch_size is set to be 45% of size because + * we want to keep load around 90%, and we support 3 epochs at once -- + * one "dead" which has been erased, one "dying" which has been marked to be + * erased next, and one "living" which new inserts add to. + */ + uint32_t epoch_size; + + /** hash_mask should be set to appropriately mask out a hash such that every + * masked hash is [0,size), eg, if floor(log2(size)) == 20, then hash_mask + * should be (1<<20)-1 + */ + uint32_t hash_mask; + + /** depth_limit determines how many elements insert should try to replace. + * Should be set to log2(n)*/ + uint8_t depth_limit; + + /** hash_function is a const instance of the hash function. It cannot be + * static or initialized at call time as it may have internal state (such as + * a nonce). + * */ + const Hash hash_function; + + /** compute_hashes is convenience for not having to write out this + * expression everywhere we use the hash values of an Element. + * + * @param e the element whose hashes will be returned + * @returns std::array<uint32_t, 8> of deterministic hashes derived from e + */ + inline std::array<uint32_t, 8> compute_hashes(const Element& e) const + { + return {{hash_function.template operator()<0>(e) & hash_mask, + hash_function.template operator()<1>(e) & hash_mask, + hash_function.template operator()<2>(e) & hash_mask, + hash_function.template operator()<3>(e) & hash_mask, + hash_function.template operator()<4>(e) & hash_mask, + hash_function.template operator()<5>(e) & hash_mask, + hash_function.template operator()<6>(e) & hash_mask, + hash_function.template operator()<7>(e) & hash_mask}}; + } + + /* end + * @returns a constexpr index that can never be inserted to */ + constexpr uint32_t invalid() const + { + return ~(uint32_t)0; + } + + /** allow_erase marks the element at index n as discardable. Threadsafe + * without any concurrent insert. + * @param n the index to allow erasure of + */ + inline void allow_erase(uint32_t n) const + { + collection_flags.bit_set(n); + } + + /** please_keep marks the element at index n as an entry that should be kept. + * Threadsafe without any concurrent insert. + * @param n the index to prioritize keeping + */ + inline void please_keep(uint32_t n) const + { + collection_flags.bit_unset(n); + } + + /** epoch_check handles the changing of epochs for elements stored in the + * cache. epoch_check should be run before every insert. + * + * First, epoch_check decrements and checks the cheap heuristic, and then does + * a more expensive scan if the cheap heuristic runs out. If the expensive + * scan suceeds, the epochs are aged and old elements are allow_erased. The + * cheap heuristic is reset to retrigger after the worst case growth of the + * current epoch's elements would exceed the epoch_size. + */ + void epoch_check() + { + if (epoch_heuristic_counter != 0) { + --epoch_heuristic_counter; + return; + } + // count the number of elements from the latest epoch which + // have not been erased. + uint32_t epoch_unused_count = 0; + for (uint32_t i = 0; i < size; ++i) + epoch_unused_count += epoch_flags[i] && + !collection_flags.bit_is_set(i); + // If there are more non-deleted entries in the current epoch than the + // epoch size, then allow_erase on all elements in the old epoch (marked + // false) and move all elements in the current epoch to the old epoch + // but do not call allow_erase on their indices. + if (epoch_unused_count >= epoch_size) { + for (uint32_t i = 0; i < size; ++i) + if (epoch_flags[i]) + epoch_flags[i] = false; + else + allow_erase(i); + epoch_heuristic_counter = epoch_size; + } else + // reset the epoch_heuristic_counter to next do a scan when worst + // case behavior (no intermittent erases) would exceed epoch size, + // with a reasonable minimum scan size. + // Ordinarily, we would have to sanity check std::min(epoch_size, + // epoch_unused_count), but we already know that `epoch_unused_count + // < epoch_size` in this branch + epoch_heuristic_counter = std::max(1u, std::max(epoch_size / 16, + epoch_size - epoch_unused_count)); + } + +public: + /** You must always construct a cache with some elements via a subsequent + * call to setup or setup_bytes, otherwise operations may segfault. + */ + cache() : table(), size(), collection_flags(0), epoch_flags(), + epoch_heuristic_counter(), epoch_size(), depth_limit(0), hash_function() + { + } + + /** setup initializes the container to store no more than new_size + * elements. setup rounds down to a power of two size. + * + * setup should only be called once. + * + * @param new_size the desired number of elements to store + * @returns the maximum number of elements storable + **/ + uint32_t setup(uint32_t new_size) + { + // depth_limit must be at least one otherwise errors can occur. + depth_limit = static_cast<uint8_t>(std::log2(static_cast<float>(std::max((uint32_t)2, new_size)))); + size = 1 << depth_limit; + hash_mask = size-1; + table.resize(size); + collection_flags.setup(size); + epoch_flags.resize(size); + // Set to 45% as described above + epoch_size = std::max((uint32_t)1, (45 * size) / 100); + // Initially set to wait for a whole epoch + epoch_heuristic_counter = epoch_size; + return size; + } + + /** setup_bytes is a convenience function which accounts for internal memory + * usage when deciding how many elements to store. It isn't perfect because + * it doesn't account for any overhead (struct size, MallocUsage, collection + * and epoch flags). This was done to simplify selecting a power of two + * size. In the expected use case, an extra two bits per entry should be + * negligible compared to the size of the elements. + * + * @param bytes the approximate number of bytes to use for this data + * structure. + * @returns the maximum number of elements storable (see setup() + * documentation for more detail) + */ + uint32_t setup_bytes(size_t bytes) + { + return setup(bytes/sizeof(Element)); + } + + /** insert loops at most depth_limit times trying to insert a hash + * at various locations in the table via a variant of the Cuckoo Algorithm + * with eight hash locations. + * + * It drops the last tried element if it runs out of depth before + * encountering an open slot. + * + * Thus + * + * insert(x); + * return contains(x, false); + * + * is not guaranteed to return true. + * + * @param e the element to insert + * @post one of the following: All previously inserted elements and e are + * now in the table, one previously inserted element is evicted from the + * table, the entry attempted to be inserted is evicted. + * + */ + inline void insert(Element e) + { + epoch_check(); + uint32_t last_loc = invalid(); + bool last_epoch = true; + std::array<uint32_t, 8> locs = compute_hashes(e); + // Make sure we have not already inserted this element + // If we have, make sure that it does not get deleted + for (uint32_t loc : locs) + if (table[loc] == e) { + please_keep(loc); + epoch_flags[loc] = last_epoch; + return; + } + for (uint8_t depth = 0; depth < depth_limit; ++depth) { + // First try to insert to an empty slot, if one exists + for (uint32_t loc : locs) { + if (!collection_flags.bit_is_set(loc)) + continue; + table[loc] = std::move(e); + please_keep(loc); + epoch_flags[loc] = last_epoch; + return; + } + /** Swap with the element at the location that was + * not the last one looked at. Example: + * + * 1) On first iteration, last_loc == invalid(), find returns last, so + * last_loc defaults to locs[0]. + * 2) On further iterations, where last_loc == locs[k], last_loc will + * go to locs[k+1 % 8], i.e., next of the 8 indicies wrapping around + * to 0 if needed. + * + * This prevents moving the element we just put in. + * + * The swap is not a move -- we must switch onto the evicted element + * for the next iteration. + */ + last_loc = locs[(1 + (std::find(locs.begin(), locs.end(), last_loc) - locs.begin())) & 7]; + std::swap(table[last_loc], e); + // Can't std::swap a std::vector<bool>::reference and a bool&. + bool epoch = last_epoch; + last_epoch = epoch_flags[last_loc]; + epoch_flags[last_loc] = epoch; + + // Recompute the locs -- unfortunately happens one too many times! + locs = compute_hashes(e); + } + } + + /* contains iterates through the hash locations for a given element + * and checks to see if it is present. + * + * contains does not check garbage collected state (in other words, + * garbage is only collected when the space is needed), so: + * + * insert(x); + * if (contains(x, true)) + * return contains(x, false); + * else + * return true; + * + * executed on a single thread will always return true! + * + * This is a great property for re-org performance for example. + * + * contains returns a bool set true if the element was found. + * + * @param e the element to check + * @param erase + * + * @post if erase is true and the element is found, then the garbage collect + * flag is set + * @returns true if the element is found, false otherwise + */ + inline bool contains(const Element& e, const bool erase) const + { + std::array<uint32_t, 8> locs = compute_hashes(e); + for (uint32_t loc : locs) + if (table[loc] == e) { + if (erase) + allow_erase(loc); + return true; + } + return false; + } +}; +} // namespace CuckooCache + +#endif diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 4a79bbd17d..08c72c25dd 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -53,38 +53,41 @@ private: const CDBWrapper &parent; leveldb::WriteBatch batch; + CDataStream ssKey; + CDataStream ssValue; + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &_parent) : parent(_parent) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) { }; template <typename K, typename V> void Write(const K& key, const V& value) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(&ssKey[0], ssKey.size()); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); ssValue << value; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); leveldb::Slice slValue(&ssValue[0], ssValue.size()); batch.Put(slKey, slValue); + ssKey.clear(); + ssValue.clear(); } template <typename K> void Erase(const K& key) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(&ssKey[0], ssKey.size()); batch.Delete(slKey); + ssKey.clear(); } }; diff --git a/src/init.cpp b/src/init.cpp index 377d196f2b..e642729a99 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -19,10 +19,11 @@ #include "httpserver.h" #include "httprpc.h" #include "key.h" -#include "main.h" +#include "validation.h" #include "miner.h" #include "netbase.h" #include "net.h" +#include "net_processing.h" #include "policy/policy.h" #include "rpc/server.h" #include "rpc/register.h" @@ -40,6 +41,7 @@ #ifdef ENABLE_WALLET #include "wallet/wallet.h" #endif +#include "warnings.h" #include <stdint.h> #include <stdio.h> #include <memory> @@ -379,6 +381,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), Params(CBaseChainParams::MAIN).GetDefaultPort(), Params(CBaseChainParams::TESTNET).GetDefaultPort())); strUsage += HelpMessageOpt("-proxy=<ip:port>", _("Connect through SOCKS5 proxy")); strUsage += HelpMessageOpt("-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), DEFAULT_PROXYRANDOMIZE)); + strUsage += HelpMessageOpt("-rpcserialversion", strprintf(_("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(1) (default: %d)"), DEFAULT_RPC_SERIALIZE_VERSION)); strUsage += HelpMessageOpt("-seednode=<ip>", _("Connect to a node to retrieve peer addresses, and disconnect")); strUsage += HelpMessageOpt("-timeout=<n>", strprintf(_("Specify connection timeout in milliseconds (minimum: 1, default: %d)"), DEFAULT_CONNECT_TIMEOUT)); strUsage += HelpMessageOpt("-torcontrol=<ip>:<port>", strprintf(_("Tor control port to use if onion listening enabled (default: %s)"), DEFAULT_TOR_CONTROL)); @@ -391,7 +394,7 @@ std::string HelpMessage(HelpMessageMode mode) #endif #endif strUsage += HelpMessageOpt("-whitebind=<addr>", _("Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-whitelist=<netmask>", _("Whitelist peers connecting from the given netmask or IP address. Can be specified multiple times.") + + strUsage += HelpMessageOpt("-whitelist=<IP address or network>", _("Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple times.") + " " + _("Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are already in the mempool, useful e.g. for a gateway")); strUsage += HelpMessageOpt("-whitelistrelay", strprintf(_("Accept relayed transactions received from whitelisted peers even when not relaying transactions (default: %d)"), DEFAULT_WHITELISTRELAY)); strUsage += HelpMessageOpt("-whitelistforcerelay", strprintf(_("Force relay of transactions from whitelisted peers even if they violate local relay policy (default: %d)"), DEFAULT_WHITELISTFORCERELAY)); @@ -983,6 +986,12 @@ bool AppInitParameterInteraction() if (GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); + if (GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) + return InitError("rpcserialversion must be non-negative."); + + if (GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) + return InitError("unknown rpcserialversion requested."); + nMaxTipAge = GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); fEnableReplacement = GetBoolArg("-mempoolreplacement", DEFAULT_ENABLE_REPLACEMENT); @@ -1098,6 +1107,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) LogPrintf("Using config file %s\n", GetConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME)).string()); LogPrintf("Using at most %i connections (%i file descriptors available)\n", nMaxConnections, nFD); + InitSignatureCache(); + LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); if (nScriptCheckThreads) { for (int i=0; i<nScriptCheckThreads-1; i++) diff --git a/src/leveldb/.travis.yml b/src/leveldb/.travis.yml new file mode 100644 index 0000000000..f5bd74c454 --- /dev/null +++ b/src/leveldb/.travis.yml @@ -0,0 +1,13 @@ +language: cpp +compiler: +- clang +- gcc +os: +- linux +- osx +sudo: false +before_install: +- echo $LANG +- echo $LC_ALL +script: +- make -j 4 check diff --git a/src/leveldb/Makefile b/src/leveldb/Makefile index 2bd2cadcdd..07a5a1ead6 100644 --- a/src/leveldb/Makefile +++ b/src/leveldb/Makefile @@ -20,208 +20,395 @@ $(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \ # this file is generated by the previous line to set build flags and sources include build_config.mk +TESTS = \ + db/autocompact_test \ + db/c_test \ + db/corruption_test \ + db/db_test \ + db/dbformat_test \ + db/fault_injection_test \ + db/filename_test \ + db/log_test \ + db/recovery_test \ + db/skiplist_test \ + db/version_edit_test \ + db/version_set_test \ + db/write_batch_test \ + helpers/memenv/memenv_test \ + issues/issue178_test \ + issues/issue200_test \ + table/filter_block_test \ + table/table_test \ + util/arena_test \ + util/bloom_test \ + util/cache_test \ + util/coding_test \ + util/crc32c_test \ + util/env_test \ + util/hash_test + +UTILS = \ + db/db_bench \ + db/leveldbutil + +# Put the object files in a subdirectory, but the application at the top of the object dir. +PROGNAMES := $(notdir $(TESTS) $(UTILS)) + +# On Linux may need libkyotocabinet-dev for dependency. +BENCHMARKS = \ + doc/bench/db_bench_sqlite3 \ + doc/bench/db_bench_tree_db + CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) LDFLAGS += $(PLATFORM_LDFLAGS) LIBS += $(PLATFORM_LIBS) -LIBOBJECTS = $(SOURCES:.cc=.o) -MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o) - -TESTUTIL = ./util/testutil.o -TESTHARNESS = ./util/testharness.o $(TESTUTIL) +SIMULATOR_OUTDIR=out-ios-x86 +DEVICE_OUTDIR=out-ios-arm -# Note: iOS should probably be using libtool, not ar. ifeq ($(PLATFORM), IOS) +# Note: iOS should probably be using libtool, not ar. AR=xcrun ar +SIMULATORSDK=$(shell xcrun -sdk iphonesimulator --show-sdk-path) +DEVICESDK=$(shell xcrun -sdk iphoneos --show-sdk-path) +DEVICE_CFLAGS = -isysroot "$(DEVICESDK)" -arch armv6 -arch armv7 -arch armv7s -arch arm64 +SIMULATOR_CFLAGS = -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64 +STATIC_OUTDIR=out-ios-universal +else +STATIC_OUTDIR=out-static +SHARED_OUTDIR=out-shared +STATIC_PROGRAMS := $(addprefix $(STATIC_OUTDIR)/, $(PROGNAMES)) +SHARED_PROGRAMS := $(addprefix $(SHARED_OUTDIR)/, db_bench) endif -TESTS = \ - arena_test \ - autocompact_test \ - bloom_test \ - c_test \ - cache_test \ - coding_test \ - corruption_test \ - crc32c_test \ - db_test \ - dbformat_test \ - env_test \ - filename_test \ - filter_block_test \ - hash_test \ - issue178_test \ - issue200_test \ - log_test \ - memenv_test \ - skiplist_test \ - table_test \ - version_edit_test \ - version_set_test \ - write_batch_test - -PROGRAMS = db_bench leveldbutil $(TESTS) -BENCHMARKS = db_bench_sqlite3 db_bench_tree_db - -LIBRARY = libleveldb.a -MEMENVLIBRARY = libmemenv.a +STATIC_LIBOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(SOURCES:.cc=.o)) +STATIC_MEMENVOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) + +DEVICE_LIBOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(SOURCES:.cc=.o)) +DEVICE_MEMENVOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) + +SIMULATOR_LIBOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(SOURCES:.cc=.o)) +SIMULATOR_MEMENVOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) + +SHARED_LIBOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(SOURCES:.cc=.o)) +SHARED_MEMENVOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) + +TESTUTIL := $(STATIC_OUTDIR)/util/testutil.o +TESTHARNESS := $(STATIC_OUTDIR)/util/testharness.o $(TESTUTIL) + +STATIC_TESTOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(TESTS))) +STATIC_UTILOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(UTILS))) +STATIC_ALLOBJS := $(STATIC_LIBOBJECTS) $(STATIC_MEMENVOBJECTS) $(STATIC_TESTOBJS) $(STATIC_UTILOBJS) $(TESTHARNESS) +DEVICE_ALLOBJS := $(DEVICE_LIBOBJECTS) $(DEVICE_MEMENVOBJECTS) +SIMULATOR_ALLOBJS := $(SIMULATOR_LIBOBJECTS) $(SIMULATOR_MEMENVOBJECTS) default: all # Should we build shared libraries? ifneq ($(PLATFORM_SHARED_EXT),) +# Many leveldb test apps use non-exported API's. Only build a subset for testing. +SHARED_ALLOBJS := $(SHARED_LIBOBJECTS) $(SHARED_MEMENVOBJECTS) $(TESTHARNESS) + ifneq ($(PLATFORM_SHARED_VERSIONED),true) -SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) -SHARED2 = $(SHARED1) -SHARED3 = $(SHARED1) -SHARED = $(SHARED1) +SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT) +SHARED_LIB2 = $(SHARED_LIB1) +SHARED_LIB3 = $(SHARED_LIB1) +SHARED_LIBS = $(SHARED_LIB1) +SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a else # Update db.h if you change these. -SHARED_MAJOR = 1 -SHARED_MINOR = 18 -SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) -SHARED2 = $(SHARED1).$(SHARED_MAJOR) -SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) -SHARED = $(SHARED1) $(SHARED2) $(SHARED3) -$(SHARED1): $(SHARED3) - ln -fs $(SHARED3) $(SHARED1) -$(SHARED2): $(SHARED3) - ln -fs $(SHARED3) $(SHARED2) +SHARED_VERSION_MAJOR = 1 +SHARED_VERSION_MINOR = 19 +SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT) +SHARED_LIB2 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR) +SHARED_LIB3 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR).$(SHARED_VERSION_MINOR) +SHARED_LIBS = $(SHARED_OUTDIR)/$(SHARED_LIB1) $(SHARED_OUTDIR)/$(SHARED_LIB2) $(SHARED_OUTDIR)/$(SHARED_LIB3) +$(SHARED_OUTDIR)/$(SHARED_LIB1): $(SHARED_OUTDIR)/$(SHARED_LIB3) + ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB1) +$(SHARED_OUTDIR)/$(SHARED_LIB2): $(SHARED_OUTDIR)/$(SHARED_LIB3) + ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB2) +SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a endif -$(SHARED3): - $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3) $(LIBS) +$(SHARED_OUTDIR)/$(SHARED_LIB3): $(SHARED_LIBOBJECTS) + $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED_LIB2) $(SHARED_LIBOBJECTS) -o $(SHARED_OUTDIR)/$(SHARED_LIB3) $(LIBS) endif # PLATFORM_SHARED_EXT -all: $(SHARED) $(LIBRARY) +all: $(SHARED_LIBS) $(SHARED_PROGRAMS) $(STATIC_OUTDIR)/libleveldb.a $(STATIC_OUTDIR)/libmemenv.a $(STATIC_PROGRAMS) -check: all $(PROGRAMS) $(TESTS) - for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done +check: $(STATIC_PROGRAMS) + for t in $(notdir $(TESTS)); do echo "***** Running $$t"; $(STATIC_OUTDIR)/$$t || exit 1; done clean: - -rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk - -rm -rf ios-x86/* ios-arm/* + -rm -rf out-static out-shared out-ios-x86 out-ios-arm out-ios-universal + -rm -f build_config.mk + -rm -rf ios-x86 ios-arm -$(LIBRARY): $(LIBOBJECTS) - rm -f $@ - $(AR) -rs $@ $(LIBOBJECTS) +$(STATIC_OUTDIR): + mkdir $@ -db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) $(LDFLAGS) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS) +$(STATIC_OUTDIR)/db: | $(STATIC_OUTDIR) + mkdir $@ -db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) $(LDFLAGS) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS) +$(STATIC_OUTDIR)/helpers/memenv: | $(STATIC_OUTDIR) + mkdir -p $@ -db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) +$(STATIC_OUTDIR)/port: | $(STATIC_OUTDIR) + mkdir $@ -leveldbutil: db/leveldb_main.o $(LIBOBJECTS) - $(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS) +$(STATIC_OUTDIR)/table: | $(STATIC_OUTDIR) + mkdir $@ -arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(STATIC_OUTDIR)/util: | $(STATIC_OUTDIR) + mkdir $@ -autocompact_test: db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +.PHONY: STATIC_OBJDIRS +STATIC_OBJDIRS: \ + $(STATIC_OUTDIR)/db \ + $(STATIC_OUTDIR)/port \ + $(STATIC_OUTDIR)/table \ + $(STATIC_OUTDIR)/util \ + $(STATIC_OUTDIR)/helpers/memenv -bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR): + mkdir $@ -c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/db: | $(SHARED_OUTDIR) + mkdir $@ -cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/helpers/memenv: | $(SHARED_OUTDIR) + mkdir -p $@ -coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/port: | $(SHARED_OUTDIR) + mkdir $@ -corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/table: | $(SHARED_OUTDIR) + mkdir $@ -crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/util: | $(SHARED_OUTDIR) + mkdir $@ -db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +.PHONY: SHARED_OBJDIRS +SHARED_OBJDIRS: \ + $(SHARED_OUTDIR)/db \ + $(SHARED_OUTDIR)/port \ + $(SHARED_OUTDIR)/table \ + $(SHARED_OUTDIR)/util \ + $(SHARED_OUTDIR)/helpers/memenv -dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR): + mkdir $@ -env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/db: | $(DEVICE_OUTDIR) + mkdir $@ -filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/helpers/memenv: | $(DEVICE_OUTDIR) + mkdir -p $@ -filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/port: | $(DEVICE_OUTDIR) + mkdir $@ -hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/table: | $(DEVICE_OUTDIR) + mkdir $@ -issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/util: | $(DEVICE_OUTDIR) + mkdir $@ -issue200_test: issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +.PHONY: DEVICE_OBJDIRS +DEVICE_OBJDIRS: \ + $(DEVICE_OUTDIR)/db \ + $(DEVICE_OUTDIR)/port \ + $(DEVICE_OUTDIR)/table \ + $(DEVICE_OUTDIR)/util \ + $(DEVICE_OUTDIR)/helpers/memenv -log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR): + mkdir $@ -table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/db: | $(SIMULATOR_OUTDIR) + mkdir $@ -skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/helpers/memenv: | $(SIMULATOR_OUTDIR) + mkdir -p $@ -version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/port: | $(SIMULATOR_OUTDIR) + mkdir $@ -version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/table: | $(SIMULATOR_OUTDIR) + mkdir $@ -write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/util: | $(SIMULATOR_OUTDIR) + mkdir $@ -$(MEMENVLIBRARY) : $(MEMENVOBJECTS) - rm -f $@ - $(AR) -rs $@ $(MEMENVOBJECTS) +.PHONY: SIMULATOR_OBJDIRS +SIMULATOR_OBJDIRS: \ + $(SIMULATOR_OUTDIR)/db \ + $(SIMULATOR_OUTDIR)/port \ + $(SIMULATOR_OUTDIR)/table \ + $(SIMULATOR_OUTDIR)/util \ + $(SIMULATOR_OUTDIR)/helpers/memenv -memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) - $(CXX) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LIBS) +$(STATIC_ALLOBJS): | STATIC_OBJDIRS +$(DEVICE_ALLOBJS): | DEVICE_OBJDIRS +$(SIMULATOR_ALLOBJS): | SIMULATOR_OBJDIRS +$(SHARED_ALLOBJS): | SHARED_OBJDIRS ifeq ($(PLATFORM), IOS) -# For iOS, create universal object files to be used on both the simulator and +$(DEVICE_OUTDIR)/libleveldb.a: $(DEVICE_LIBOBJECTS) + rm -f $@ + $(AR) -rs $@ $(DEVICE_LIBOBJECTS) + +$(SIMULATOR_OUTDIR)/libleveldb.a: $(SIMULATOR_LIBOBJECTS) + rm -f $@ + $(AR) -rs $@ $(SIMULATOR_LIBOBJECTS) + +$(DEVICE_OUTDIR)/libmemenv.a: $(DEVICE_MEMENVOBJECTS) + rm -f $@ + $(AR) -rs $@ $(DEVICE_MEMENVOBJECTS) + +$(SIMULATOR_OUTDIR)/libmemenv.a: $(SIMULATOR_MEMENVOBJECTS) + rm -f $@ + $(AR) -rs $@ $(SIMULATOR_MEMENVOBJECTS) + +# For iOS, create universal object libraries to be used on both the simulator and # a device. -PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms -SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer -DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer -IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString) -IOSARCH=-arch armv6 -arch armv7 -arch armv7s -arch arm64 - -.cc.o: - mkdir -p ios-x86/$(dir $@) - xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ - mkdir -p ios-arm/$(dir $@) - xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ - xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@ - -.c.o: - mkdir -p ios-x86/$(dir $@) - xcrun -sdk iphonesimulator $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ - mkdir -p ios-arm/$(dir $@) - xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ - xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@ +$(STATIC_OUTDIR)/libleveldb.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a + lipo -create $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a -output $@ +$(STATIC_OUTDIR)/libmemenv.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a + lipo -create $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a -output $@ else -.cc.o: +$(STATIC_OUTDIR)/libleveldb.a:$(STATIC_LIBOBJECTS) + rm -f $@ + $(AR) -rs $@ $(STATIC_LIBOBJECTS) + +$(STATIC_OUTDIR)/libmemenv.a:$(STATIC_MEMENVOBJECTS) + rm -f $@ + $(AR) -rs $@ $(STATIC_MEMENVOBJECTS) +endif + +$(SHARED_MEMENVLIB):$(SHARED_MEMENVOBJECTS) + rm -f $@ + $(AR) -rs $@ $(SHARED_MEMENVOBJECTS) + +$(STATIC_OUTDIR)/db_bench:db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/db_bench_sqlite3:doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) + $(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS) + +$(STATIC_OUTDIR)/db_bench_tree_db:doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) + $(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) + +$(STATIC_OUTDIR)/leveldbutil:db/leveldbutil.cc $(STATIC_LIBOBJECTS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/leveldbutil.cc $(STATIC_LIBOBJECTS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/arena_test:util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/autocompact_test:db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/bloom_test:util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/c_test:$(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/cache_test:util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/coding_test:util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/corruption_test:db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/crc32c_test:util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/db_test:db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/dbformat_test:db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/env_test:util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/fault_injection_test:db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/filename_test:db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/filter_block_test:table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/hash_test:util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/issue178_test:issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/issue200_test:issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/log_test:db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/recovery_test:db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/table_test:table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/skiplist_test:db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/version_edit_test:db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/version_set_test:db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/write_batch_test:db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/memenv_test:$(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) + $(XCRUN) $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) -o $@ $(LIBS) + +$(SHARED_OUTDIR)/db_bench:$(SHARED_OUTDIR)/db/db_bench.o $(SHARED_LIBS) $(TESTUTIL) + $(XCRUN) $(CXX) $(LDFLAGS) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SHARED_OUTDIR)/db/db_bench.o $(TESTUTIL) $(SHARED_OUTDIR)/$(SHARED_LIB3) -o $@ $(LIBS) + +.PHONY: run-shared +run-shared: $(SHARED_OUTDIR)/db_bench + LD_LIBRARY_PATH=$(SHARED_OUTDIR) $(SHARED_OUTDIR)/db_bench + +$(SIMULATOR_OUTDIR)/%.o: %.cc + xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@ + +$(DEVICE_OUTDIR)/%.o: %.cc + xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) $(DEVICE_CFLAGS) -c $< -o $@ + +$(SIMULATOR_OUTDIR)/%.o: %.c + xcrun -sdk iphonesimulator $(CC) $(CFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@ + +$(DEVICE_OUTDIR)/%.o: %.c + xcrun -sdk iphoneos $(CC) $(CFLAGS) $(DEVICE_CFLAGS) -c $< -o $@ + +$(STATIC_OUTDIR)/%.o: %.cc $(CXX) $(CXXFLAGS) -c $< -o $@ -.c.o: +$(STATIC_OUTDIR)/%.o: %.c $(CC) $(CFLAGS) -c $< -o $@ -endif + +$(SHARED_OUTDIR)/%.o: %.cc + $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@ + +$(SHARED_OUTDIR)/%.o: %.c + $(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@ diff --git a/src/leveldb/README b/src/leveldb/README deleted file mode 100644 index 3618adeeed..0000000000 --- a/src/leveldb/README +++ /dev/null @@ -1,51 +0,0 @@ -leveldb: A key-value store -Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) - -The code under this directory implements a system for maintaining a -persistent key/value store. - -See doc/index.html for more explanation. -See doc/impl.html for a brief overview of the implementation. - -The public interface is in include/*.h. Callers should not include or -rely on the details of any other header files in this package. Those -internal APIs may be changed without warning. - -Guide to header files: - -include/db.h - Main interface to the DB: Start here - -include/options.h - Control over the behavior of an entire database, and also - control over the behavior of individual reads and writes. - -include/comparator.h - Abstraction for user-specified comparison function. If you want - just bytewise comparison of keys, you can use the default comparator, - but clients can write their own comparator implementations if they - want custom ordering (e.g. to handle different character - encodings, etc.) - -include/iterator.h - Interface for iterating over data. You can get an iterator - from a DB object. - -include/write_batch.h - Interface for atomically applying multiple updates to a database. - -include/slice.h - A simple module for maintaining a pointer and a length into some - other byte array. - -include/status.h - Status is returned from many of the public interfaces and is used - to report success and various kinds of errors. - -include/env.h - Abstraction of the OS environment. A posix implementation of - this interface is in util/env_posix.cc - -include/table.h -include/table_builder.h - Lower-level modules that most clients probably won't use directly diff --git a/src/leveldb/README.md b/src/leveldb/README.md index 480affb5ca..c75b185e0e 100644 --- a/src/leveldb/README.md +++ b/src/leveldb/README.md @@ -1,5 +1,7 @@ **LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.** +[](https://travis-ci.org/google/leveldb) + Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) # Features @@ -10,9 +12,11 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) * Multiple changes can be made in one atomic batch. * Users can create a transient snapshot to get a consistent view of data. * Forward and backward iteration is supported over the data. - * Data is automatically compressed using the [Snappy compression library](http://code.google.com/p/snappy). + * Data is automatically compressed using the [Snappy compression library](http://google.github.io/snappy/). * External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions. - * [Detailed documentation](http://htmlpreview.github.io/?https://github.com/google/leveldb/blob/master/doc/index.html) about how to use the library is included with the source code. + +# Documentation + [LevelDB library documentation](https://rawgit.com/google/leveldb/master/doc/index.html) is online and bundled with the source code. # Limitations @@ -20,6 +24,37 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) * Only a single process (possibly multi-threaded) can access a particular database at a time. * There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library. +# Contributing to the leveldb Project +The leveldb project welcomes contributions. leveldb's primary goal is to be +a reliable and fast key/value store. Changes that are in line with the +features/limitations outlined above, and meet the requirements below, +will be considered. + +Contribution requirements: + +1. **POSIX only**. We _generally_ will only accept changes that are both + compiled, and tested on a POSIX platform - usually Linux. Very small + changes will sometimes be accepted, but consider that more of an + exception than the rule. + +2. **Stable API**. We strive very hard to maintain a stable API. Changes that + require changes for projects using leveldb _might_ be rejected without + sufficient benefit to the project. + +3. **Tests**: All changes must be accompanied by a new (or changed) test, or + a sufficient explanation as to why a new (or changed) test is not required. + +## Submitting a Pull Request +Before any pull request will be accepted the author must first sign a +Contributor License Agreement (CLA) at https://cla.developers.google.com/. + +In order to keep the commit timeline linear +[squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits) +your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase) +on google/leveldb/master. This keeps the commit timeline linear and more easily sync'ed +with the internal repository at Google. More information at GitHub's +[About Git rebase](https://help.github.com/articles/about-git-rebase/) page. + # Performance Here is a performance report (with explanations) from the run of the diff --git a/src/leveldb/build_detect_platform b/src/leveldb/build_detect_platform index a1101c1bda..d7edab1d87 100755 --- a/src/leveldb/build_detect_platform +++ b/src/leveldb/build_detect_platform @@ -175,7 +175,7 @@ DIRS="$PREFIX/db $PREFIX/util $PREFIX/table" set -f # temporarily disable globbing so that our patterns aren't expanded PRUNE_TEST="-name *test*.cc -prune" PRUNE_BENCH="-name *_bench.cc -prune" -PRUNE_TOOL="-name leveldb_main.cc -prune" +PRUNE_TOOL="-name leveldbutil.cc -prune" PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "` set +f # re-enable globbing diff --git a/src/leveldb/db/corruption_test.cc b/src/leveldb/db/corruption_test.cc index 96afc68913..37a484d25f 100644 --- a/src/leveldb/db/corruption_test.cc +++ b/src/leveldb/db/corruption_test.cc @@ -36,7 +36,7 @@ class CorruptionTest { tiny_cache_ = NewLRUCache(100); options_.env = &env_; options_.block_cache = tiny_cache_; - dbname_ = test::TmpDir() + "/db_test"; + dbname_ = test::TmpDir() + "/corruption_test"; DestroyDB(dbname_, options_); db_ = NULL; diff --git a/src/leveldb/db/db_bench.cc b/src/leveldb/db/db_bench.cc index 705a170aae..7a0f5e08cd 100644 --- a/src/leveldb/db/db_bench.cc +++ b/src/leveldb/db/db_bench.cc @@ -33,6 +33,7 @@ // readmissing -- read N missing keys in random order // readhot -- read N times in random order from 1% section of DB // seekrandom -- N random seeks +// open -- cost of opening a DB // crc32c -- repeated crc32c of 4K of data // acquireload -- load N*1000 times // Meta operations: @@ -99,6 +100,9 @@ static int FLAGS_bloom_bits = -1; // benchmark will fail. static bool FLAGS_use_existing_db = false; +// If true, reuse existing log/MANIFEST files when re-opening a database. +static bool FLAGS_reuse_logs = false; + // Use the db with the following name. static const char* FLAGS_db = NULL; @@ -138,6 +142,7 @@ class RandomGenerator { } }; +#if defined(__linux) static Slice TrimSpace(Slice s) { size_t start = 0; while (start < s.size() && isspace(s[start])) { @@ -149,6 +154,7 @@ static Slice TrimSpace(Slice s) { } return Slice(s.data() + start, limit - start); } +#endif static void AppendWithSpace(std::string* str, Slice msg) { if (msg.empty()) return; @@ -442,7 +448,11 @@ class Benchmark { bool fresh_db = false; int num_threads = FLAGS_threads; - if (name == Slice("fillseq")) { + if (name == Slice("open")) { + method = &Benchmark::OpenBench; + num_ /= 10000; + if (num_ < 1) num_ = 1; + } else if (name == Slice("fillseq")) { fresh_db = true; method = &Benchmark::WriteSeq; } else if (name == Slice("fillbatch")) { @@ -695,6 +705,7 @@ class Benchmark { options.write_buffer_size = FLAGS_write_buffer_size; options.max_open_files = FLAGS_open_files; options.filter_policy = filter_policy_; + options.reuse_logs = FLAGS_reuse_logs; Status s = DB::Open(options, FLAGS_db, &db_); if (!s.ok()) { fprintf(stderr, "open error: %s\n", s.ToString().c_str()); @@ -702,6 +713,14 @@ class Benchmark { } } + void OpenBench(ThreadState* thread) { + for (int i = 0; i < num_; i++) { + delete db_; + Open(); + thread->stats.FinishedSingleOp(); + } + } + void WriteSeq(ThreadState* thread) { DoWrite(thread, true); } @@ -941,6 +960,9 @@ int main(int argc, char** argv) { } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && (n == 0 || n == 1)) { FLAGS_use_existing_db = n; + } else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_reuse_logs = n; } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { FLAGS_num = n; } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc index 49b95953b4..60f4e66e55 100644 --- a/src/leveldb/db/db_impl.cc +++ b/src/leveldb/db/db_impl.cc @@ -125,7 +125,7 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) db_lock_(NULL), shutting_down_(NULL), bg_cv_(&mutex_), - mem_(new MemTable(internal_comparator_)), + mem_(NULL), imm_(NULL), logfile_(NULL), logfile_number_(0), @@ -134,7 +134,6 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) tmp_batch_(new WriteBatch), bg_compaction_scheduled_(false), manual_compaction_(NULL) { - mem_->Ref(); has_imm_.Release_Store(NULL); // Reserve ten files or so for other uses and give the rest to TableCache. @@ -271,7 +270,7 @@ void DBImpl::DeleteObsoleteFiles() { } } -Status DBImpl::Recover(VersionEdit* edit) { +Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { mutex_.AssertHeld(); // Ignore error from CreateDir since the creation of the DB is @@ -301,66 +300,69 @@ Status DBImpl::Recover(VersionEdit* edit) { } } - s = versions_->Recover(); - if (s.ok()) { - SequenceNumber max_sequence(0); - - // Recover from all newer log files than the ones named in the - // descriptor (new log files may have been added by the previous - // incarnation without registering them in the descriptor). - // - // Note that PrevLogNumber() is no longer used, but we pay - // attention to it in case we are recovering a database - // produced by an older version of leveldb. - const uint64_t min_log = versions_->LogNumber(); - const uint64_t prev_log = versions_->PrevLogNumber(); - std::vector<std::string> filenames; - s = env_->GetChildren(dbname_, &filenames); + s = versions_->Recover(save_manifest); + if (!s.ok()) { + return s; + } + SequenceNumber max_sequence(0); + + // Recover from all newer log files than the ones named in the + // descriptor (new log files may have been added by the previous + // incarnation without registering them in the descriptor). + // + // Note that PrevLogNumber() is no longer used, but we pay + // attention to it in case we are recovering a database + // produced by an older version of leveldb. + const uint64_t min_log = versions_->LogNumber(); + const uint64_t prev_log = versions_->PrevLogNumber(); + std::vector<std::string> filenames; + s = env_->GetChildren(dbname_, &filenames); + if (!s.ok()) { + return s; + } + std::set<uint64_t> expected; + versions_->AddLiveFiles(&expected); + uint64_t number; + FileType type; + std::vector<uint64_t> logs; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type)) { + expected.erase(number); + if (type == kLogFile && ((number >= min_log) || (number == prev_log))) + logs.push_back(number); + } + } + if (!expected.empty()) { + char buf[50]; + snprintf(buf, sizeof(buf), "%d missing files; e.g.", + static_cast<int>(expected.size())); + return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); + } + + // Recover in the order in which the logs were generated + std::sort(logs.begin(), logs.end()); + for (size_t i = 0; i < logs.size(); i++) { + s = RecoverLogFile(logs[i], (i == logs.size() - 1), save_manifest, edit, + &max_sequence); if (!s.ok()) { return s; } - std::set<uint64_t> expected; - versions_->AddLiveFiles(&expected); - uint64_t number; - FileType type; - std::vector<uint64_t> logs; - for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type)) { - expected.erase(number); - if (type == kLogFile && ((number >= min_log) || (number == prev_log))) - logs.push_back(number); - } - } - if (!expected.empty()) { - char buf[50]; - snprintf(buf, sizeof(buf), "%d missing files; e.g.", - static_cast<int>(expected.size())); - return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); - } - - // Recover in the order in which the logs were generated - std::sort(logs.begin(), logs.end()); - for (size_t i = 0; i < logs.size(); i++) { - s = RecoverLogFile(logs[i], edit, &max_sequence); - // The previous incarnation may not have written any MANIFEST - // records after allocating this log number. So we manually - // update the file number allocation counter in VersionSet. - versions_->MarkFileNumberUsed(logs[i]); - } + // The previous incarnation may not have written any MANIFEST + // records after allocating this log number. So we manually + // update the file number allocation counter in VersionSet. + versions_->MarkFileNumberUsed(logs[i]); + } - if (s.ok()) { - if (versions_->LastSequence() < max_sequence) { - versions_->SetLastSequence(max_sequence); - } - } + if (versions_->LastSequence() < max_sequence) { + versions_->SetLastSequence(max_sequence); } - return s; + return Status::OK(); } -Status DBImpl::RecoverLogFile(uint64_t log_number, - VersionEdit* edit, +Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, + bool* save_manifest, VersionEdit* edit, SequenceNumber* max_sequence) { struct LogReporter : public log::Reader::Reporter { Env* env; @@ -405,6 +407,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, std::string scratch; Slice record; WriteBatch batch; + int compactions = 0; MemTable* mem = NULL; while (reader.ReadRecord(&record, &scratch) && status.ok()) { @@ -432,25 +435,52 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, } if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) { + compactions++; + *save_manifest = true; status = WriteLevel0Table(mem, edit, NULL); + mem->Unref(); + mem = NULL; if (!status.ok()) { // Reflect errors immediately so that conditions like full // file-systems cause the DB::Open() to fail. break; } - mem->Unref(); - mem = NULL; } } - if (status.ok() && mem != NULL) { - status = WriteLevel0Table(mem, edit, NULL); - // Reflect errors immediately so that conditions like full - // file-systems cause the DB::Open() to fail. + delete file; + + // See if we should keep reusing the last log file. + if (status.ok() && options_.reuse_logs && last_log && compactions == 0) { + assert(logfile_ == NULL); + assert(log_ == NULL); + assert(mem_ == NULL); + uint64_t lfile_size; + if (env_->GetFileSize(fname, &lfile_size).ok() && + env_->NewAppendableFile(fname, &logfile_).ok()) { + Log(options_.info_log, "Reusing old log %s \n", fname.c_str()); + log_ = new log::Writer(logfile_, lfile_size); + logfile_number_ = log_number; + if (mem != NULL) { + mem_ = mem; + mem = NULL; + } else { + // mem can be NULL if lognum exists but was empty. + mem_ = new MemTable(internal_comparator_); + mem_->Ref(); + } + } + } + + if (mem != NULL) { + // mem did not get reused; compact it. + if (status.ok()) { + *save_manifest = true; + status = WriteLevel0Table(mem, edit, NULL); + } + mem->Unref(); } - if (mem != NULL) mem->Unref(); - delete file; return status; } @@ -821,8 +851,9 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, delete iter; if (s.ok()) { Log(options_.info_log, - "Generated table #%llu: %lld keys, %lld bytes", + "Generated table #%llu@%d: %lld keys, %lld bytes", (unsigned long long) output_number, + compact->compaction->level(), (unsigned long long) current_entries, (unsigned long long) current_bytes); } @@ -1395,6 +1426,19 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) { } else if (in == "sstables") { *value = versions_->current()->DebugString(); return true; + } else if (in == "approximate-memory-usage") { + size_t total_usage = options_.block_cache->TotalCharge(); + if (mem_) { + total_usage += mem_->ApproximateMemoryUsage(); + } + if (imm_) { + total_usage += imm_->ApproximateMemoryUsage(); + } + char buf[50]; + snprintf(buf, sizeof(buf), "%llu", + static_cast<unsigned long long>(total_usage)); + value->append(buf); + return true; } return false; @@ -1449,8 +1493,11 @@ Status DB::Open(const Options& options, const std::string& dbname, DBImpl* impl = new DBImpl(options, dbname); impl->mutex_.Lock(); VersionEdit edit; - Status s = impl->Recover(&edit); // Handles create_if_missing, error_if_exists - if (s.ok()) { + // Recover handles create_if_missing, error_if_exists + bool save_manifest = false; + Status s = impl->Recover(&edit, &save_manifest); + if (s.ok() && impl->mem_ == NULL) { + // Create new log and a corresponding memtable. uint64_t new_log_number = impl->versions_->NewFileNumber(); WritableFile* lfile; s = options.env->NewWritableFile(LogFileName(dbname, new_log_number), @@ -1460,15 +1507,22 @@ Status DB::Open(const Options& options, const std::string& dbname, impl->logfile_ = lfile; impl->logfile_number_ = new_log_number; impl->log_ = new log::Writer(lfile); - s = impl->versions_->LogAndApply(&edit, &impl->mutex_); - } - if (s.ok()) { - impl->DeleteObsoleteFiles(); - impl->MaybeScheduleCompaction(); + impl->mem_ = new MemTable(impl->internal_comparator_); + impl->mem_->Ref(); } } + if (s.ok() && save_manifest) { + edit.SetPrevLogNumber(0); // No older logs needed after recovery. + edit.SetLogNumber(impl->logfile_number_); + s = impl->versions_->LogAndApply(&edit, &impl->mutex_); + } + if (s.ok()) { + impl->DeleteObsoleteFiles(); + impl->MaybeScheduleCompaction(); + } impl->mutex_.Unlock(); if (s.ok()) { + assert(impl->mem_ != NULL); *dbptr = impl; } else { delete impl; diff --git a/src/leveldb/db/db_impl.h b/src/leveldb/db/db_impl.h index cfc998164a..8ff323e728 100644 --- a/src/leveldb/db/db_impl.h +++ b/src/leveldb/db/db_impl.h @@ -78,7 +78,8 @@ class DBImpl : public DB { // Recover the descriptor from persistent storage. May do a significant // amount of work to recover recently logged updates. Any changes to // be made to the descriptor are added to *edit. - Status Recover(VersionEdit* edit) EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Status Recover(VersionEdit* edit, bool* save_manifest) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); void MaybeIgnoreError(Status* s) const; @@ -90,9 +91,8 @@ class DBImpl : public DB { // Errors are recorded in bg_error_. void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_); - Status RecoverLogFile(uint64_t log_number, - VersionEdit* edit, - SequenceNumber* max_sequence) + Status RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest, + VersionEdit* edit, SequenceNumber* max_sequence) EXCLUSIVE_LOCKS_REQUIRED(mutex_); Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base) diff --git a/src/leveldb/db/db_test.cc b/src/leveldb/db/db_test.cc index 0fed9137d5..a0b08bc19c 100644 --- a/src/leveldb/db/db_test.cc +++ b/src/leveldb/db/db_test.cc @@ -193,6 +193,7 @@ class DBTest { // Sequence of option configurations to try enum OptionConfig { kDefault, + kReuse, kFilter, kUncompressed, kEnd @@ -237,7 +238,11 @@ class DBTest { // Return the current option configuration. Options CurrentOptions() { Options options; + options.reuse_logs = false; switch (option_config_) { + case kReuse: + options.reuse_logs = true; + break; case kFilter: options.filter_policy = filter_policy_; break; @@ -558,6 +563,17 @@ TEST(DBTest, GetFromVersions) { } while (ChangeOptions()); } +TEST(DBTest, GetMemUsage) { + do { + ASSERT_OK(Put("foo", "v1")); + std::string val; + ASSERT_TRUE(db_->GetProperty("leveldb.approximate-memory-usage", &val)); + int mem_usage = atoi(val.c_str()); + ASSERT_GT(mem_usage, 0); + ASSERT_LT(mem_usage, 5*1024*1024); + } while (ChangeOptions()); +} + TEST(DBTest, GetSnapshot) { do { // Try with both a short key and a long key @@ -1080,6 +1096,14 @@ TEST(DBTest, ApproximateSizes) { // 0 because GetApproximateSizes() does not account for memtable space ASSERT_TRUE(Between(Size("", Key(50)), 0, 0)); + if (options.reuse_logs) { + // Recovery will reuse memtable, and GetApproximateSizes() does not + // account for memtable usage; + Reopen(&options); + ASSERT_TRUE(Between(Size("", Key(50)), 0, 0)); + continue; + } + // Check sizes across recovery by reopening a few times for (int run = 0; run < 3; run++) { Reopen(&options); @@ -1123,6 +1147,11 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) { ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000))); ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000))); + if (options.reuse_logs) { + // Need to force a memtable compaction since recovery does not do so. + ASSERT_OK(dbfull()->TEST_CompactMemTable()); + } + // Check sizes across recovery by reopening a few times for (int run = 0; run < 3; run++) { Reopen(&options); @@ -2084,7 +2113,8 @@ void BM_LogAndApply(int iters, int num_base_files) { InternalKeyComparator cmp(BytewiseComparator()); Options options; VersionSet vset(dbname, &options, NULL, &cmp); - ASSERT_OK(vset.Recover()); + bool save_manifest; + ASSERT_OK(vset.Recover(&save_manifest)); VersionEdit vbase; uint64_t fnum = 1; for (int i = 0; i < num_base_files; i++) { diff --git a/src/leveldb/db/fault_injection_test.cc b/src/leveldb/db/fault_injection_test.cc new file mode 100644 index 0000000000..875dfe81ee --- /dev/null +++ b/src/leveldb/db/fault_injection_test.cc @@ -0,0 +1,554 @@ +// Copyright 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// This test uses a custom Env to keep track of the state of a filesystem as of +// the last "sync". It then checks for data loss errors by purposely dropping +// file data (or entire files) not protected by a "sync". + +#include "leveldb/db.h" + +#include <map> +#include <set> +#include "db/db_impl.h" +#include "db/filename.h" +#include "db/log_format.h" +#include "db/version_set.h" +#include "leveldb/cache.h" +#include "leveldb/env.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" +#include "util/mutexlock.h" +#include "util/testharness.h" +#include "util/testutil.h" + +namespace leveldb { + +static const int kValueSize = 1000; +static const int kMaxNumValues = 2000; +static const size_t kNumIterations = 3; + +class FaultInjectionTestEnv; + +namespace { + +// Assume a filename, and not a directory name like "/foo/bar/" +static std::string GetDirName(const std::string filename) { + size_t found = filename.find_last_of("/\\"); + if (found == std::string::npos) { + return ""; + } else { + return filename.substr(0, found); + } +} + +Status SyncDir(const std::string& dir) { + // As this is a test it isn't required to *actually* sync this directory. + return Status::OK(); +} + +// A basic file truncation function suitable for this test. +Status Truncate(const std::string& filename, uint64_t length) { + leveldb::Env* env = leveldb::Env::Default(); + + SequentialFile* orig_file; + Status s = env->NewSequentialFile(filename, &orig_file); + if (!s.ok()) + return s; + + char* scratch = new char[length]; + leveldb::Slice result; + s = orig_file->Read(length, &result, scratch); + delete orig_file; + if (s.ok()) { + std::string tmp_name = GetDirName(filename) + "/truncate.tmp"; + WritableFile* tmp_file; + s = env->NewWritableFile(tmp_name, &tmp_file); + if (s.ok()) { + s = tmp_file->Append(result); + delete tmp_file; + if (s.ok()) { + s = env->RenameFile(tmp_name, filename); + } else { + env->DeleteFile(tmp_name); + } + } + } + + delete[] scratch; + + return s; +} + +struct FileState { + std::string filename_; + ssize_t pos_; + ssize_t pos_at_last_sync_; + ssize_t pos_at_last_flush_; + + FileState(const std::string& filename) + : filename_(filename), + pos_(-1), + pos_at_last_sync_(-1), + pos_at_last_flush_(-1) { } + + FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} + + bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; } + + Status DropUnsyncedData() const; +}; + +} // anonymous namespace + +// A wrapper around WritableFile which informs another Env whenever this file +// is written to or sync'ed. +class TestWritableFile : public WritableFile { + public: + TestWritableFile(const FileState& state, + WritableFile* f, + FaultInjectionTestEnv* env); + virtual ~TestWritableFile(); + virtual Status Append(const Slice& data); + virtual Status Close(); + virtual Status Flush(); + virtual Status Sync(); + + private: + FileState state_; + WritableFile* target_; + bool writable_file_opened_; + FaultInjectionTestEnv* env_; + + Status SyncParent(); +}; + +class FaultInjectionTestEnv : public EnvWrapper { + public: + FaultInjectionTestEnv() : EnvWrapper(Env::Default()), filesystem_active_(true) {} + virtual ~FaultInjectionTestEnv() { } + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result); + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result); + virtual Status DeleteFile(const std::string& f); + virtual Status RenameFile(const std::string& s, const std::string& t); + + void WritableFileClosed(const FileState& state); + Status DropUnsyncedFileData(); + Status DeleteFilesCreatedAfterLastDirSync(); + void DirWasSynced(); + bool IsFileCreatedSinceLastDirSync(const std::string& filename); + void ResetState(); + void UntrackFile(const std::string& f); + // Setting the filesystem to inactive is the test equivalent to simulating a + // system reset. Setting to inactive will freeze our saved filesystem state so + // that it will stop being recorded. It can then be reset back to the state at + // the time of the reset. + bool IsFilesystemActive() const { return filesystem_active_; } + void SetFilesystemActive(bool active) { filesystem_active_ = active; } + + private: + port::Mutex mutex_; + std::map<std::string, FileState> db_file_state_; + std::set<std::string> new_files_since_last_dir_sync_; + bool filesystem_active_; // Record flushes, syncs, writes +}; + +TestWritableFile::TestWritableFile(const FileState& state, + WritableFile* f, + FaultInjectionTestEnv* env) + : state_(state), + target_(f), + writable_file_opened_(true), + env_(env) { + assert(f != NULL); +} + +TestWritableFile::~TestWritableFile() { + if (writable_file_opened_) { + Close(); + } + delete target_; +} + +Status TestWritableFile::Append(const Slice& data) { + Status s = target_->Append(data); + if (s.ok() && env_->IsFilesystemActive()) { + state_.pos_ += data.size(); + } + return s; +} + +Status TestWritableFile::Close() { + writable_file_opened_ = false; + Status s = target_->Close(); + if (s.ok()) { + env_->WritableFileClosed(state_); + } + return s; +} + +Status TestWritableFile::Flush() { + Status s = target_->Flush(); + if (s.ok() && env_->IsFilesystemActive()) { + state_.pos_at_last_flush_ = state_.pos_; + } + return s; +} + +Status TestWritableFile::SyncParent() { + Status s = SyncDir(GetDirName(state_.filename_)); + if (s.ok()) { + env_->DirWasSynced(); + } + return s; +} + +Status TestWritableFile::Sync() { + if (!env_->IsFilesystemActive()) { + return Status::OK(); + } + // Ensure new files referred to by the manifest are in the filesystem. + Status s = target_->Sync(); + if (s.ok()) { + state_.pos_at_last_sync_ = state_.pos_; + } + if (env_->IsFileCreatedSinceLastDirSync(state_.filename_)) { + Status ps = SyncParent(); + if (s.ok() && !ps.ok()) { + s = ps; + } + } + return s; +} + +Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname, + WritableFile** result) { + WritableFile* actual_writable_file; + Status s = target()->NewWritableFile(fname, &actual_writable_file); + if (s.ok()) { + FileState state(fname); + state.pos_ = 0; + *result = new TestWritableFile(state, actual_writable_file, this); + // NewWritableFile doesn't append to files, so if the same file is + // opened again then it will be truncated - so forget our saved + // state. + UntrackFile(fname); + MutexLock l(&mutex_); + new_files_since_last_dir_sync_.insert(fname); + } + return s; +} + +Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname, + WritableFile** result) { + WritableFile* actual_writable_file; + Status s = target()->NewAppendableFile(fname, &actual_writable_file); + if (s.ok()) { + FileState state(fname); + state.pos_ = 0; + { + MutexLock l(&mutex_); + if (db_file_state_.count(fname) == 0) { + new_files_since_last_dir_sync_.insert(fname); + } else { + state = db_file_state_[fname]; + } + } + *result = new TestWritableFile(state, actual_writable_file, this); + } + return s; +} + +Status FaultInjectionTestEnv::DropUnsyncedFileData() { + Status s; + MutexLock l(&mutex_); + for (std::map<std::string, FileState>::const_iterator it = + db_file_state_.begin(); + s.ok() && it != db_file_state_.end(); ++it) { + const FileState& state = it->second; + if (!state.IsFullySynced()) { + s = state.DropUnsyncedData(); + } + } + return s; +} + +void FaultInjectionTestEnv::DirWasSynced() { + MutexLock l(&mutex_); + new_files_since_last_dir_sync_.clear(); +} + +bool FaultInjectionTestEnv::IsFileCreatedSinceLastDirSync( + const std::string& filename) { + MutexLock l(&mutex_); + return new_files_since_last_dir_sync_.find(filename) != + new_files_since_last_dir_sync_.end(); +} + +void FaultInjectionTestEnv::UntrackFile(const std::string& f) { + MutexLock l(&mutex_); + db_file_state_.erase(f); + new_files_since_last_dir_sync_.erase(f); +} + +Status FaultInjectionTestEnv::DeleteFile(const std::string& f) { + Status s = EnvWrapper::DeleteFile(f); + ASSERT_OK(s); + if (s.ok()) { + UntrackFile(f); + } + return s; +} + +Status FaultInjectionTestEnv::RenameFile(const std::string& s, + const std::string& t) { + Status ret = EnvWrapper::RenameFile(s, t); + + if (ret.ok()) { + MutexLock l(&mutex_); + if (db_file_state_.find(s) != db_file_state_.end()) { + db_file_state_[t] = db_file_state_[s]; + db_file_state_.erase(s); + } + + if (new_files_since_last_dir_sync_.erase(s) != 0) { + assert(new_files_since_last_dir_sync_.find(t) == + new_files_since_last_dir_sync_.end()); + new_files_since_last_dir_sync_.insert(t); + } + } + + return ret; +} + +void FaultInjectionTestEnv::ResetState() { + // Since we are not destroying the database, the existing files + // should keep their recorded synced/flushed state. Therefore + // we do not reset db_file_state_ and new_files_since_last_dir_sync_. + MutexLock l(&mutex_); + SetFilesystemActive(true); +} + +Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() { + // Because DeleteFile access this container make a copy to avoid deadlock + mutex_.Lock(); + std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(), + new_files_since_last_dir_sync_.end()); + mutex_.Unlock(); + Status s; + std::set<std::string>::const_iterator it; + for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) { + s = DeleteFile(*it); + } + return s; +} + +void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { + MutexLock l(&mutex_); + db_file_state_[state.filename_] = state; +} + +Status FileState::DropUnsyncedData() const { + ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; + return Truncate(filename_, sync_pos); +} + +class FaultInjectionTest { + public: + enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR }; + enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES }; + + FaultInjectionTestEnv* env_; + std::string dbname_; + Cache* tiny_cache_; + Options options_; + DB* db_; + + FaultInjectionTest() + : env_(new FaultInjectionTestEnv), + tiny_cache_(NewLRUCache(100)), + db_(NULL) { + dbname_ = test::TmpDir() + "/fault_test"; + DestroyDB(dbname_, Options()); // Destroy any db from earlier run + options_.reuse_logs = true; + options_.env = env_; + options_.paranoid_checks = true; + options_.block_cache = tiny_cache_; + options_.create_if_missing = true; + } + + ~FaultInjectionTest() { + CloseDB(); + DestroyDB(dbname_, Options()); + delete tiny_cache_; + delete env_; + } + + void ReuseLogs(bool reuse) { + options_.reuse_logs = reuse; + } + + void Build(int start_idx, int num_vals) { + std::string key_space, value_space; + WriteBatch batch; + for (int i = start_idx; i < start_idx + num_vals; i++) { + Slice key = Key(i, &key_space); + batch.Clear(); + batch.Put(key, Value(i, &value_space)); + WriteOptions options; + ASSERT_OK(db_->Write(options, &batch)); + } + } + + Status ReadValue(int i, std::string* val) const { + std::string key_space, value_space; + Slice key = Key(i, &key_space); + Value(i, &value_space); + ReadOptions options; + return db_->Get(options, key, val); + } + + Status Verify(int start_idx, int num_vals, + ExpectedVerifResult expected) const { + std::string val; + std::string value_space; + Status s; + for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) { + Value(i, &value_space); + s = ReadValue(i, &val); + if (expected == VAL_EXPECT_NO_ERROR) { + if (s.ok()) { + ASSERT_EQ(value_space, val); + } + } else if (s.ok()) { + fprintf(stderr, "Expected an error at %d, but was OK\n", i); + s = Status::IOError(dbname_, "Expected value error:"); + } else { + s = Status::OK(); // An expected error + } + } + return s; + } + + // Return the ith key + Slice Key(int i, std::string* storage) const { + char buf[100]; + snprintf(buf, sizeof(buf), "%016d", i); + storage->assign(buf, strlen(buf)); + return Slice(*storage); + } + + // Return the value to associate with the specified key + Slice Value(int k, std::string* storage) const { + Random r(k); + return test::RandomString(&r, kValueSize, storage); + } + + Status OpenDB() { + delete db_; + db_ = NULL; + env_->ResetState(); + return DB::Open(options_, dbname_, &db_); + } + + void CloseDB() { + delete db_; + db_ = NULL; + } + + void DeleteAllData() { + Iterator* iter = db_->NewIterator(ReadOptions()); + WriteOptions options; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(db_->Delete(WriteOptions(), iter->key())); + } + + delete iter; + } + + void ResetDBState(ResetMethod reset_method) { + switch (reset_method) { + case RESET_DROP_UNSYNCED_DATA: + ASSERT_OK(env_->DropUnsyncedFileData()); + break; + case RESET_DELETE_UNSYNCED_FILES: + ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync()); + break; + default: + assert(false); + } + } + + void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) { + DeleteAllData(); + Build(0, num_pre_sync); + db_->CompactRange(NULL, NULL); + Build(num_pre_sync, num_post_sync); + } + + void PartialCompactTestReopenWithFault(ResetMethod reset_method, + int num_pre_sync, + int num_post_sync) { + env_->SetFilesystemActive(false); + CloseDB(); + ResetDBState(reset_method); + ASSERT_OK(OpenDB()); + ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR)); + ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR)); + } + + void NoWriteTestPreFault() { + } + + void NoWriteTestReopenWithFault(ResetMethod reset_method) { + CloseDB(); + ResetDBState(reset_method); + ASSERT_OK(OpenDB()); + } + + void DoTest() { + Random rnd(0); + ASSERT_OK(OpenDB()); + for (size_t idx = 0; idx < kNumIterations; idx++) { + int num_pre_sync = rnd.Uniform(kMaxNumValues); + int num_post_sync = rnd.Uniform(kMaxNumValues); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, + num_pre_sync, + num_post_sync); + + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + // No new files created so we expect all values since no files will be + // dropped. + PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES, + num_pre_sync + num_post_sync, + 0); + + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES); + } + } +}; + +TEST(FaultInjectionTest, FaultTestNoLogReuse) { + ReuseLogs(false); + DoTest(); +} + +TEST(FaultInjectionTest, FaultTestWithLogReuse) { + ReuseLogs(true); + DoTest(); +} + +} // namespace leveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/src/leveldb/db/leveldb_main.cc b/src/leveldb/db/leveldbutil.cc index 9f4b7dd70c..9f4b7dd70c 100644 --- a/src/leveldb/db/leveldb_main.cc +++ b/src/leveldb/db/leveldbutil.cc diff --git a/src/leveldb/db/log_reader.cc b/src/leveldb/db/log_reader.cc index e44b66c85b..a6d304545d 100644 --- a/src/leveldb/db/log_reader.cc +++ b/src/leveldb/db/log_reader.cc @@ -25,7 +25,8 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, eof_(false), last_record_offset_(0), end_of_buffer_offset_(0), - initial_offset_(initial_offset) { + initial_offset_(initial_offset), + resyncing_(initial_offset > 0) { } Reader::~Reader() { @@ -72,8 +73,25 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { Slice fragment; while (true) { - uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size(); const unsigned int record_type = ReadPhysicalRecord(&fragment); + + // ReadPhysicalRecord may have only had an empty trailer remaining in its + // internal buffer. Calculate the offset of the next physical record now + // that it has returned, properly accounting for its header size. + uint64_t physical_record_offset = + end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size(); + + if (resyncing_) { + if (record_type == kMiddleType) { + continue; + } else if (record_type == kLastType) { + resyncing_ = false; + continue; + } else { + resyncing_ = false; + } + } + switch (record_type) { case kFullType: if (in_fragmented_record) { diff --git a/src/leveldb/db/log_reader.h b/src/leveldb/db/log_reader.h index 6aff791716..8389d61f8f 100644 --- a/src/leveldb/db/log_reader.h +++ b/src/leveldb/db/log_reader.h @@ -73,6 +73,11 @@ class Reader { // Offset at which to start looking for the first record to return uint64_t const initial_offset_; + // True if we are resynchronizing after a seek (initial_offset_ > 0). In + // particular, a run of kMiddleType and kLastType records can be silently + // skipped in this mode + bool resyncing_; + // Extend record types with the following special values enum { kEof = kMaxRecordType + 1, diff --git a/src/leveldb/db/log_test.cc b/src/leveldb/db/log_test.cc index dcf0562652..48a5928657 100644 --- a/src/leveldb/db/log_test.cc +++ b/src/leveldb/db/log_test.cc @@ -79,7 +79,7 @@ class LogTest { virtual Status Skip(uint64_t n) { if (n > contents_.size()) { contents_.clear(); - return Status::NotFound("in-memory file skipepd past end"); + return Status::NotFound("in-memory file skipped past end"); } contents_.remove_prefix(n); @@ -104,23 +104,34 @@ class LogTest { StringSource source_; ReportCollector report_; bool reading_; - Writer writer_; - Reader reader_; + Writer* writer_; + Reader* reader_; // Record metadata for testing initial offset functionality static size_t initial_offset_record_sizes_[]; static uint64_t initial_offset_last_record_offsets_[]; + static int num_initial_offset_records_; public: LogTest() : reading_(false), - writer_(&dest_), - reader_(&source_, &report_, true/*checksum*/, - 0/*initial_offset*/) { + writer_(new Writer(&dest_)), + reader_(new Reader(&source_, &report_, true/*checksum*/, + 0/*initial_offset*/)) { + } + + ~LogTest() { + delete writer_; + delete reader_; + } + + void ReopenForAppend() { + delete writer_; + writer_ = new Writer(&dest_, dest_.contents_.size()); } void Write(const std::string& msg) { ASSERT_TRUE(!reading_) << "Write() after starting to read"; - writer_.AddRecord(Slice(msg)); + writer_->AddRecord(Slice(msg)); } size_t WrittenBytes() const { @@ -134,7 +145,7 @@ class LogTest { } std::string scratch; Slice record; - if (reader_.ReadRecord(&record, &scratch)) { + if (reader_->ReadRecord(&record, &scratch)) { return record.ToString(); } else { return "EOF"; @@ -182,13 +193,18 @@ class LogTest { } void WriteInitialOffsetLog() { - for (int i = 0; i < 4; i++) { + for (int i = 0; i < num_initial_offset_records_; i++) { std::string record(initial_offset_record_sizes_[i], static_cast<char>('a' + i)); Write(record); } } + void StartReadingAt(uint64_t initial_offset) { + delete reader_; + reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset); + } + void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) { WriteInitialOffsetLog(); reading_ = true; @@ -208,32 +224,48 @@ class LogTest { source_.contents_ = Slice(dest_.contents_); Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, initial_offset); - Slice record; - std::string scratch; - ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch)); - ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset], - record.size()); - ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset], - offset_reader->LastRecordOffset()); - ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]); + + // Read all records from expected_record_offset through the last one. + ASSERT_LT(expected_record_offset, num_initial_offset_records_); + for (; expected_record_offset < num_initial_offset_records_; + ++expected_record_offset) { + Slice record; + std::string scratch; + ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch)); + ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset], + record.size()); + ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset], + offset_reader->LastRecordOffset()); + ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]); + } delete offset_reader; } - }; size_t LogTest::initial_offset_record_sizes_[] = {10000, // Two sizable records in first block 10000, 2 * log::kBlockSize - 1000, // Span three blocks - 1}; + 1, + 13716, // Consume all but two bytes of block 3. + log::kBlockSize - kHeaderSize, // Consume the entirety of block 4. + }; uint64_t LogTest::initial_offset_last_record_offsets_[] = {0, kHeaderSize + 10000, 2 * (kHeaderSize + 10000), 2 * (kHeaderSize + 10000) + - (2 * log::kBlockSize - 1000) + 3 * kHeaderSize}; + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, + 2 * (kHeaderSize + 10000) + + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize + + kHeaderSize + 1, + 3 * log::kBlockSize, + }; +// LogTest::initial_offset_last_record_offsets_ must be defined before this. +int LogTest::num_initial_offset_records_ = + sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t); TEST(LogTest, Empty) { ASSERT_EQ("EOF", Read()); @@ -318,6 +350,15 @@ TEST(LogTest, AlignedEof) { ASSERT_EQ("EOF", Read()); } +TEST(LogTest, OpenForAppend) { + Write("hello"); + ReopenForAppend(); + Write("world"); + ASSERT_EQ("hello", Read()); + ASSERT_EQ("world", Read()); + ASSERT_EQ("EOF", Read()); +} + TEST(LogTest, RandomRead) { const int N = 500; Random write_rnd(301); @@ -445,6 +486,22 @@ TEST(LogTest, PartialLastIsIgnored) { ASSERT_EQ(0, DroppedBytes()); } +TEST(LogTest, SkipIntoMultiRecord) { + // Consider a fragmented record: + // first(R1), middle(R1), last(R1), first(R2) + // If initial_offset points to a record after first(R1) but before first(R2) + // incomplete fragment errors are not actual errors, and must be suppressed + // until a new first or full record is encountered. + Write(BigString("foo", 3*kBlockSize)); + Write("correct"); + StartReadingAt(kBlockSize); + + ASSERT_EQ("correct", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("EOF", Read()); +} + TEST(LogTest, ErrorJoinsRecords) { // Consider two fragmented records: // first(R1) last(R1) first(R2) last(R2) @@ -514,6 +571,10 @@ TEST(LogTest, ReadFourthStart) { 3); } +TEST(LogTest, ReadInitialOffsetIntoBlockPadding) { + CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5); +} + TEST(LogTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); } diff --git a/src/leveldb/db/log_writer.cc b/src/leveldb/db/log_writer.cc index 2da99ac088..74a03270da 100644 --- a/src/leveldb/db/log_writer.cc +++ b/src/leveldb/db/log_writer.cc @@ -12,15 +12,24 @@ namespace leveldb { namespace log { -Writer::Writer(WritableFile* dest) - : dest_(dest), - block_offset_(0) { +static void InitTypeCrc(uint32_t* type_crc) { for (int i = 0; i <= kMaxRecordType; i++) { char t = static_cast<char>(i); - type_crc_[i] = crc32c::Value(&t, 1); + type_crc[i] = crc32c::Value(&t, 1); } } +Writer::Writer(WritableFile* dest) + : dest_(dest), + block_offset_(0) { + InitTypeCrc(type_crc_); +} + +Writer::Writer(WritableFile* dest, uint64_t dest_length) + : dest_(dest), block_offset_(dest_length % kBlockSize) { + InitTypeCrc(type_crc_); +} + Writer::~Writer() { } diff --git a/src/leveldb/db/log_writer.h b/src/leveldb/db/log_writer.h index a3a954d967..9e7cc4705b 100644 --- a/src/leveldb/db/log_writer.h +++ b/src/leveldb/db/log_writer.h @@ -22,6 +22,12 @@ class Writer { // "*dest" must be initially empty. // "*dest" must remain live while this Writer is in use. explicit Writer(WritableFile* dest); + + // Create a writer that will append data to "*dest". + // "*dest" must have initial length "dest_length". + // "*dest" must remain live while this Writer is in use. + Writer(WritableFile* dest, uint64_t dest_length); + ~Writer(); Status AddRecord(const Slice& slice); diff --git a/src/leveldb/db/memtable.h b/src/leveldb/db/memtable.h index 92e90bb099..9f41567cde 100644 --- a/src/leveldb/db/memtable.h +++ b/src/leveldb/db/memtable.h @@ -36,10 +36,7 @@ class MemTable { } // Returns an estimate of the number of bytes of data in use by this - // data structure. - // - // REQUIRES: external synchronization to prevent simultaneous - // operations on the same MemTable. + // data structure. It is safe to call when MemTable is being modified. size_t ApproximateMemoryUsage(); // Return an iterator that yields the contents of the memtable. diff --git a/src/leveldb/db/recovery_test.cc b/src/leveldb/db/recovery_test.cc new file mode 100644 index 0000000000..9596f4288a --- /dev/null +++ b/src/leveldb/db/recovery_test.cc @@ -0,0 +1,324 @@ +// Copyright (c) 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_impl.h" +#include "db/filename.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" +#include "util/testharness.h" +#include "util/testutil.h" + +namespace leveldb { + +class RecoveryTest { + public: + RecoveryTest() : env_(Env::Default()), db_(NULL) { + dbname_ = test::TmpDir() + "/recovery_test"; + DestroyDB(dbname_, Options()); + Open(); + } + + ~RecoveryTest() { + Close(); + DestroyDB(dbname_, Options()); + } + + DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); } + Env* env() const { return env_; } + + bool CanAppend() { + WritableFile* tmp; + Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp); + delete tmp; + if (s.IsNotSupportedError()) { + return false; + } else { + return true; + } + } + + void Close() { + delete db_; + db_ = NULL; + } + + void Open(Options* options = NULL) { + Close(); + Options opts; + if (options != NULL) { + opts = *options; + } else { + opts.reuse_logs = true; // TODO(sanjay): test both ways + opts.create_if_missing = true; + } + if (opts.env == NULL) { + opts.env = env_; + } + ASSERT_OK(DB::Open(opts, dbname_, &db_)); + ASSERT_EQ(1, NumLogs()); + } + + Status Put(const std::string& k, const std::string& v) { + return db_->Put(WriteOptions(), k, v); + } + + std::string Get(const std::string& k, const Snapshot* snapshot = NULL) { + std::string result; + Status s = db_->Get(ReadOptions(), k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + std::string ManifestFileName() { + std::string current; + ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), ¤t)); + size_t len = current.size(); + if (len > 0 && current[len-1] == '\n') { + current.resize(len - 1); + } + return dbname_ + "/" + current; + } + + std::string LogName(uint64_t number) { + return LogFileName(dbname_, number); + } + + size_t DeleteLogFiles() { + std::vector<uint64_t> logs = GetFiles(kLogFile); + for (size_t i = 0; i < logs.size(); i++) { + ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]); + } + return logs.size(); + } + + uint64_t FirstLogFile() { + return GetFiles(kLogFile)[0]; + } + + std::vector<uint64_t> GetFiles(FileType t) { + std::vector<std::string> filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + std::vector<uint64_t> result; + for (size_t i = 0; i < filenames.size(); i++) { + uint64_t number; + FileType type; + if (ParseFileName(filenames[i], &number, &type) && type == t) { + result.push_back(number); + } + } + return result; + } + + int NumLogs() { + return GetFiles(kLogFile).size(); + } + + int NumTables() { + return GetFiles(kTableFile).size(); + } + + uint64_t FileSize(const std::string& fname) { + uint64_t result; + ASSERT_OK(env_->GetFileSize(fname, &result)) << fname; + return result; + } + + void CompactMemTable() { + dbfull()->TEST_CompactMemTable(); + } + + // Directly construct a log file that sets key to val. + void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) { + std::string fname = LogFileName(dbname_, lognum); + WritableFile* file; + ASSERT_OK(env_->NewWritableFile(fname, &file)); + log::Writer writer(file); + WriteBatch batch; + batch.Put(key, val); + WriteBatchInternal::SetSequence(&batch, seq); + ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); + ASSERT_OK(file->Flush()); + delete file; + } + + private: + std::string dbname_; + Env* env_; + DB* db_; +}; + +TEST(RecoveryTest, ManifestReused) { + if (!CanAppend()) { + fprintf(stderr, "skipping test because env does not support appending\n"); + return; + } + ASSERT_OK(Put("foo", "bar")); + Close(); + std::string old_manifest = ManifestFileName(); + Open(); + ASSERT_EQ(old_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); + Open(); + ASSERT_EQ(old_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); +} + +TEST(RecoveryTest, LargeManifestCompacted) { + if (!CanAppend()) { + fprintf(stderr, "skipping test because env does not support appending\n"); + return; + } + ASSERT_OK(Put("foo", "bar")); + Close(); + std::string old_manifest = ManifestFileName(); + + // Pad with zeroes to make manifest file very big. + { + uint64_t len = FileSize(old_manifest); + WritableFile* file; + ASSERT_OK(env()->NewAppendableFile(old_manifest, &file)); + std::string zeroes(3*1048576 - static_cast<size_t>(len), 0); + ASSERT_OK(file->Append(zeroes)); + ASSERT_OK(file->Flush()); + delete file; + } + + Open(); + std::string new_manifest = ManifestFileName(); + ASSERT_NE(old_manifest, new_manifest); + ASSERT_GT(10000, FileSize(new_manifest)); + ASSERT_EQ("bar", Get("foo")); + + Open(); + ASSERT_EQ(new_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); +} + +TEST(RecoveryTest, NoLogFiles) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ(1, DeleteLogFiles()); + Open(); + ASSERT_EQ("NOT_FOUND", Get("foo")); + Open(); + ASSERT_EQ("NOT_FOUND", Get("foo")); +} + +TEST(RecoveryTest, LogFileReuse) { + if (!CanAppend()) { + fprintf(stderr, "skipping test because env does not support appending\n"); + return; + } + for (int i = 0; i < 2; i++) { + ASSERT_OK(Put("foo", "bar")); + if (i == 0) { + // Compact to ensure current log is empty + CompactMemTable(); + } + Close(); + ASSERT_EQ(1, NumLogs()); + uint64_t number = FirstLogFile(); + if (i == 0) { + ASSERT_EQ(0, FileSize(LogName(number))); + } else { + ASSERT_LT(0, FileSize(LogName(number))); + } + Open(); + ASSERT_EQ(1, NumLogs()); + ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; + ASSERT_EQ("bar", Get("foo")); + Open(); + ASSERT_EQ(1, NumLogs()); + ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; + ASSERT_EQ("bar", Get("foo")); + } +} + +TEST(RecoveryTest, MultipleMemTables) { + // Make a large log. + const int kNum = 1000; + for (int i = 0; i < kNum; i++) { + char buf[100]; + snprintf(buf, sizeof(buf), "%050d", i); + ASSERT_OK(Put(buf, buf)); + } + ASSERT_EQ(0, NumTables()); + Close(); + ASSERT_EQ(0, NumTables()); + ASSERT_EQ(1, NumLogs()); + uint64_t old_log_file = FirstLogFile(); + + // Force creation of multiple memtables by reducing the write buffer size. + Options opt; + opt.reuse_logs = true; + opt.write_buffer_size = (kNum*100) / 2; + Open(&opt); + ASSERT_LE(2, NumTables()); + ASSERT_EQ(1, NumLogs()); + ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log"; + for (int i = 0; i < kNum; i++) { + char buf[100]; + snprintf(buf, sizeof(buf), "%050d", i); + ASSERT_EQ(buf, Get(buf)); + } +} + +TEST(RecoveryTest, MultipleLogFiles) { + ASSERT_OK(Put("foo", "bar")); + Close(); + ASSERT_EQ(1, NumLogs()); + + // Make a bunch of uncompacted log files. + uint64_t old_log = FirstLogFile(); + MakeLogFile(old_log+1, 1000, "hello", "world"); + MakeLogFile(old_log+2, 1001, "hi", "there"); + MakeLogFile(old_log+3, 1002, "foo", "bar2"); + + // Recover and check that all log files were processed. + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + uint64_t new_log = FirstLogFile(); + ASSERT_LE(old_log+3, new_log); + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); + + // Test that previous recovery produced recoverable state. + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + if (CanAppend()) { + ASSERT_EQ(new_log, FirstLogFile()); + } + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); + + // Check that introducing an older log file does not cause it to be re-read. + Close(); + MakeLogFile(old_log+1, 2000, "hello", "stale write"); + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + if (CanAppend()) { + ASSERT_EQ(new_log, FirstLogFile()); + } + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); +} + +} // namespace leveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/src/leveldb/db/skiplist.h b/src/leveldb/db/skiplist.h index ed8b092203..8bd77764d8 100644 --- a/src/leveldb/db/skiplist.h +++ b/src/leveldb/db/skiplist.h @@ -1,10 +1,10 @@ -#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_ -#define STORAGE_LEVELDB_DB_SKIPLIST_H_ - // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -// + +#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_ +#define STORAGE_LEVELDB_DB_SKIPLIST_H_ + // Thread safety // ------------- // diff --git a/src/leveldb/db/skiplist_test.cc b/src/leveldb/db/skiplist_test.cc index c78f4b4fb1..aee1461e1b 100644 --- a/src/leveldb/db/skiplist_test.cc +++ b/src/leveldb/db/skiplist_test.cc @@ -250,7 +250,7 @@ class ConcurrentTest { // Note that generation 0 is never inserted, so it is ok if // <*,0,*> is missing. ASSERT_TRUE((gen(pos) == 0) || - (gen(pos) > initial_state.Get(key(pos))) + (gen(pos) > static_cast<Key>(initial_state.Get(key(pos)))) ) << "key: " << key(pos) << "; gen: " << gen(pos) << "; initgen: " diff --git a/src/leveldb/db/snapshot.h b/src/leveldb/db/snapshot.h index e7f8fd2c37..6ed413c42d 100644 --- a/src/leveldb/db/snapshot.h +++ b/src/leveldb/db/snapshot.h @@ -5,6 +5,7 @@ #ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_ #define STORAGE_LEVELDB_DB_SNAPSHOT_H_ +#include "db/dbformat.h" #include "leveldb/db.h" namespace leveldb { diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index aa83df55e4..a5e0f77a6a 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -893,7 +893,7 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { return s; } -Status VersionSet::Recover() { +Status VersionSet::Recover(bool *save_manifest) { struct LogReporter : public log::Reader::Reporter { Status* status; virtual void Corruption(size_t bytes, const Status& s) { @@ -1003,11 +1003,49 @@ Status VersionSet::Recover() { last_sequence_ = last_sequence; log_number_ = log_number; prev_log_number_ = prev_log_number; + + // See if we can reuse the existing MANIFEST file. + if (ReuseManifest(dscname, current)) { + // No need to save new manifest + } else { + *save_manifest = true; + } } return s; } +bool VersionSet::ReuseManifest(const std::string& dscname, + const std::string& dscbase) { + if (!options_->reuse_logs) { + return false; + } + FileType manifest_type; + uint64_t manifest_number; + uint64_t manifest_size; + if (!ParseFileName(dscbase, &manifest_number, &manifest_type) || + manifest_type != kDescriptorFile || + !env_->GetFileSize(dscname, &manifest_size).ok() || + // Make new compacted MANIFEST if old one is too big + manifest_size >= kTargetFileSize) { + return false; + } + + assert(descriptor_file_ == NULL); + assert(descriptor_log_ == NULL); + Status r = env_->NewAppendableFile(dscname, &descriptor_file_); + if (!r.ok()) { + Log(options_->info_log, "Reuse MANIFEST: %s\n", r.ToString().c_str()); + assert(descriptor_file_ == NULL); + return false; + } + + Log(options_->info_log, "Reusing MANIFEST %s\n", dscname.c_str()); + descriptor_log_ = new log::Writer(descriptor_file_, manifest_size); + manifest_file_number_ = manifest_number; + return true; +} + void VersionSet::MarkFileNumberUsed(uint64_t number) { if (next_file_number_ <= number) { next_file_number_ = number + 1; diff --git a/src/leveldb/db/version_set.h b/src/leveldb/db/version_set.h index 8dc14b8e01..1dec745673 100644 --- a/src/leveldb/db/version_set.h +++ b/src/leveldb/db/version_set.h @@ -179,7 +179,7 @@ class VersionSet { EXCLUSIVE_LOCKS_REQUIRED(mu); // Recover the last saved descriptor from persistent storage. - Status Recover(); + Status Recover(bool *save_manifest); // Return the current version. Version* current() const { return current_; } @@ -274,6 +274,8 @@ class VersionSet { friend class Compaction; friend class Version; + bool ReuseManifest(const std::string& dscname, const std::string& dscbase); + void Finalize(Version* v); void GetRange(const std::vector<FileMetaData*>& inputs, diff --git a/src/leveldb/db/write_batch_internal.h b/src/leveldb/db/write_batch_internal.h index 310a3c8912..9448ef7b21 100644 --- a/src/leveldb/db/write_batch_internal.h +++ b/src/leveldb/db/write_batch_internal.h @@ -5,6 +5,7 @@ #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ +#include "db/dbformat.h" #include "leveldb/write_batch.h" namespace leveldb { diff --git a/src/leveldb/doc/index.html b/src/leveldb/doc/index.html index 3ed0ed9d9e..2155192581 100644 --- a/src/leveldb/doc/index.html +++ b/src/leveldb/doc/index.html @@ -22,7 +22,7 @@ directory. The following example shows how to open a database, creating it if necessary: <p> <pre> - #include <assert> + #include <cassert> #include "leveldb/db.h" leveldb::DB* db; diff --git a/src/leveldb/helpers/memenv/memenv.cc b/src/leveldb/helpers/memenv/memenv.cc index 43ef2e0729..9a98884daf 100644 --- a/src/leveldb/helpers/memenv/memenv.cc +++ b/src/leveldb/helpers/memenv/memenv.cc @@ -277,6 +277,19 @@ class InMemoryEnv : public EnvWrapper { return Status::OK(); } + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result) { + MutexLock lock(&mutex_); + FileState** sptr = &file_map_[fname]; + FileState* file = *sptr; + if (file == NULL) { + file = new FileState(); + file->Ref(); + } + *result = new WritableFileImpl(file); + return Status::OK(); + } + virtual bool FileExists(const std::string& fname) { MutexLock lock(&mutex_); return file_map_.find(fname) != file_map_.end(); diff --git a/src/leveldb/helpers/memenv/memenv_test.cc b/src/leveldb/helpers/memenv/memenv_test.cc index a44310fed8..5cff77613f 100644 --- a/src/leveldb/helpers/memenv/memenv_test.cc +++ b/src/leveldb/helpers/memenv/memenv_test.cc @@ -40,6 +40,8 @@ TEST(MemEnvTest, Basics) { // Create a file. ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(0, file_size); delete writable_file; // Check that the file exists. @@ -55,9 +57,16 @@ TEST(MemEnvTest, Basics) { ASSERT_OK(writable_file->Append("abc")); delete writable_file; - // Check for expected size. + // Check that append works. + ASSERT_OK(env_->NewAppendableFile("/dir/f", &writable_file)); ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); ASSERT_EQ(3, file_size); + ASSERT_OK(writable_file->Append("hello")); + delete writable_file; + + // Check for expected size. + ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(8, file_size); // Check that renaming works. ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); @@ -65,7 +74,7 @@ TEST(MemEnvTest, Basics) { ASSERT_TRUE(!env_->FileExists("/dir/f")); ASSERT_TRUE(env_->FileExists("/dir/g")); ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); - ASSERT_EQ(3, file_size); + ASSERT_EQ(8, file_size); // Check that opening non-existent file fails. SequentialFile* seq_file; diff --git a/src/leveldb/include/leveldb/cache.h b/src/leveldb/include/leveldb/cache.h index 1a201e5e0a..6819d5bc49 100644 --- a/src/leveldb/include/leveldb/cache.h +++ b/src/leveldb/include/leveldb/cache.h @@ -81,6 +81,17 @@ class Cache { // its cache keys. virtual uint64_t NewId() = 0; + // Remove all cache entries that are not actively in use. Memory-constrained + // applications may wish to call this method to reduce memory usage. + // Default implementation of Prune() does nothing. Subclasses are strongly + // encouraged to override the default implementation. A future release of + // leveldb may change Prune() to a pure abstract method. + virtual void Prune() {} + + // Return an estimate of the combined charges of all elements stored in the + // cache. + virtual size_t TotalCharge() const = 0; + private: void LRU_Remove(Handle* e); void LRU_Append(Handle* e); diff --git a/src/leveldb/include/leveldb/db.h b/src/leveldb/include/leveldb/db.h index 4c169bf22e..9752cbad51 100644 --- a/src/leveldb/include/leveldb/db.h +++ b/src/leveldb/include/leveldb/db.h @@ -14,7 +14,7 @@ namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 18; +static const int kMinorVersion = 19; struct Options; struct ReadOptions; @@ -115,6 +115,8 @@ class DB { // about the internal operation of the DB. // "leveldb.sstables" - returns a multi-line string that describes all // of the sstables that make up the db contents. + // "leveldb.approximate-memory-usage" - returns the approximate number of + // bytes of memory in use by the DB. virtual bool GetProperty(const Slice& property, std::string* value) = 0; // For each i in [0,n-1], store in "sizes[i]", the approximate diff --git a/src/leveldb/include/leveldb/env.h b/src/leveldb/include/leveldb/env.h index f709514da6..99b6c21414 100644 --- a/src/leveldb/include/leveldb/env.h +++ b/src/leveldb/include/leveldb/env.h @@ -69,6 +69,21 @@ class Env { virtual Status NewWritableFile(const std::string& fname, WritableFile** result) = 0; + // Create an object that either appends to an existing file, or + // writes to a new file (if the file does not exist to begin with). + // On success, stores a pointer to the new file in *result and + // returns OK. On failure stores NULL in *result and returns + // non-OK. + // + // The returned file will only be accessed by one thread at a time. + // + // May return an IsNotSupportedError error if this Env does + // not allow appending to an existing file. Users of Env (including + // the leveldb implementation) must be prepared to deal with + // an Env that does not support appending. + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result); + // Returns true iff the named file exists. virtual bool FileExists(const std::string& fname) = 0; @@ -289,6 +304,9 @@ class EnvWrapper : public Env { Status NewWritableFile(const std::string& f, WritableFile** r) { return target_->NewWritableFile(f, r); } + Status NewAppendableFile(const std::string& f, WritableFile** r) { + return target_->NewAppendableFile(f, r); + } bool FileExists(const std::string& f) { return target_->FileExists(f); } Status GetChildren(const std::string& dir, std::vector<std::string>* r) { return target_->GetChildren(dir, r); diff --git a/src/leveldb/include/leveldb/iterator.h b/src/leveldb/include/leveldb/iterator.h index 76aced04bd..da631ed9d8 100644 --- a/src/leveldb/include/leveldb/iterator.h +++ b/src/leveldb/include/leveldb/iterator.h @@ -37,7 +37,7 @@ class Iterator { // Valid() after this call iff the source is not empty. virtual void SeekToLast() = 0; - // Position at the first key in the source that at or past target + // Position at the first key in the source that is at or past target. // The iterator is Valid() after this call iff the source contains // an entry that comes at or past target. virtual void Seek(const Slice& target) = 0; diff --git a/src/leveldb/include/leveldb/options.h b/src/leveldb/include/leveldb/options.h index 7c9b973454..83a1ef39a4 100644 --- a/src/leveldb/include/leveldb/options.h +++ b/src/leveldb/include/leveldb/options.h @@ -128,6 +128,12 @@ struct Options { // efficiently detect that and will switch to uncompressed mode. CompressionType compression; + // EXPERIMENTAL: If true, append to existing MANIFEST and log files + // when a database is opened. This can significantly speed up open. + // + // Default: currently false, but may become true later. + bool reuse_logs; + // If non-NULL, use the specified filter policy to reduce disk reads. // Many applications will benefit from passing the result of // NewBloomFilterPolicy() here. diff --git a/src/leveldb/include/leveldb/status.h b/src/leveldb/include/leveldb/status.h index 11dbd4b47e..d9575f9753 100644 --- a/src/leveldb/include/leveldb/status.h +++ b/src/leveldb/include/leveldb/status.h @@ -60,6 +60,12 @@ class Status { // Returns true iff the status indicates an IOError. bool IsIOError() const { return code() == kIOError; } + // Returns true iff the status indicates a NotSupportedError. + bool IsNotSupportedError() const { return code() == kNotSupported; } + + // Returns true iff the status indicates an InvalidArgument. + bool IsInvalidArgument() const { return code() == kInvalidArgument; } + // Return a string representation of this status suitable for printing. // Returns the string "OK" for success. std::string ToString() const; diff --git a/src/leveldb/port/atomic_pointer.h b/src/leveldb/port/atomic_pointer.h index 9bf091f757..1c4c7aafc6 100644 --- a/src/leveldb/port/atomic_pointer.h +++ b/src/leveldb/port/atomic_pointer.h @@ -35,8 +35,12 @@ #define ARCH_CPU_X86_FAMILY 1 #elif defined(__ARMEL__) #define ARCH_CPU_ARM_FAMILY 1 +#elif defined(__aarch64__) +#define ARCH_CPU_ARM64_FAMILY 1 #elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__) #define ARCH_CPU_PPC_FAMILY 1 +#elif defined(__mips__) +#define ARCH_CPU_MIPS_FAMILY 1 #endif namespace leveldb { @@ -92,6 +96,13 @@ inline void MemoryBarrier() { } #define LEVELDB_HAVE_MEMORY_BARRIER +// ARM64 +#elif defined(ARCH_CPU_ARM64_FAMILY) +inline void MemoryBarrier() { + asm volatile("dmb sy" : : : "memory"); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + // PPC #elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__) inline void MemoryBarrier() { @@ -101,6 +112,13 @@ inline void MemoryBarrier() { } #define LEVELDB_HAVE_MEMORY_BARRIER +// MIPS +#elif defined(ARCH_CPU_MIPS_FAMILY) && defined(__GNUC__) +inline void MemoryBarrier() { + __asm__ __volatile__("sync" : : : "memory"); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + #endif // AtomicPointer built using platform-specific MemoryBarrier() @@ -215,6 +233,7 @@ class AtomicPointer { #undef LEVELDB_HAVE_MEMORY_BARRIER #undef ARCH_CPU_X86_FAMILY #undef ARCH_CPU_ARM_FAMILY +#undef ARCH_CPU_ARM64_FAMILY #undef ARCH_CPU_PPC_FAMILY } // namespace port diff --git a/src/leveldb/port/port_posix.cc b/src/leveldb/port/port_posix.cc index 5ba127a5b9..30e8007ae3 100644 --- a/src/leveldb/port/port_posix.cc +++ b/src/leveldb/port/port_posix.cc @@ -7,7 +7,6 @@ #include <cstdlib> #include <stdio.h> #include <string.h> -#include "util/logging.h" namespace leveldb { namespace port { diff --git a/src/leveldb/table/filter_block.cc b/src/leveldb/table/filter_block.cc index 203e15c8bc..4e78b954f8 100644 --- a/src/leveldb/table/filter_block.cc +++ b/src/leveldb/table/filter_block.cc @@ -68,7 +68,7 @@ void FilterBlockBuilder::GenerateFilter() { // Generate filter for current set of keys and append to result_. filter_offsets_.push_back(result_.size()); - policy_->CreateFilter(&tmp_keys_[0], num_keys, &result_); + policy_->CreateFilter(&tmp_keys_[0], static_cast<int>(num_keys), &result_); tmp_keys_.clear(); keys_.clear(); @@ -97,7 +97,7 @@ bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) { if (index < num_) { uint32_t start = DecodeFixed32(offset_ + index*4); uint32_t limit = DecodeFixed32(offset_ + index*4 + 4); - if (start <= limit && limit <= (offset_ - data_)) { + if (start <= limit && limit <= static_cast<size_t>(offset_ - data_)) { Slice filter = Slice(data_ + start, limit - start); return policy_->KeyMayMatch(key, filter); } else if (start == limit) { diff --git a/src/leveldb/table/format.cc b/src/leveldb/table/format.cc index aa63144c9e..24e4e02445 100644 --- a/src/leveldb/table/format.cc +++ b/src/leveldb/table/format.cc @@ -30,15 +30,14 @@ Status BlockHandle::DecodeFrom(Slice* input) { } void Footer::EncodeTo(std::string* dst) const { -#ifndef NDEBUG const size_t original_size = dst->size(); -#endif metaindex_handle_.EncodeTo(dst); index_handle_.EncodeTo(dst); dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber & 0xffffffffu)); PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber >> 32)); assert(dst->size() == original_size + kEncodedLength); + (void)original_size; // Disable unused variable warning. } Status Footer::DecodeFrom(Slice* input) { diff --git a/src/leveldb/table/iterator_wrapper.h b/src/leveldb/table/iterator_wrapper.h index 9e16b3dbed..f410c3fabe 100644 --- a/src/leveldb/table/iterator_wrapper.h +++ b/src/leveldb/table/iterator_wrapper.h @@ -5,6 +5,9 @@ #ifndef STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ #define STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ +#include "leveldb/iterator.h" +#include "leveldb/slice.h" + namespace leveldb { // A internal wrapper class with an interface similar to Iterator that diff --git a/src/leveldb/table/table.cc b/src/leveldb/table/table.cc index dff8a82590..decf8082cc 100644 --- a/src/leveldb/table/table.cc +++ b/src/leveldb/table/table.cc @@ -82,7 +82,7 @@ Status Table::Open(const Options& options, *table = new Table(rep); (*table)->ReadMeta(footer); } else { - if (index_block) delete index_block; + delete index_block; } return s; diff --git a/src/leveldb/table/table_test.cc b/src/leveldb/table/table_test.cc index c723bf84cf..abf6e246ff 100644 --- a/src/leveldb/table/table_test.cc +++ b/src/leveldb/table/table_test.cc @@ -853,12 +853,20 @@ TEST(TableTest, ApproximateOffsetOfCompressed) { options.compression = kSnappyCompression; c.Finish(options, &keys, &kvmap); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 2000, 3000)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 2000, 3000)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 4000, 6000)); + // Expected upper and lower bounds of space used by compressible strings. + static const int kSlop = 1000; // Compressor effectiveness varies. + const int expected = 2500; // 10000 * compression ratio (0.25) + const int min_z = expected - kSlop; + const int max_z = expected + kSlop; + + ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, kSlop)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, kSlop)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, kSlop)); + // Have now emitted a large compressible string, so adjust expected offset. + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), min_z, max_z)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), min_z, max_z)); + // Have now emitted two large compressible strings, so adjust expected offset. + ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 2 * min_z, 2 * max_z)); } } // namespace leveldb diff --git a/src/leveldb/util/arena.cc b/src/leveldb/util/arena.cc index 9367f71492..74078213ee 100644 --- a/src/leveldb/util/arena.cc +++ b/src/leveldb/util/arena.cc @@ -9,8 +9,7 @@ namespace leveldb { static const int kBlockSize = 4096; -Arena::Arena() { - blocks_memory_ = 0; +Arena::Arena() : memory_usage_(0) { alloc_ptr_ = NULL; // First allocation will allocate a block alloc_bytes_remaining_ = 0; } @@ -60,8 +59,9 @@ char* Arena::AllocateAligned(size_t bytes) { char* Arena::AllocateNewBlock(size_t block_bytes) { char* result = new char[block_bytes]; - blocks_memory_ += block_bytes; blocks_.push_back(result); + memory_usage_.NoBarrier_Store( + reinterpret_cast<void*>(MemoryUsage() + block_bytes + sizeof(char*))); return result; } diff --git a/src/leveldb/util/arena.h b/src/leveldb/util/arena.h index 73bbf1cb9b..48bab33741 100644 --- a/src/leveldb/util/arena.h +++ b/src/leveldb/util/arena.h @@ -9,6 +9,7 @@ #include <assert.h> #include <stddef.h> #include <stdint.h> +#include "port/port.h" namespace leveldb { @@ -24,10 +25,9 @@ class Arena { char* AllocateAligned(size_t bytes); // Returns an estimate of the total memory usage of data allocated - // by the arena (including space allocated but not yet used for user - // allocations). + // by the arena. size_t MemoryUsage() const { - return blocks_memory_ + blocks_.capacity() * sizeof(char*); + return reinterpret_cast<uintptr_t>(memory_usage_.NoBarrier_Load()); } private: @@ -41,8 +41,8 @@ class Arena { // Array of new[] allocated memory blocks std::vector<char*> blocks_; - // Bytes of memory in blocks allocated so far - size_t blocks_memory_; + // Total memory usage of the arena. + port::AtomicPointer memory_usage_; // No copying allowed Arena(const Arena&); diff --git a/src/leveldb/util/bloom.cc b/src/leveldb/util/bloom.cc index a27a2ace28..bf3e4ca6e9 100644 --- a/src/leveldb/util/bloom.cc +++ b/src/leveldb/util/bloom.cc @@ -47,7 +47,7 @@ class BloomFilterPolicy : public FilterPolicy { dst->resize(init_size + bytes, 0); dst->push_back(static_cast<char>(k_)); // Remember # of probes in filter char* array = &(*dst)[init_size]; - for (size_t i = 0; i < n; i++) { + for (int i = 0; i < n; i++) { // Use double-hashing to generate a sequence of hash values. // See analysis in [Kirsch,Mitzenmacher 2006]. uint32_t h = BloomHash(keys[i]); diff --git a/src/leveldb/util/bloom_test.cc b/src/leveldb/util/bloom_test.cc index 77fb1b3159..1b87a2be3f 100644 --- a/src/leveldb/util/bloom_test.cc +++ b/src/leveldb/util/bloom_test.cc @@ -46,7 +46,8 @@ class BloomTest { key_slices.push_back(Slice(keys_[i])); } filter_.clear(); - policy_->CreateFilter(&key_slices[0], key_slices.size(), &filter_); + policy_->CreateFilter(&key_slices[0], static_cast<int>(key_slices.size()), + &filter_); keys_.clear(); if (kVerbose >= 2) DumpFilter(); } diff --git a/src/leveldb/util/cache.cc b/src/leveldb/util/cache.cc index 8b197bc02a..ce46886171 100644 --- a/src/leveldb/util/cache.cc +++ b/src/leveldb/util/cache.cc @@ -19,6 +19,23 @@ Cache::~Cache() { namespace { // LRU cache implementation +// +// Cache entries have an "in_cache" boolean indicating whether the cache has a +// reference on the entry. The only ways that this can become false without the +// entry being passed to its "deleter" are via Erase(), via Insert() when +// an element with a duplicate key is inserted, or on destruction of the cache. +// +// The cache keeps two linked lists of items in the cache. All items in the +// cache are in one list or the other, and never both. Items still referenced +// by clients but erased from the cache are in neither list. The lists are: +// - in-use: contains the items currently referenced by clients, in no +// particular order. (This list is used for invariant checking. If we +// removed the check, elements that would otherwise be on this list could be +// left as disconnected singleton lists.) +// - LRU: contains the items not currently referenced by clients, in LRU order +// Elements are moved between these lists by the Ref() and Unref() methods, +// when they detect an element in the cache acquiring or losing its only +// external reference. // An entry is a variable length heap-allocated structure. Entries // are kept in a circular doubly linked list ordered by access time. @@ -30,7 +47,8 @@ struct LRUHandle { LRUHandle* prev; size_t charge; // TODO(opt): Only allow uint32_t? size_t key_length; - uint32_t refs; + bool in_cache; // Whether entry is in the cache. + uint32_t refs; // References, including cache reference, if present. uint32_t hash; // Hash of key(); used for fast sharding and comparisons char key_data[1]; // Beginning of key @@ -147,49 +165,77 @@ class LRUCache { Cache::Handle* Lookup(const Slice& key, uint32_t hash); void Release(Cache::Handle* handle); void Erase(const Slice& key, uint32_t hash); + void Prune(); + size_t TotalCharge() const { + MutexLock l(&mutex_); + return usage_; + } private: void LRU_Remove(LRUHandle* e); - void LRU_Append(LRUHandle* e); + void LRU_Append(LRUHandle*list, LRUHandle* e); + void Ref(LRUHandle* e); void Unref(LRUHandle* e); + bool FinishErase(LRUHandle* e); // Initialized before use. size_t capacity_; // mutex_ protects the following state. - port::Mutex mutex_; + mutable port::Mutex mutex_; size_t usage_; // Dummy head of LRU list. // lru.prev is newest entry, lru.next is oldest entry. + // Entries have refs==1 and in_cache==true. LRUHandle lru_; + // Dummy head of in-use list. + // Entries are in use by clients, and have refs >= 2 and in_cache==true. + LRUHandle in_use_; + HandleTable table_; }; LRUCache::LRUCache() : usage_(0) { - // Make empty circular linked list + // Make empty circular linked lists. lru_.next = &lru_; lru_.prev = &lru_; + in_use_.next = &in_use_; + in_use_.prev = &in_use_; } LRUCache::~LRUCache() { + assert(in_use_.next == &in_use_); // Error if caller has an unreleased handle for (LRUHandle* e = lru_.next; e != &lru_; ) { LRUHandle* next = e->next; - assert(e->refs == 1); // Error if caller has an unreleased handle + assert(e->in_cache); + e->in_cache = false; + assert(e->refs == 1); // Invariant of lru_ list. Unref(e); e = next; } } +void LRUCache::Ref(LRUHandle* e) { + if (e->refs == 1 && e->in_cache) { // If on lru_ list, move to in_use_ list. + LRU_Remove(e); + LRU_Append(&in_use_, e); + } + e->refs++; +} + void LRUCache::Unref(LRUHandle* e) { assert(e->refs > 0); e->refs--; - if (e->refs <= 0) { - usage_ -= e->charge; + if (e->refs == 0) { // Deallocate. + assert(!e->in_cache); (*e->deleter)(e->key(), e->value); free(e); + } else if (e->in_cache && e->refs == 1) { // No longer in use; move to lru_ list. + LRU_Remove(e); + LRU_Append(&lru_, e); } } @@ -198,10 +244,10 @@ void LRUCache::LRU_Remove(LRUHandle* e) { e->prev->next = e->next; } -void LRUCache::LRU_Append(LRUHandle* e) { - // Make "e" newest entry by inserting just before lru_ - e->next = &lru_; - e->prev = lru_.prev; +void LRUCache::LRU_Append(LRUHandle* list, LRUHandle* e) { + // Make "e" newest entry by inserting just before *list + e->next = list; + e->prev = list->prev; e->prev->next = e; e->next->prev = e; } @@ -210,9 +256,7 @@ Cache::Handle* LRUCache::Lookup(const Slice& key, uint32_t hash) { MutexLock l(&mutex_); LRUHandle* e = table_.Lookup(key, hash); if (e != NULL) { - e->refs++; - LRU_Remove(e); - LRU_Append(e); + Ref(e); } return reinterpret_cast<Cache::Handle*>(e); } @@ -234,34 +278,58 @@ Cache::Handle* LRUCache::Insert( e->charge = charge; e->key_length = key.size(); e->hash = hash; - e->refs = 2; // One from LRUCache, one for the returned handle + e->in_cache = false; + e->refs = 1; // for the returned handle. memcpy(e->key_data, key.data(), key.size()); - LRU_Append(e); - usage_ += charge; - LRUHandle* old = table_.Insert(e); - if (old != NULL) { - LRU_Remove(old); - Unref(old); - } + if (capacity_ > 0) { + e->refs++; // for the cache's reference. + e->in_cache = true; + LRU_Append(&in_use_, e); + usage_ += charge; + FinishErase(table_.Insert(e)); + } // else don't cache. (Tests use capacity_==0 to turn off caching.) while (usage_ > capacity_ && lru_.next != &lru_) { LRUHandle* old = lru_.next; - LRU_Remove(old); - table_.Remove(old->key(), old->hash); - Unref(old); + assert(old->refs == 1); + bool erased = FinishErase(table_.Remove(old->key(), old->hash)); + if (!erased) { // to avoid unused variable when compiled NDEBUG + assert(erased); + } } return reinterpret_cast<Cache::Handle*>(e); } -void LRUCache::Erase(const Slice& key, uint32_t hash) { - MutexLock l(&mutex_); - LRUHandle* e = table_.Remove(key, hash); +// If e != NULL, finish removing *e from the cache; it has already been removed +// from the hash table. Return whether e != NULL. Requires mutex_ held. +bool LRUCache::FinishErase(LRUHandle* e) { if (e != NULL) { + assert(e->in_cache); LRU_Remove(e); + e->in_cache = false; + usage_ -= e->charge; Unref(e); } + return e != NULL; +} + +void LRUCache::Erase(const Slice& key, uint32_t hash) { + MutexLock l(&mutex_); + FinishErase(table_.Remove(key, hash)); +} + +void LRUCache::Prune() { + MutexLock l(&mutex_); + while (lru_.next != &lru_) { + LRUHandle* e = lru_.next; + assert(e->refs == 1); + bool erased = FinishErase(table_.Remove(e->key(), e->hash)); + if (!erased) { // to avoid unused variable when compiled NDEBUG + assert(erased); + } + } } static const int kNumShardBits = 4; @@ -314,6 +382,18 @@ class ShardedLRUCache : public Cache { MutexLock l(&id_mutex_); return ++(last_id_); } + virtual void Prune() { + for (int s = 0; s < kNumShards; s++) { + shard_[s].Prune(); + } + } + virtual size_t TotalCharge() const { + size_t total = 0; + for (int s = 0; s < kNumShards; s++) { + total += shard_[s].TotalCharge(); + } + return total; + } }; } // end anonymous namespace diff --git a/src/leveldb/util/cache_test.cc b/src/leveldb/util/cache_test.cc index 43716715a8..468f7a6425 100644 --- a/src/leveldb/util/cache_test.cc +++ b/src/leveldb/util/cache_test.cc @@ -59,6 +59,11 @@ class CacheTest { &CacheTest::Deleter)); } + Cache::Handle* InsertAndReturnHandle(int key, int value, int charge = 1) { + return cache_->Insert(EncodeKey(key), EncodeValue(value), charge, + &CacheTest::Deleter); + } + void Erase(int key) { cache_->Erase(EncodeKey(key)); } @@ -135,8 +140,11 @@ TEST(CacheTest, EntriesArePinned) { TEST(CacheTest, EvictionPolicy) { Insert(100, 101); Insert(200, 201); + Insert(300, 301); + Cache::Handle* h = cache_->Lookup(EncodeKey(300)); - // Frequently used entry must be kept around + // Frequently used entry must be kept around, + // as must things that are still in use. for (int i = 0; i < kCacheSize + 100; i++) { Insert(1000+i, 2000+i); ASSERT_EQ(2000+i, Lookup(1000+i)); @@ -144,6 +152,25 @@ TEST(CacheTest, EvictionPolicy) { } ASSERT_EQ(101, Lookup(100)); ASSERT_EQ(-1, Lookup(200)); + ASSERT_EQ(301, Lookup(300)); + cache_->Release(h); +} + +TEST(CacheTest, UseExceedsCacheSize) { + // Overfill the cache, keeping handles on all inserted entries. + std::vector<Cache::Handle*> h; + for (int i = 0; i < kCacheSize + 100; i++) { + h.push_back(InsertAndReturnHandle(1000+i, 2000+i)); + } + + // Check that all the entries can be found in the cache. + for (int i = 0; i < h.size(); i++) { + ASSERT_EQ(2000+i, Lookup(1000+i)); + } + + for (int i = 0; i < h.size(); i++) { + cache_->Release(h[i]); + } } TEST(CacheTest, HeavyEntries) { @@ -179,6 +206,19 @@ TEST(CacheTest, NewId) { ASSERT_NE(a, b); } +TEST(CacheTest, Prune) { + Insert(1, 100); + Insert(2, 200); + + Cache::Handle* handle = cache_->Lookup(EncodeKey(1)); + ASSERT_TRUE(handle); + cache_->Prune(); + cache_->Release(handle); + + ASSERT_EQ(100, Lookup(1)); + ASSERT_EQ(-1, Lookup(2)); +} + } // namespace leveldb int main(int argc, char** argv) { diff --git a/src/leveldb/util/env.cc b/src/leveldb/util/env.cc index c2600e964a..c58a0821ef 100644 --- a/src/leveldb/util/env.cc +++ b/src/leveldb/util/env.cc @@ -9,6 +9,10 @@ namespace leveldb { Env::~Env() { } +Status Env::NewAppendableFile(const std::string& fname, WritableFile** result) { + return Status::NotSupported("NewAppendableFile", fname); +} + SequentialFile::~SequentialFile() { } diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index ba2667864a..e0fca52f46 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -351,6 +351,19 @@ class PosixEnv : public Env { return s; } + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result) { + Status s; + FILE* f = fopen(fname.c_str(), "a"); + if (f == NULL) { + *result = NULL; + s = IOError(fname, errno); + } else { + *result = new PosixWritableFile(fname, f); + } + return s; + } + virtual bool FileExists(const std::string& fname) { return access(fname.c_str(), F_OK) == 0; } diff --git a/src/leveldb/util/env_win.cc b/src/leveldb/util/env_win.cc index e11a96b791..b074b7579e 100644 --- a/src/leveldb/util/env_win.cc +++ b/src/leveldb/util/env_win.cc @@ -106,7 +106,7 @@ private: class Win32WritableFile : public WritableFile { public: - Win32WritableFile(const std::string& fname); + Win32WritableFile(const std::string& fname, bool append); ~Win32WritableFile(); virtual Status Append(const Slice& data); @@ -158,6 +158,8 @@ public: RandomAccessFile** result); virtual Status NewWritableFile(const std::string& fname, WritableFile** result); + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result); virtual bool FileExists(const std::string& fname); @@ -423,17 +425,23 @@ void Win32RandomAccessFile::_CleanUp() } } -Win32WritableFile::Win32WritableFile(const std::string& fname) +Win32WritableFile::Win32WritableFile(const std::string& fname, bool append) : filename_(fname) { std::wstring path; ToWidePath(fname, path); - DWORD Flag = PathFileExistsW(path.c_str()) ? OPEN_EXISTING : CREATE_ALWAYS; + // NewAppendableFile: append to an existing file, or create a new one + // if none exists - this is OPEN_ALWAYS behavior, with + // FILE_APPEND_DATA to avoid having to manually position the file + // pointer at the end of the file. + // NewWritableFile: create a new file, delete if it exists - this is + // CREATE_ALWAYS behavior. This file is used for writing only so + // use GENERIC_WRITE. _hFile = CreateFileW(path.c_str(), - GENERIC_READ | GENERIC_WRITE, + append ? FILE_APPEND_DATA : GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_DELETE|FILE_SHARE_WRITE, NULL, - Flag, + append ? OPEN_ALWAYS : CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); // CreateFileW returns INVALID_HANDLE_VALUE in case of error, always check isEnable() before use @@ -823,7 +831,9 @@ Status Win32Env::NewLogger( const std::string& fname, Logger** result ) { Status sRet; std::string path = fname; - Win32WritableFile* pMapFile = new Win32WritableFile(ModifyPath(path)); + // Logs are opened with write semantics, not with append semantics + // (see PosixEnv::NewLogger) + Win32WritableFile* pMapFile = new Win32WritableFile(ModifyPath(path), false); if(!pMapFile->isEnable()){ delete pMapFile; *result = NULL; @@ -837,7 +847,20 @@ Status Win32Env::NewWritableFile( const std::string& fname, WritableFile** resul { Status sRet; std::string path = fname; - Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path)); + Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path), false); + if(!pFile->isEnable()){ + *result = NULL; + sRet = Status::IOError(fname,Win32::GetLastErrSz()); + }else + *result = pFile; + return sRet; +} + +Status Win32Env::NewAppendableFile( const std::string& fname, WritableFile** result ) +{ + Status sRet; + std::string path = fname; + Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path), true); if(!pFile->isEnable()){ *result = NULL; sRet = Status::IOError(fname,Win32::GetLastErrSz()); diff --git a/src/leveldb/util/options.cc b/src/leveldb/util/options.cc index 76af5b9302..8b618fb1ae 100644 --- a/src/leveldb/util/options.cc +++ b/src/leveldb/util/options.cc @@ -22,8 +22,8 @@ Options::Options() block_size(4096), block_restart_interval(16), compression(kSnappyCompression), + reuse_logs(false), filter_policy(NULL) { } - } // namespace leveldb diff --git a/src/leveldb/util/testutil.h b/src/leveldb/util/testutil.h index adad3fc1ea..d7e4583702 100644 --- a/src/leveldb/util/testutil.h +++ b/src/leveldb/util/testutil.h @@ -45,6 +45,16 @@ class ErrorEnv : public EnvWrapper { } return target()->NewWritableFile(fname, result); } + + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result) { + if (writable_file_error_) { + ++num_writable_file_errors_; + *result = NULL; + return Status::IOError(fname, "fake error"); + } + return target()->NewAppendableFile(fname, result); + } }; } // namespace test diff --git a/src/miner.cpp b/src/miner.cpp index c40b12cd8e..e80e8a2656 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -13,7 +13,7 @@ #include "consensus/merkle.h" #include "consensus/validation.h" #include "hash.h" -#include "main.h" +#include "validation.h" #include "net.h" #include "policy/policy.h" #include "pow.h" diff --git a/src/net_processing.cpp b/src/net_processing.cpp new file mode 100644 index 0000000000..e10694ed18 --- /dev/null +++ b/src/net_processing.cpp @@ -0,0 +1,3077 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "net_processing.h" + +#include "addrman.h" +#include "arith_uint256.h" +#include "blockencodings.h" +#include "chainparams.h" +#include "consensus/validation.h" +#include "hash.h" +#include "init.h" +#include "validation.h" +#include "merkleblock.h" +#include "net.h" +#include "netmessagemaker.h" +#include "netbase.h" +#include "policy/fees.h" +#include "policy/policy.h" +#include "primitives/block.h" +#include "primitives/transaction.h" +#include "random.h" +#include "tinyformat.h" +#include "txmempool.h" +#include "ui_interface.h" +#include "util.h" +#include "utilmoneystr.h" +#include "utilstrencodings.h" +#include "validationinterface.h" + +#include <boost/thread.hpp> + +using namespace std; + +#if defined(NDEBUG) +# error "Bitcoin cannot be compiled without assertions." +#endif + +int64_t nTimeBestReceived = 0; // Used only to inform the wallet of when we last received a block + +struct IteratorComparator +{ + template<typename I> + bool operator()(const I& a, const I& b) + { + return &(*a) < &(*b); + } +}; + +struct COrphanTx { + // When modifying, adapt the copy of this definition in tests/DoS_tests. + CTransaction tx; + NodeId fromPeer; + int64_t nTimeExpire; +}; +map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main); +map<COutPoint, set<map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); +void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + +static const uint64_t RANDOMIZER_ID_ADDRESS_RELAY = 0x3cac0035b5866b90ULL; // SHA256("main address relay")[0:8] + +// Internal stuff +namespace { + /** Number of nodes with fSyncStarted. */ + int nSyncStarted = 0; + + /** + * Sources of received blocks, saved to be able to send them reject + * messages or ban them when processing happens afterwards. Protected by + * cs_main. + * Set mapBlockSource[hash].second to false if the node should not be + * punished if the block is invalid. + */ + map<uint256, std::pair<NodeId, bool>> mapBlockSource; + + /** + * Filter for transactions that were recently rejected by + * AcceptToMemoryPool. These are not rerequested until the chain tip + * changes, at which point the entire filter is reset. Protected by + * cs_main. + * + * Without this filter we'd be re-requesting txs from each of our peers, + * increasing bandwidth consumption considerably. For instance, with 100 + * peers, half of which relay a tx we don't accept, that might be a 50x + * bandwidth increase. A flooding attacker attempting to roll-over the + * filter using minimum-sized, 60byte, transactions might manage to send + * 1000/sec if we have fast peers, so we pick 120,000 to give our peers a + * two minute window to send invs to us. + * + * Decreasing the false positive rate is fairly cheap, so we pick one in a + * million to make it highly unlikely for users to have issues with this + * filter. + * + * Memory used: 1.3 MB + */ + std::unique_ptr<CRollingBloomFilter> recentRejects; + uint256 hashRecentRejectsChainTip; + + /** Blocks that are in flight, and that are in the queue to be downloaded. Protected by cs_main. */ + struct QueuedBlock { + uint256 hash; + CBlockIndex* pindex; //!< Optional. + bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + std::unique_ptr<PartiallyDownloadedBlock> partialBlock; //!< Optional, used for CMPCTBLOCK downloads + }; + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> > mapBlocksInFlight; + + /** Stack of nodes which we have set to announce using compact blocks */ + list<NodeId> lNodesAnnouncingHeaderAndIDs; + + /** Number of preferable block download peers. */ + int nPreferredDownload = 0; + + /** Number of peers from which we're downloading blocks. */ + int nPeersWithValidatedDownloads = 0; + + /** Relay map, protected by cs_main. */ + typedef std::map<uint256, CTransactionRef> MapRelay; + MapRelay mapRelay; + /** Expiration-time ordered list of (expire time, relay map entry) pairs, protected by cs_main). */ + std::deque<std::pair<int64_t, MapRelay::iterator>> vRelayExpiration; +} // anon namespace + +////////////////////////////////////////////////////////////////////////////// +// +// Registration of network node signals. +// + +namespace { + +struct CBlockReject { + unsigned char chRejectCode; + string strRejectReason; + uint256 hashBlock; +}; + +/** + * Maintain validation-specific state about nodes, protected by cs_main, instead + * by CNode's own locks. This simplifies asynchronous operation, where + * processing of incoming data is done after the ProcessMessage call returns, + * and we're no longer holding the node's locks. + */ +struct CNodeState { + //! The peer's address + const CService address; + //! Whether we have a fully established connection. + bool fCurrentlyConnected; + //! Accumulated misbehaviour score for this peer. + int nMisbehavior; + //! Whether this peer should be disconnected and banned (unless whitelisted). + bool fShouldBan; + //! String name of this peer (debugging/logging purposes). + const std::string name; + //! List of asynchronously-determined block rejections to notify this peer about. + std::vector<CBlockReject> rejects; + //! The best known block we know this peer has announced. + CBlockIndex *pindexBestKnownBlock; + //! The hash of the last unknown block this peer has announced. + uint256 hashLastUnknownBlock; + //! The last full block we both have. + CBlockIndex *pindexLastCommonBlock; + //! The best header we have sent our peer. + CBlockIndex *pindexBestHeaderSent; + //! Length of current-streak of unconnecting headers announcements + int nUnconnectingHeaders; + //! Whether we've started headers synchronization with this peer. + bool fSyncStarted; + //! Since when we're stalling block download progress (in microseconds), or 0. + int64_t nStallingSince; + list<QueuedBlock> vBlocksInFlight; + //! When the first entry in vBlocksInFlight started downloading. Don't care when vBlocksInFlight is empty. + int64_t nDownloadingSince; + int nBlocksInFlight; + int nBlocksInFlightValidHeaders; + //! Whether we consider this a preferred download peer. + bool fPreferredDownload; + //! Whether this peer wants invs or headers (when possible) for block announcements. + bool fPreferHeaders; + //! Whether this peer wants invs or cmpctblocks (when possible) for block announcements. + bool fPreferHeaderAndIDs; + /** + * Whether this peer will send us cmpctblocks if we request them. + * This is not used to gate request logic, as we really only care about fSupportsDesiredCmpctVersion, + * but is used as a flag to "lock in" the version of compact blocks (fWantsCmpctWitness) we send. + */ + bool fProvidesHeaderAndIDs; + //! Whether this peer can give us witnesses + bool fHaveWitness; + //! Whether this peer wants witnesses in cmpctblocks/blocktxns + bool fWantsCmpctWitness; + /** + * If we've announced NODE_WITNESS to this peer: whether the peer sends witnesses in cmpctblocks/blocktxns, + * otherwise: whether this peer sends non-witnesses in cmpctblocks/blocktxns. + */ + bool fSupportsDesiredCmpctVersion; + + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { + fCurrentlyConnected = false; + nMisbehavior = 0; + fShouldBan = false; + pindexBestKnownBlock = NULL; + hashLastUnknownBlock.SetNull(); + pindexLastCommonBlock = NULL; + pindexBestHeaderSent = NULL; + nUnconnectingHeaders = 0; + fSyncStarted = false; + nStallingSince = 0; + nDownloadingSince = 0; + nBlocksInFlight = 0; + nBlocksInFlightValidHeaders = 0; + fPreferredDownload = false; + fPreferHeaders = false; + fPreferHeaderAndIDs = false; + fProvidesHeaderAndIDs = false; + fHaveWitness = false; + fWantsCmpctWitness = false; + fSupportsDesiredCmpctVersion = false; + } +}; + +/** Map maintaining per-node state. Requires cs_main. */ +map<NodeId, CNodeState> mapNodeState; + +// Requires cs_main. +CNodeState *State(NodeId pnode) { + map<NodeId, CNodeState>::iterator it = mapNodeState.find(pnode); + if (it == mapNodeState.end()) + return NULL; + return &it->second; +} + +void UpdatePreferredDownload(CNode* node, CNodeState* state) +{ + nPreferredDownload -= state->fPreferredDownload; + + // Whether this node should be marked as a preferred download node. + state->fPreferredDownload = (!node->fInbound || node->fWhitelisted) && !node->fOneShot && !node->fClient; + + nPreferredDownload += state->fPreferredDownload; +} + +void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) +{ + ServiceFlags nLocalNodeServices = pnode->GetLocalServices(); + uint64_t nonce = pnode->GetLocalNonce(); + int nNodeStartingHeight = pnode->GetMyStartingHeight(); + NodeId nodeid = pnode->GetId(); + CAddress addr = pnode->addr; + + CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); + CAddress addrMe = CAddress(CService(), nLocalNodeServices); + + connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes)); + + if (fLogIPs) + LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + else + LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); +} + +void InitializeNode(CNode *pnode, CConnman& connman) { + CAddress addr = pnode->addr; + std::string addrName = pnode->addrName; + NodeId nodeid = pnode->GetId(); + { + LOCK(cs_main); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); + } + if(!pnode->fInbound) + PushNodeVersion(pnode, connman, GetTime()); +} + +void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { + fUpdateConnectionTime = false; + LOCK(cs_main); + CNodeState *state = State(nodeid); + + if (state->fSyncStarted) + nSyncStarted--; + + if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { + fUpdateConnectionTime = true; + } + + BOOST_FOREACH(const QueuedBlock& entry, state->vBlocksInFlight) { + mapBlocksInFlight.erase(entry.hash); + } + EraseOrphansFor(nodeid); + nPreferredDownload -= state->fPreferredDownload; + nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); + assert(nPeersWithValidatedDownloads >= 0); + + mapNodeState.erase(nodeid); + + if (mapNodeState.empty()) { + // Do a consistency check after the last peer is removed. + assert(mapBlocksInFlight.empty()); + assert(nPreferredDownload == 0); + assert(nPeersWithValidatedDownloads == 0); + } +} + +// Requires cs_main. +// Returns a bool indicating whether we requested this block. +// Also used if a block was /not/ received and timed out or started with another peer +bool MarkBlockAsReceived(const uint256& hash) { + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); + if (itInFlight != mapBlocksInFlight.end()) { + CNodeState *state = State(itInFlight->second.first); + state->nBlocksInFlightValidHeaders -= itInFlight->second.second->fValidatedHeaders; + if (state->nBlocksInFlightValidHeaders == 0 && itInFlight->second.second->fValidatedHeaders) { + // Last validated block on the queue was received. + nPeersWithValidatedDownloads--; + } + if (state->vBlocksInFlight.begin() == itInFlight->second.second) { + // First block on the queue was received, update the start download time for the next one + state->nDownloadingSince = std::max(state->nDownloadingSince, GetTimeMicros()); + } + state->vBlocksInFlight.erase(itInFlight->second.second); + state->nBlocksInFlight--; + state->nStallingSince = 0; + mapBlocksInFlight.erase(itInFlight); + return true; + } + return false; +} + +// Requires cs_main. +// returns false, still setting pit, if the block was already in flight from the same peer +// pit will only be valid as long as the same cs_main lock is being held +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, CBlockIndex *pindex = NULL, list<QueuedBlock>::iterator **pit = NULL) { + CNodeState *state = State(nodeid); + assert(state != NULL); + + // Short-circuit most stuff in case its from the same node + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); + if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { + *pit = &itInFlight->second.second; + return false; + } + + // Make sure it's not listed somewhere already. + MarkBlockAsReceived(hash); + + list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), + {hash, pindex, pindex != NULL, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); + state->nBlocksInFlight++; + state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; + if (state->nBlocksInFlight == 1) { + // We're starting a block download (batch) from this peer. + state->nDownloadingSince = GetTimeMicros(); + } + if (state->nBlocksInFlightValidHeaders == 1 && pindex != NULL) { + nPeersWithValidatedDownloads++; + } + itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; + if (pit) + *pit = &itInFlight->second.second; + return true; +} + +/** Check whether the last unknown block a peer advertised is not yet known. */ +void ProcessBlockAvailability(NodeId nodeid) { + CNodeState *state = State(nodeid); + assert(state != NULL); + + if (!state->hashLastUnknownBlock.IsNull()) { + BlockMap::iterator itOld = mapBlockIndex.find(state->hashLastUnknownBlock); + if (itOld != mapBlockIndex.end() && itOld->second->nChainWork > 0) { + if (state->pindexBestKnownBlock == NULL || itOld->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) + state->pindexBestKnownBlock = itOld->second; + state->hashLastUnknownBlock.SetNull(); + } + } +} + +/** Update tracking information about which blocks a peer is assumed to have. */ +void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { + CNodeState *state = State(nodeid); + assert(state != NULL); + + ProcessBlockAvailability(nodeid); + + BlockMap::iterator it = mapBlockIndex.find(hash); + if (it != mapBlockIndex.end() && it->second->nChainWork > 0) { + // An actually better block was announced. + if (state->pindexBestKnownBlock == NULL || it->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) + state->pindexBestKnownBlock = it->second; + } else { + // An unknown block was announced; just assume that the latest one is the best one. + state->hashLastUnknownBlock = hash; + } +} + +void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom, CConnman& connman) { + if (!nodestate->fSupportsDesiredCmpctVersion) { + // Never ask from peers who can't provide witnesses. + return; + } + if (nodestate->fProvidesHeaderAndIDs) { + for (std::list<NodeId>::iterator it = lNodesAnnouncingHeaderAndIDs.begin(); it != lNodesAnnouncingHeaderAndIDs.end(); it++) { + if (*it == pfrom->GetId()) { + lNodesAnnouncingHeaderAndIDs.erase(it); + lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + return; + } + } + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; + if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { + // As per BIP152, we only get 3 of our peers to announce + // blocks using compact encodings. + connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + return true; + }); + lNodesAnnouncingHeaderAndIDs.pop_front(); + } + fAnnounceUsingCMPCTBLOCK = true; + connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + } +} + +// Requires cs_main +bool CanDirectFetch(const Consensus::Params &consensusParams) +{ + return chainActive.Tip()->GetBlockTime() > GetAdjustedTime() - consensusParams.nPowTargetSpacing * 20; +} + +// Requires cs_main +bool PeerHasHeader(CNodeState *state, CBlockIndex *pindex) +{ + if (state->pindexBestKnownBlock && pindex == state->pindexBestKnownBlock->GetAncestor(pindex->nHeight)) + return true; + if (state->pindexBestHeaderSent && pindex == state->pindexBestHeaderSent->GetAncestor(pindex->nHeight)) + return true; + return false; +} + +/** Find the last common ancestor two blocks have. + * Both pa and pb must be non-NULL. */ +CBlockIndex* LastCommonAncestor(CBlockIndex* pa, CBlockIndex* pb) { + if (pa->nHeight > pb->nHeight) { + pa = pa->GetAncestor(pb->nHeight); + } else if (pb->nHeight > pa->nHeight) { + pb = pb->GetAncestor(pa->nHeight); + } + + while (pa != pb && pa && pb) { + pa = pa->pprev; + pb = pb->pprev; + } + + // Eventually all chain branches meet at the genesis block. + assert(pa == pb); + return pa; +} + +/** Update pindexLastCommonBlock and add not-in-flight missing successors to vBlocks, until it has + * at most count entries. */ +void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<CBlockIndex*>& vBlocks, NodeId& nodeStaller, const Consensus::Params& consensusParams) { + if (count == 0) + return; + + vBlocks.reserve(vBlocks.size() + count); + CNodeState *state = State(nodeid); + assert(state != NULL); + + // Make sure pindexBestKnownBlock is up to date, we'll need it. + ProcessBlockAvailability(nodeid); + + if (state->pindexBestKnownBlock == NULL || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork) { + // This peer has nothing interesting. + return; + } + + if (state->pindexLastCommonBlock == NULL) { + // Bootstrap quickly by guessing a parent of our best tip is the forking point. + // Guessing wrong in either direction is not a problem. + state->pindexLastCommonBlock = chainActive[std::min(state->pindexBestKnownBlock->nHeight, chainActive.Height())]; + } + + // If the peer reorganized, our previous pindexLastCommonBlock may not be an ancestor + // of its current tip anymore. Go back enough to fix that. + state->pindexLastCommonBlock = LastCommonAncestor(state->pindexLastCommonBlock, state->pindexBestKnownBlock); + if (state->pindexLastCommonBlock == state->pindexBestKnownBlock) + return; + + std::vector<CBlockIndex*> vToFetch; + CBlockIndex *pindexWalk = state->pindexLastCommonBlock; + // Never fetch further than the best block we know the peer has, or more than BLOCK_DOWNLOAD_WINDOW + 1 beyond the last + // linked block we have in common with this peer. The +1 is so we can detect stalling, namely if we would be able to + // download that next block if the window were 1 larger. + int nWindowEnd = state->pindexLastCommonBlock->nHeight + BLOCK_DOWNLOAD_WINDOW; + int nMaxHeight = std::min<int>(state->pindexBestKnownBlock->nHeight, nWindowEnd + 1); + NodeId waitingfor = -1; + while (pindexWalk->nHeight < nMaxHeight) { + // Read up to 128 (or more, if more blocks than that are needed) successors of pindexWalk (towards + // pindexBestKnownBlock) into vToFetch. We fetch 128, because CBlockIndex::GetAncestor may be as expensive + // as iterating over ~100 CBlockIndex* entries anyway. + int nToFetch = std::min(nMaxHeight - pindexWalk->nHeight, std::max<int>(count - vBlocks.size(), 128)); + vToFetch.resize(nToFetch); + pindexWalk = state->pindexBestKnownBlock->GetAncestor(pindexWalk->nHeight + nToFetch); + vToFetch[nToFetch - 1] = pindexWalk; + for (unsigned int i = nToFetch - 1; i > 0; i--) { + vToFetch[i - 1] = vToFetch[i]->pprev; + } + + // Iterate over those blocks in vToFetch (in forward direction), adding the ones that + // are not yet downloaded and not in flight to vBlocks. In the mean time, update + // pindexLastCommonBlock as long as all ancestors are already downloaded, or if it's + // already part of our chain (and therefore don't need it even if pruned). + BOOST_FOREACH(CBlockIndex* pindex, vToFetch) { + if (!pindex->IsValid(BLOCK_VALID_TREE)) { + // We consider the chain that this peer is on invalid. + return; + } + if (!State(nodeid)->fHaveWitness && IsWitnessEnabled(pindex->pprev, consensusParams)) { + // We wouldn't download this block or its descendants from this peer. + return; + } + if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) { + if (pindex->nChainTx) + state->pindexLastCommonBlock = pindex; + } else if (mapBlocksInFlight.count(pindex->GetBlockHash()) == 0) { + // The block is not already downloaded, and not yet in flight. + if (pindex->nHeight > nWindowEnd) { + // We reached the end of the window. + if (vBlocks.size() == 0 && waitingfor != nodeid) { + // We aren't able to fetch anything, but we would be if the download window was one larger. + nodeStaller = waitingfor; + } + return; + } + vBlocks.push_back(pindex); + if (vBlocks.size() == count) { + return; + } + } else if (waitingfor == -1) { + // This is the first already-in-flight block. + waitingfor = mapBlocksInFlight[pindex->GetBlockHash()].first; + } + } + } +} + +} // anon namespace + +bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { + LOCK(cs_main); + CNodeState *state = State(nodeid); + if (state == NULL) + return false; + stats.nMisbehavior = state->nMisbehavior; + stats.nSyncHeight = state->pindexBestKnownBlock ? state->pindexBestKnownBlock->nHeight : -1; + stats.nCommonHeight = state->pindexLastCommonBlock ? state->pindexLastCommonBlock->nHeight : -1; + BOOST_FOREACH(const QueuedBlock& queue, state->vBlocksInFlight) { + if (queue.pindex) + stats.vHeightInFlight.push_back(queue.pindex->nHeight); + } + return true; +} + +void RegisterNodeSignals(CNodeSignals& nodeSignals) +{ + nodeSignals.ProcessMessages.connect(&ProcessMessages); + nodeSignals.SendMessages.connect(&SendMessages); + nodeSignals.InitializeNode.connect(&InitializeNode); + nodeSignals.FinalizeNode.connect(&FinalizeNode); +} + +void UnregisterNodeSignals(CNodeSignals& nodeSignals) +{ + nodeSignals.ProcessMessages.disconnect(&ProcessMessages); + nodeSignals.SendMessages.disconnect(&SendMessages); + nodeSignals.InitializeNode.disconnect(&InitializeNode); + nodeSignals.FinalizeNode.disconnect(&FinalizeNode); +} + +////////////////////////////////////////////////////////////////////////////// +// +// mapOrphanTransactions +// + +bool AddOrphanTx(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + uint256 hash = tx.GetHash(); + if (mapOrphanTransactions.count(hash)) + return false; + + // Ignore big transactions, to avoid a + // send-big-orphans memory exhaustion attack. If a peer has a legitimate + // large transaction with a missing parent then we assume + // it will rebroadcast it later, after the parent transaction(s) + // have been mined or received. + // 100 orphans, each of which is at most 99,999 bytes big is + // at most 10 megabytes of orphans and somewhat more byprev index (in the worst case): + unsigned int sz = GetTransactionWeight(tx); + if (sz >= MAX_STANDARD_TX_WEIGHT) + { + LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); + return false; + } + + auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME}); + assert(ret.second); + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); + } + + LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), + mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); + return true; +} + +int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.find(hash); + if (it == mapOrphanTransactions.end()) + return 0; + BOOST_FOREACH(const CTxIn& txin, it->second.tx.vin) + { + auto itPrev = mapOrphanTransactionsByPrev.find(txin.prevout); + if (itPrev == mapOrphanTransactionsByPrev.end()) + continue; + itPrev->second.erase(it); + if (itPrev->second.empty()) + mapOrphanTransactionsByPrev.erase(itPrev); + } + mapOrphanTransactions.erase(it); + return 1; +} + +void EraseOrphansFor(NodeId peer) +{ + int nErased = 0; + map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); + while (iter != mapOrphanTransactions.end()) + { + map<uint256, COrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid + if (maybeErase->second.fromPeer == peer) + { + nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); + } + } + if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer %d\n", nErased, peer); +} + + +unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + unsigned int nEvicted = 0; + static int64_t nNextSweep; + int64_t nNow = GetTime(); + if (nNextSweep <= nNow) { + // Sweep out expired orphan pool entries: + int nErased = 0; + int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL; + map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); + while (iter != mapOrphanTransactions.end()) + { + map<uint256, COrphanTx>::iterator maybeErase = iter++; + if (maybeErase->second.nTimeExpire <= nNow) { + nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); + } else { + nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime); + } + } + // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. + nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; + if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx due to expiration\n", nErased); + } + while (mapOrphanTransactions.size() > nMaxOrphans) + { + // Evict a random orphan: + uint256 randomhash = GetRandHash(); + map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.lower_bound(randomhash); + if (it == mapOrphanTransactions.end()) + it = mapOrphanTransactions.begin(); + EraseOrphanTx(it->first); + ++nEvicted; + } + return nEvicted; +} + +// Requires cs_main. +void Misbehaving(NodeId pnode, int howmuch) +{ + if (howmuch == 0) + return; + + CNodeState *state = State(pnode); + if (state == NULL) + return; + + state->nMisbehavior += howmuch; + int banscore = GetArg("-banscore", DEFAULT_BANSCORE_THRESHOLD); + if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) + { + LogPrintf("%s: %s peer=%d (%d -> %d) BAN THRESHOLD EXCEEDED\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior); + state->fShouldBan = true; + } else + LogPrintf("%s: %s peer=%d (%d -> %d)\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior); +} + + + + + + + + +////////////////////////////////////////////////////////////////////////////// +// +// blockchain -> download logic notification +// + +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) { + // Initialize global variables that cannot be constructed at startup. + recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); +} + +void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) { + if (nPosInBlock == CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK) + return; + + LOCK(cs_main); + + std::vector<uint256> vOrphanErase; + // Which orphan pool entries must we evict? + for (size_t j = 0; j < tx.vin.size(); j++) { + auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } + } + + // Erase orphan transactions include or precluded by this block + if (vOrphanErase.size()) { + int nErased = 0; + BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { + nErased += EraseOrphanTx(orphanHash); + } + LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); + } +} + +void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { + const int nNewHeight = pindexNew->nHeight; + connman->SetBestHeight(nNewHeight); + + if (!fInitialDownload) { + // Find the hashes of all blocks that weren't previously in the best chain. + std::vector<uint256> vHashes; + const CBlockIndex *pindexToAnnounce = pindexNew; + while (pindexToAnnounce != pindexFork) { + vHashes.push_back(pindexToAnnounce->GetBlockHash()); + pindexToAnnounce = pindexToAnnounce->pprev; + if (vHashes.size() == MAX_BLOCKS_TO_ANNOUNCE) { + // Limit announcements in case of a huge reorganization. + // Rely on the peer's synchronization mechanism in that case. + break; + } + } + // Relay inventory, but don't relay old inventory during initial block download. + connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { + if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { + BOOST_REVERSE_FOREACH(const uint256& hash, vHashes) { + pnode->PushBlockHash(hash); + } + } + }); + } + + nTimeBestReceived = GetTime(); +} + +void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationState& state) { + LOCK(cs_main); + + const uint256 hash(block.GetHash()); + std::map<uint256, std::pair<NodeId, bool>>::iterator it = mapBlockSource.find(hash); + + int nDoS = 0; + if (state.IsInvalid(nDoS)) { + if (it != mapBlockSource.end() && State(it->second.first)) { + assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; + State(it->second.first)->rejects.push_back(reject); + if (nDoS > 0 && it->second.second) + Misbehaving(it->second.first, nDoS); + } + } + if (it != mapBlockSource.end()) + mapBlockSource.erase(it); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Messages +// + + +bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + switch (inv.type) + { + case MSG_TX: + case MSG_WITNESS_TX: + { + assert(recentRejects); + if (chainActive.Tip()->GetBlockHash() != hashRecentRejectsChainTip) + { + // If the chain tip has changed previously rejected transactions + // might be now valid, e.g. due to a nLockTime'd tx becoming valid, + // or a double-spend. Reset the rejects filter and give those + // txs a second chance. + hashRecentRejectsChainTip = chainActive.Tip()->GetBlockHash(); + recentRejects->reset(); + } + + // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude + // requesting or processing some txs which have already been included in a block + return recentRejects->contains(inv.hash) || + mempool.exists(inv.hash) || + mapOrphanTransactions.count(inv.hash) || + pcoinsTip->HaveCoinsInCache(inv.hash); + } + case MSG_BLOCK: + case MSG_WITNESS_BLOCK: + return mapBlockIndex.count(inv.hash); + } + // Don't know what it is, just say we already got one + return true; +} + +static void RelayTransaction(const CTransaction& tx, CConnman& connman) +{ + CInv inv(MSG_TX, tx.GetHash()); + connman.ForEachNode([&inv](CNode* pnode) + { + pnode->PushInventory(inv); + }); +} + +static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connman) +{ + unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) + + // Relay to a limited number of other nodes + // Use deterministic randomness to send to the same nodes for 24 hours + // at a time so the addrKnowns of the chosen nodes prevent repeats + uint64_t hashAddr = addr.GetHash(); + const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); + FastRandomContext insecure_rand; + + std::array<std::pair<uint64_t, CNode*>,2> best{{{0, nullptr}, {0, nullptr}}}; + assert(nRelayNodes <= best.size()); + + auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { + if (pnode->nVersion >= CADDR_TIME_VERSION) { + uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); + for (unsigned int i = 0; i < nRelayNodes; i++) { + if (hashKey > best[i].first) { + std::copy(best.begin() + i, best.begin() + nRelayNodes - 1, best.begin() + i + 1); + best[i] = std::make_pair(hashKey, pnode); + break; + } + } + } + }; + + auto pushfunc = [&addr, &best, nRelayNodes, &insecure_rand] { + for (unsigned int i = 0; i < nRelayNodes && best[i].first != 0; i++) { + best[i].second->PushAddress(addr, insecure_rand); + } + }; + + connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); +} + +void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParams, CConnman& connman) +{ + std::deque<CInv>::iterator it = pfrom->vRecvGetData.begin(); + unsigned int nMaxSendBufferSize = connman.GetSendBufferSize(); + vector<CInv> vNotFound; + CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + LOCK(cs_main); + + while (it != pfrom->vRecvGetData.end()) { + // Don't bother if send buffer is too full to respond anyway + if (pfrom->nSendSize >= nMaxSendBufferSize) + break; + + const CInv &inv = *it; + { + boost::this_thread::interruption_point(); + it++; + + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) + { + bool send = false; + BlockMap::iterator mi = mapBlockIndex.find(inv.hash); + if (mi != mapBlockIndex.end()) + { + if (chainActive.Contains(mi->second)) { + send = true; + } else { + static const int nOneMonth = 30 * 24 * 60 * 60; + // To prevent fingerprinting attacks, only send blocks outside of the active + // chain if they are valid, and no more than a month older (both in time, and in + // best equivalent proof of work) than the best header chain we know about. + send = mi->second->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != NULL) && + (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() < nOneMonth) && + (GetBlockProofEquivalentTime(*pindexBestHeader, *mi->second, *pindexBestHeader, consensusParams) < nOneMonth); + if (!send) { + LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); + } + } + } + // disconnect node in case we have reached the outbound limit for serving historical blocks + // never disconnect whitelisted nodes + static const int nOneWeek = 7 * 24 * 60 * 60; // assume > 1 week = historical + if (send && connman.OutboundTargetReached(true) && ( ((pindexBestHeader != NULL) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > nOneWeek)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) + { + LogPrint("net", "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); + + //disconnect node + pfrom->fDisconnect = true; + send = false; + } + // Pruned nodes may have deleted the block, so check whether + // it's available before trying to send. + if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) + { + // Send block from disk + CBlock block; + if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) + assert(!"cannot load block from disk"); + if (inv.type == MSG_BLOCK) + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block)); + else if (inv.type == MSG_WITNESS_BLOCK) + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); + else if (inv.type == MSG_FILTERED_BLOCK) + { + bool sendMerkleBlock = false; + CMerkleBlock merkleBlock; + { + LOCK(pfrom->cs_filter); + if (pfrom->pfilter) { + sendMerkleBlock = true; + merkleBlock = CMerkleBlock(block, *pfrom->pfilter); + } + } + if (sendMerkleBlock) { + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see + // This avoids hurting performance by pointlessly requiring a round-trip + // Note that there is currently no way for a node to request any single transactions we didn't send here - + // they must either disconnect and retry or request the full block. + // Thus, the protocol spec specified allows for us to provide duplicate txn here, + // however we MUST always provide at least what the remote peer needs + typedef std::pair<unsigned int, uint256> PairType; + BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *block.vtx[pair.first])); + } + // else + // no response + } + else if (inv.type == MSG_CMPCT_BLOCK) + { + // If a peer is asking for old blocks, we're almost guaranteed + // they won't have a useful mempool to match against a compact block, + // and we don't feel like constructing the object for them, so + // instead we respond with the full, non-compact block. + bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; + int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; + if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { + CBlockHeaderAndShortTxIDs cmpctblock(block, fPeerWantsWitness); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + } else + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, block)); + } + + // Trigger the peer node to send a getblocks request for the next batch of inventory + if (inv.hash == pfrom->hashContinue) + { + // Bypass PushInventory, this must send even if redundant, + // and we want it right after the last block so they don't + // wait for other stuff first. + vector<CInv> vInv; + vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + pfrom->hashContinue.SetNull(); + } + } + } + else if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) + { + // Send stream from relay memory + bool push = false; + auto mi = mapRelay.find(inv.hash); + int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); + if (mi != mapRelay.end()) { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *mi->second)); + push = true; + } else if (pfrom->timeLastMempoolReq) { + auto txinfo = mempool.info(inv.hash); + // To protect privacy, do not answer getdata using the mempool when + // that TX couldn't have been INVed in reply to a MEMPOOL request. + if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *txinfo.tx)); + push = true; + } + } + if (!push) { + vNotFound.push_back(inv); + } + } + + // Track requests for our stuff. + GetMainSignals().Inventory(inv.hash); + + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) + break; + } + } + + pfrom->vRecvGetData.erase(pfrom->vRecvGetData.begin(), it); + + if (!vNotFound.empty()) { + // Let the peer know that we didn't find what it asked for, so it doesn't + // have to wait around forever. Currently only SPV clients actually care + // about this message: it's needed when they are recursively walking the + // dependencies of relevant unconfirmed transactions. SPV clients want to + // do that because they want to know about (and store and rebroadcast and + // risk analyze) the dependencies of transactions relevant to them, without + // having to download the entire memory pool. + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + } +} + +uint32_t GetFetchFlags(CNode* pfrom, CBlockIndex* pprev, const Consensus::Params& chainparams) { + uint32_t nFetchFlags = 0; + if ((pfrom->GetLocalServices() & NODE_WITNESS) && State(pfrom->GetId())->fHaveWitness) { + nFetchFlags |= MSG_WITNESS_FLAG; + } + return nFetchFlags; +} + +bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman) +{ + unsigned int nMaxSendBufferSize = connman.GetSendBufferSize(); + + LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); + if (mapArgs.count("-dropmessagestest") && GetRand(atoi(mapArgs["-dropmessagestest"])) == 0) + { + LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); + return true; + } + + + if (!(pfrom->GetLocalServices() & NODE_BLOOM) && + (strCommand == NetMsgType::FILTERLOAD || + strCommand == NetMsgType::FILTERADD)) + { + if (pfrom->nVersion >= NO_BLOOM_VERSION) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 100); + return false; + } else { + pfrom->fDisconnect = true; + return false; + } + } + + + if (strCommand == NetMsgType::VERSION) + { + // Each connection can only send one version message + if (pfrom->nVersion != 0) + { + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message"))); + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 1); + return false; + } + + int64_t nTime; + CAddress addrMe; + CAddress addrFrom; + uint64_t nNonce = 1; + uint64_t nServiceInt; + vRecv >> pfrom->nVersion >> nServiceInt >> nTime >> addrMe; + pfrom->nServices = ServiceFlags(nServiceInt); + if (!pfrom->fInbound) + { + connman.SetServices(pfrom->addr, pfrom->nServices); + } + if (pfrom->nServicesExpected & ~pfrom->nServices) + { + LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, pfrom->nServices, pfrom->nServicesExpected); + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); + pfrom->fDisconnect = true; + return false; + } + + if (pfrom->nVersion < MIN_PEER_PROTO_VERSION) + { + // disconnect from peers older than this proto version + LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, pfrom->nVersion); + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, + strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); + pfrom->fDisconnect = true; + return false; + } + + if (pfrom->nVersion == 10300) + pfrom->nVersion = 300; + if (!vRecv.empty()) + vRecv >> addrFrom >> nNonce; + if (!vRecv.empty()) { + vRecv >> LIMITED_STRING(pfrom->strSubVer, MAX_SUBVERSION_LENGTH); + pfrom->cleanSubVer = SanitizeString(pfrom->strSubVer); + } + if (!vRecv.empty()) { + vRecv >> pfrom->nStartingHeight; + } + { + LOCK(pfrom->cs_filter); + if (!vRecv.empty()) + vRecv >> pfrom->fRelayTxes; // set to true after we get the first filter* message + else + pfrom->fRelayTxes = true; + } + + // Disconnect if we connected to ourself + if (pfrom->fInbound && !connman.CheckIncomingNonce(nNonce)) + { + LogPrintf("connected to self at %s, disconnecting\n", pfrom->addr.ToString()); + pfrom->fDisconnect = true; + return true; + } + + pfrom->addrLocal = addrMe; + if (pfrom->fInbound && addrMe.IsRoutable()) + { + SeenLocal(addrMe); + } + + // Be shy and don't send version until we hear + if (pfrom->fInbound) + PushNodeVersion(pfrom, connman, GetAdjustedTime()); + + pfrom->fClient = !(pfrom->nServices & NODE_NETWORK); + + if((pfrom->nServices & NODE_WITNESS)) + { + LOCK(cs_main); + State(pfrom->GetId())->fHaveWitness = true; + } + + // Potentially mark this peer as a preferred download peer. + { + LOCK(cs_main); + UpdatePreferredDownload(pfrom, State(pfrom->GetId())); + } + + // Change version + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + int nSendVersion = std::min(pfrom->nVersion, PROTOCOL_VERSION); + pfrom->SetSendVersion(nSendVersion); + + if (!pfrom->fInbound) + { + // Advertise our address + if (fListen && !IsInitialBlockDownload()) + { + CAddress addr = GetLocalAddress(&pfrom->addr, pfrom->GetLocalServices()); + FastRandomContext insecure_rand; + if (addr.IsRoutable()) + { + LogPrint("net", "ProcessMessages: advertising address %s\n", addr.ToString()); + pfrom->PushAddress(addr, insecure_rand); + } else if (IsPeerAddrLocalGood(pfrom)) { + addr.SetIP(pfrom->addrLocal); + LogPrint("net", "ProcessMessages: advertising address %s\n", addr.ToString()); + pfrom->PushAddress(addr, insecure_rand); + } + } + + // Get recent addresses + if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman.GetAddressCount() < 1000) + { + connman.PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + pfrom->fGetAddr = true; + } + connman.MarkAddressGood(pfrom->addr); + } + + pfrom->fSuccessfullyConnected = true; + + string remoteAddr; + if (fLogIPs) + remoteAddr = ", peeraddr=" + pfrom->addr.ToString(); + + LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", + pfrom->cleanSubVer, pfrom->nVersion, + pfrom->nStartingHeight, addrMe.ToString(), pfrom->id, + remoteAddr); + + int64_t nTimeOffset = nTime - GetTime(); + pfrom->nTimeOffset = nTimeOffset; + AddTimeData(pfrom->addr, nTimeOffset); + + // Feeler connections exist only to verify if address is online. + if (pfrom->fFeeler) { + assert(pfrom->fInbound == false); + pfrom->fDisconnect = true; + } + return true; + } + + + else if (pfrom->nVersion == 0) + { + // Must have a version message before anything else + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 1); + return false; + } + + // At this point, the outgoing message serialization version can't change. + CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + + if (strCommand == NetMsgType::VERACK) + { + pfrom->SetRecvVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); + + if (!pfrom->fInbound) { + // Mark this node as currently connected, so we update its timestamp later. + LOCK(cs_main); + State(pfrom->GetId())->fCurrentlyConnected = true; + } + + if (pfrom->nVersion >= SENDHEADERS_VERSION) { + // Tell our peer we prefer to receive headers rather than inv's + // We send this to non-NODE NETWORK peers as well, because even + // non-NODE NETWORK peers can announce blocks (such as pruning + // nodes) + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + } + if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { + // Tell our peer we are willing to provide version 1 or 2 cmpctblocks + // However, we do not request new block announcements using + // cmpctblock messages. + // We send this to non-NODE NETWORK peers as well, because + // they may wish to request compact blocks from us + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 2; + if (pfrom->GetLocalServices() & NODE_WITNESS) + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + nCMPCTBLOCKVersion = 1; + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + } + } + + + else if (strCommand == NetMsgType::ADDR) + { + vector<CAddress> vAddr; + vRecv >> vAddr; + + // Don't want addr from older versions unless seeding + if (pfrom->nVersion < CADDR_TIME_VERSION && connman.GetAddressCount() > 1000) + return true; + if (vAddr.size() > 1000) + { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 20); + return error("message addr size() = %u", vAddr.size()); + } + + // Store the new addresses + vector<CAddress> vAddrOk; + int64_t nNow = GetAdjustedTime(); + int64_t nSince = nNow - 10 * 60; + BOOST_FOREACH(CAddress& addr, vAddr) + { + boost::this_thread::interruption_point(); + + if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + continue; + + if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) + addr.nTime = nNow - 5 * 24 * 60 * 60; + pfrom->AddAddressKnown(addr); + bool fReachable = IsReachable(addr); + if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) + { + // Relay to a limited number of other nodes + RelayAddress(addr, fReachable, connman); + } + // Do not store addresses outside our network + if (fReachable) + vAddrOk.push_back(addr); + } + connman.AddNewAddresses(vAddrOk, pfrom->addr, 2 * 60 * 60); + if (vAddr.size() < 1000) + pfrom->fGetAddr = false; + if (pfrom->fOneShot) + pfrom->fDisconnect = true; + } + + else if (strCommand == NetMsgType::SENDHEADERS) + { + LOCK(cs_main); + State(pfrom->GetId())->fPreferHeaders = true; + } + + else if (strCommand == NetMsgType::SENDCMPCT) + { + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 0; + vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; + if (nCMPCTBLOCKVersion == 1 || ((pfrom->GetLocalServices() & NODE_WITNESS) && nCMPCTBLOCKVersion == 2)) { + LOCK(cs_main); + // fProvidesHeaderAndIDs is used to "lock in" version of compact blocks we send (fWantsCmpctWitness) + if (!State(pfrom->GetId())->fProvidesHeaderAndIDs) { + State(pfrom->GetId())->fProvidesHeaderAndIDs = true; + State(pfrom->GetId())->fWantsCmpctWitness = nCMPCTBLOCKVersion == 2; + } + if (State(pfrom->GetId())->fWantsCmpctWitness == (nCMPCTBLOCKVersion == 2)) // ignore later version announces + State(pfrom->GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; + if (!State(pfrom->GetId())->fSupportsDesiredCmpctVersion) { + if (pfrom->GetLocalServices() & NODE_WITNESS) + State(pfrom->GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 2); + else + State(pfrom->GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 1); + } + } + } + + + else if (strCommand == NetMsgType::INV) + { + vector<CInv> vInv; + vRecv >> vInv; + if (vInv.size() > MAX_INV_SZ) + { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 20); + return error("message inv size() = %u", vInv.size()); + } + + bool fBlocksOnly = !fRelayTxes; + + // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true + if (pfrom->fWhitelisted && GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY)) + fBlocksOnly = false; + + LOCK(cs_main); + + uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + + std::vector<CInv> vToFetch; + + for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) + { + CInv &inv = vInv[nInv]; + + boost::this_thread::interruption_point(); + + bool fAlreadyHave = AlreadyHave(inv); + LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); + + if (inv.type == MSG_TX) { + inv.type |= nFetchFlags; + } + + if (inv.type == MSG_BLOCK) { + UpdateBlockAvailability(pfrom->GetId(), inv.hash); + if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { + // We used to request the full block here, but since headers-announcements are now the + // primary method of announcement on the network, and since, in the case that a node + // fell back to inv we probably have a reorg which we should get the headers for first, + // we now only provide a getheaders response here. When we receive the headers, we will + // then ask for the blocks we need. + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); + LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); + } + } + else + { + pfrom->AddInventoryKnown(inv); + if (fBlocksOnly) + LogPrint("net", "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); + else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) + pfrom->AskFor(inv); + } + + // Track requests for our stuff + GetMainSignals().Inventory(inv.hash); + + if (pfrom->nSendSize > (nMaxSendBufferSize * 2)) { + Misbehaving(pfrom->GetId(), 50); + return error("send buffer size() = %u", pfrom->nSendSize); + } + } + + if (!vToFetch.empty()) + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vToFetch)); + } + + + else if (strCommand == NetMsgType::GETDATA) + { + vector<CInv> vInv; + vRecv >> vInv; + if (vInv.size() > MAX_INV_SZ) + { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 20); + return error("message getdata size() = %u", vInv.size()); + } + + if (fDebug || (vInv.size() != 1)) + LogPrint("net", "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); + + if ((fDebug && vInv.size() > 0) || (vInv.size() == 1)) + LogPrint("net", "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); + + pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); + ProcessGetData(pfrom, chainparams.GetConsensus(), connman); + } + + + else if (strCommand == NetMsgType::GETBLOCKS) + { + CBlockLocator locator; + uint256 hashStop; + vRecv >> locator >> hashStop; + + LOCK(cs_main); + + // Find the last block the caller has in the main chain + CBlockIndex* pindex = FindForkInGlobalIndex(chainActive, locator); + + // Send the rest of the chain + if (pindex) + pindex = chainActive.Next(pindex); + int nLimit = 500; + LogPrint("net", "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); + for (; pindex; pindex = chainActive.Next(pindex)) + { + if (pindex->GetBlockHash() == hashStop) + { + LogPrint("net", " getblocks stopping at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + break; + } + // If pruning, don't inv blocks unless we have on disk and are likely to still have + // for some reasonable time window (1 hour) that block relay might require. + const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / chainparams.GetConsensus().nPowTargetSpacing; + if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= chainActive.Tip()->nHeight - nPrunedBlocksLikelyToHave)) + { + LogPrint("net", " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + break; + } + pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); + if (--nLimit <= 0) + { + // When this block is requested, we'll send an inv that'll + // trigger the peer to getblocks the next batch of inventory. + LogPrint("net", " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + pfrom->hashContinue = pindex->GetBlockHash(); + break; + } + } + } + + + else if (strCommand == NetMsgType::GETBLOCKTXN) + { + BlockTransactionsRequest req; + vRecv >> req; + + LOCK(cs_main); + + BlockMap::iterator it = mapBlockIndex.find(req.blockhash); + if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + return true; + } + + if (it->second->nHeight < chainActive.Height() - MAX_BLOCKTXN_DEPTH) { + // If an older block is requested (should never happen in practice, + // but can happen in tests) send a block response instead of a + // blocktxn response. Sending a full block response instead of a + // small blocktxn response is preferable in the case where a peer + // might maliciously send lots of getblocktxn requests to trigger + // expensive disk reads, because it will require the peer to + // actually receive all the data read from disk over the network. + LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + CInv inv; + inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; + inv.hash = req.blockhash; + pfrom->vRecvGetData.push_back(inv); + ProcessGetData(pfrom, chainparams.GetConsensus(), connman); + return true; + } + + CBlock block; + bool ret = ReadBlockFromDisk(block, it->second, chainparams.GetConsensus()); + assert(ret); + + BlockTransactions resp(req); + for (size_t i = 0; i < req.indexes.size(); i++) { + if (req.indexes[i] >= block.vtx.size()) { + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + return true; + } + resp.txn[i] = block.vtx[req.indexes[i]]; + } + int nSendFlags = State(pfrom->GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); + } + + + else if (strCommand == NetMsgType::GETHEADERS) + { + CBlockLocator locator; + uint256 hashStop; + vRecv >> locator >> hashStop; + + LOCK(cs_main); + if (IsInitialBlockDownload() && !pfrom->fWhitelisted) { + LogPrint("net", "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); + return true; + } + + CNodeState *nodestate = State(pfrom->GetId()); + CBlockIndex* pindex = NULL; + if (locator.IsNull()) + { + // If locator is null, return the hashStop block + BlockMap::iterator mi = mapBlockIndex.find(hashStop); + if (mi == mapBlockIndex.end()) + return true; + pindex = (*mi).second; + } + else + { + // Find the last block the caller has in the main chain + pindex = FindForkInGlobalIndex(chainActive, locator); + if (pindex) + pindex = chainActive.Next(pindex); + } + + // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end + vector<CBlock> vHeaders; + int nLimit = MAX_HEADERS_RESULTS; + LogPrint("net", "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->id); + for (; pindex; pindex = chainActive.Next(pindex)) + { + vHeaders.push_back(pindex->GetBlockHeader()); + if (--nLimit <= 0 || pindex->GetBlockHash() == hashStop) + break; + } + // pindex can be NULL either if we sent chainActive.Tip() OR + // if our peer has chainActive.Tip() (and thus we are sending an empty + // headers message). In both cases it's safe to update + // pindexBestHeaderSent to be our tip. + nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + } + + + else if (strCommand == NetMsgType::TX) + { + // Stop processing the transaction early if + // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off + if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) + { + LogPrint("net", "transaction sent in violation of protocol peer=%d\n", pfrom->id); + return true; + } + + deque<COutPoint> vWorkQueue; + vector<uint256> vEraseQueue; + CTransaction tx(deserialize, vRecv); + + CInv inv(MSG_TX, tx.GetHash()); + pfrom->AddInventoryKnown(inv); + + LOCK(cs_main); + + bool fMissingInputs = false; + CValidationState state; + + pfrom->setAskFor.erase(inv.hash); + mapAlreadyAskedFor.erase(inv.hash); + + if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) { + mempool.check(pcoinsTip); + RelayTransaction(tx, connman); + for (unsigned int i = 0; i < tx.vout.size(); i++) { + vWorkQueue.emplace_back(inv.hash, i); + } + + pfrom->nLastTXTime = GetTime(); + + LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", + pfrom->id, + tx.GetHash().ToString(), + mempool.size(), mempool.DynamicMemoryUsage() / 1000); + + // Recursively process any orphan transactions that depended on this one + set<NodeId> setMisbehaving; + while (!vWorkQueue.empty()) { + auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); + vWorkQueue.pop_front(); + if (itByPrev == mapOrphanTransactionsByPrev.end()) + continue; + for (auto mi = itByPrev->second.begin(); + mi != itByPrev->second.end(); + ++mi) + { + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + NodeId fromPeer = (*mi)->second.fromPeer; + bool fMissingInputs2 = false; + // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan + // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get + // anyone relaying LegitTxX banned) + CValidationState stateDummy; + + + if (setMisbehaving.count(fromPeer)) + continue; + if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { + LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); + RelayTransaction(orphanTx, connman); + for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { + vWorkQueue.emplace_back(orphanHash, i); + } + vEraseQueue.push_back(orphanHash); + } + else if (!fMissingInputs2) + { + int nDos = 0; + if (stateDummy.IsInvalid(nDos) && nDos > 0) + { + // Punish peer that gave us an invalid orphan tx + Misbehaving(fromPeer, nDos); + setMisbehaving.insert(fromPeer); + LogPrint("mempool", " invalid orphan tx %s\n", orphanHash.ToString()); + } + // Has inputs but not accepted to mempool + // Probably non-standard or insufficient fee/priority + LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); + vEraseQueue.push_back(orphanHash); + if (orphanTx.wit.IsNull() && !stateDummy.CorruptionPossible()) { + // Do not use rejection cache for witness transactions or + // witness-stripped transactions, as they can have been malleated. + // See https://github.com/bitcoin/bitcoin/issues/8279 for details. + assert(recentRejects); + recentRejects->insert(orphanHash); + } + } + mempool.check(pcoinsTip); + } + } + + BOOST_FOREACH(uint256 hash, vEraseQueue) + EraseOrphanTx(hash); + } + else if (fMissingInputs) + { + bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + if (recentRejects->contains(txin.prevout.hash)) { + fRejectedParents = true; + break; + } + } + if (!fRejectedParents) { + uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); + pfrom->AddInventoryKnown(_inv); + if (!AlreadyHave(_inv)) pfrom->AskFor(_inv); + } + AddOrphanTx(tx, pfrom->GetId()); + + // DoS prevention: do not allow mapOrphanTransactions to grow unbounded + unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); + if (nEvicted > 0) + LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + } else { + LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); + } + } else { + if (tx.wit.IsNull() && !state.CorruptionPossible()) { + // Do not use rejection cache for witness transactions or + // witness-stripped transactions, as they can have been malleated. + // See https://github.com/bitcoin/bitcoin/issues/8279 for details. + assert(recentRejects); + recentRejects->insert(tx.GetHash()); + } + + if (pfrom->fWhitelisted && GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { + // Always relay transactions received from whitelisted peers, even + // if they were already in the mempool or rejected from it due + // to policy, allowing the node to function as a gateway for + // nodes hidden behind it. + // + // Never relay transactions that we would assign a non-zero DoS + // score for, as we expect peers to do the same with us in that + // case. + int nDoS = 0; + if (!state.IsInvalid(nDoS) || nDoS == 0) { + LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); + RelayTransaction(tx, connman); + } else { + LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); + } + } + } + int nDoS = 0; + if (state.IsInvalid(nDoS)) + { + LogPrint("mempoolrej", "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), + pfrom->id, + FormatStateMessage(state)); + if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); + if (nDoS > 0) { + Misbehaving(pfrom->GetId(), nDoS); + } + } + } + + + else if (strCommand == NetMsgType::CMPCTBLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + { + CBlockHeaderAndShortTxIDs cmpctblock; + vRecv >> cmpctblock; + + { + LOCK(cs_main); + + if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { + // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers + if (!IsInitialBlockDownload()) + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + return true; + } + } + + CBlockIndex *pindex = NULL; + CValidationState state; + if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), nDoS); + } + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + return true; + } + } + + // Keep a CBlock for "optimistic" compactblock reconstructions (see + // below) + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); + bool fBlockReconstructed = false; + + LOCK(cs_main); + // If AcceptBlockHeader returned true, it set pindex + assert(pindex); + UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + + std::map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); + bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); + + if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here + return true; + + if (pindex->nChainWork <= chainActive.Tip()->nChainWork || // We know something better + pindex->nTx != 0) { // We had this block at some point, but pruned it + if (fAlreadyInFlight) { + // We requested this block for some reason, but our mempool will probably be useless + // so we just grab the block via normal getdata + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + } + return true; + } + + // If we're not close to tip yet, give up and let parallel block fetch work its magic + if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) + return true; + + CNodeState *nodestate = State(pfrom->GetId()); + + if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus()) && !nodestate->fSupportsDesiredCmpctVersion) { + // Don't bother trying to process compact blocks from v1 peers + // after segwit activates. + return true; + } + + // We want to be a bit conservative just to be extra careful about DoS + // possibilities in compact block processing... + if (pindex->nHeight <= chainActive.Height() + 2) { + if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || + (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { + list<QueuedBlock>::iterator *queuedBlockIt = NULL; + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!(*queuedBlockIt)->partialBlock) + (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); + else { + // The block was already in flight using compact blocks from the same peer + LogPrint("net", "Peer sent us compact block we were already syncing!\n"); + return true; + } + } + + PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; + ReadStatus status = partialBlock.InitData(cmpctblock); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Duplicate txindexes, the block is now in-flight, so just request it + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return true; + } + + if (!fAlreadyInFlight && mapBlocksInFlight.size() == 1 && pindex->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // We seem to be rather well-synced, so it appears pfrom was the first to provide us + // with this block! Let's get them to announce using compact blocks in the future. + MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman); + } + + BlockTransactionsRequest req; + for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { + if (!partialBlock.IsTxAvailable(i)) + req.indexes.push_back(i); + } + if (req.indexes.empty()) { + // Dirty hack to jump to BLOCKTXN code (TODO: move message handling into their own functions) + BlockTransactions txn; + txn.blockhash = cmpctblock.header.GetHash(); + CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); + blockTxnMsg << txn; + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman); + } else { + req.blockhash = pindex->GetBlockHash(); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } + } else { + // This block is either already in flight from a different + // peer, or this peer has too many blocks outstanding to + // download from. + // Optimistically try to reconstruct anyway since we might be + // able to without any round trips. + PartiallyDownloadedBlock tempBlock(&mempool); + ReadStatus status = tempBlock.InitData(cmpctblock); + if (status != READ_STATUS_OK) { + // TODO: don't ignore failures + return true; + } + std::vector<CTransactionRef> dummy; + status = tempBlock.FillBlock(*pblock, dummy); + if (status == READ_STATUS_OK) { + fBlockReconstructed = true; + } + } + } else { + if (fAlreadyInFlight) { + // We requested this block, but its far into the future, so our + // mempool will probably be useless - request the block normally + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return true; + } else { + // If this was an announce-cmpctblock, we want the same treatment as a header message + // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) + std::vector<CBlock> headers; + headers.push_back(cmpctblock.header); + CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); + vHeadersMsg << headers; + return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman); + } + } + + if (fBlockReconstructed) { + // If we got here, we were able to optimistically reconstruct a + // block that is in flight from some other peer. + { + LOCK(cs_main); + mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom->GetId(), false)); + } + bool fNewBlock = false; + ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); + + LOCK(cs_main); // hold cs_main for CBlockIndex::IsValid() + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS)) { + // Clear download state for this block, which is in + // process from some other peer. We do this after calling + // ProcessNewBlock so that a malleated cmpctblock announcement + // can't be used to interfere with block relay. + MarkBlockAsReceived(pblock->GetHash()); + } + } + + } + + else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing + { + BlockTransactions resp; + vRecv >> resp; + + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); + bool fBlockRead = false; + { + LOCK(cs_main); + + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); + if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || + it->second.first != pfrom->GetId()) { + LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + return true; + } + + PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Might have collided, fall back to getdata now :( + std::vector<CInv> invs; + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else { + // Block is either okay, or possibly we received + // READ_STATUS_CHECKBLOCK_FAILED. + // Note that CheckBlock can only fail for one of a few reasons: + // 1. bad-proof-of-work (impossible here, because we've already + // accepted the header) + // 2. merkleroot doesn't match the transactions given (already + // caught in FillBlock with READ_STATUS_FAILED, so + // impossible here) + // 3. the block is otherwise invalid (eg invalid coinbase, + // block is too big, too many legacy sigops, etc). + // So if CheckBlock failed, #3 is the only possibility. + // Under BIP 152, we don't DoS-ban unless proof of work is + // invalid (we don't require all the stateless checks to have + // been run). This is handled below, so just treat this as + // though the block was successfully read, and rely on the + // handling in ProcessNewBlock to ensure the block index is + // updated, reject messages go out, etc. + MarkBlockAsReceived(resp.blockhash); // it is now an empty pointer + fBlockRead = true; + // mapBlockSource is only used for sending reject messages and DoS scores, + // so the race between here and cs_main in ProcessNewBlock is fine. + // BIP 152 permits peers to relay compact blocks after validating + // the header only; we should not punish peers if the block turns + // out to be invalid. + mapBlockSource.emplace(resp.blockhash, std::make_pair(pfrom->GetId(), false)); + } + } // Don't hold cs_main when we call into ProcessNewBlock + if (fBlockRead) { + bool fNewBlock = false; + // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, + // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) + ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); + } + } + + + else if (strCommand == NetMsgType::HEADERS && !fImporting && !fReindex) // Ignore headers received while importing + { + std::vector<CBlockHeader> headers; + + // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. + unsigned int nCount = ReadCompactSize(vRecv); + if (nCount > MAX_HEADERS_RESULTS) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 20); + return error("headers message size = %u", nCount); + } + headers.resize(nCount); + for (unsigned int n = 0; n < nCount; n++) { + vRecv >> headers[n]; + ReadCompactSize(vRecv); // ignore tx count; assume it is 0. + } + + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + return true; + } + + CBlockIndex *pindexLast = NULL; + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this looks like it could be a block announcement (nCount < + // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that + // don't connect: + // - Send a getheaders message in response to try to connect the chain. + // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + // don't connect before giving DoS points + // - Once a headers message is received that is valid and does connect, + // nUnconnectingHeaders gets reset back to 0. + if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + nodestate->nUnconnectingHeaders++; + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + LogPrint("net", "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + pindexBestHeader->nHeight, + pfrom->id, nodestate->nUnconnectingHeaders); + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); + + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(pfrom->GetId(), 20); + } + return true; + } + + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + Misbehaving(pfrom->GetId(), 20); + return error("non-continuous headers sequence"); + } + hashLastBlock = header.GetHash(); + } + } + + CValidationState state; + if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), nDoS); + } + return error("invalid header received"); + } + } + + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint("net", "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue + // from there instead. + LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); + } + + bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); + // If this set of headers is valid and ends in a block with at least as + // much work as our tip, download as much as possible. + if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { + vector<CBlockIndex *> vToFetch; + CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && + (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!chainActive.Contains(pindexWalk)) { + LogPrint("net", "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + vector<CInv> vGetData; + // Download as much as possible, from earliest to latest. + BOOST_REVERSE_FOREACH(CBlockIndex *pindex, vToFetch) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); + LogPrint("net", "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom->id); + } + if (vGetData.size() > 1) { + LogPrint("net", "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // We seem to be rather well-synced, so it appears pfrom was the first to provide us + // with this block! Let's get them to announce using compact blocks in the future. + MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman); + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } + } + } + + else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + { + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); + vRecv >> *pblock; + + LogPrint("net", "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->id); + + // Process all blocks from whitelisted peers, even if not requested, + // unless we're still syncing with the network. + // Such an unrequested block may still be processed, subject to the + // conditions in AcceptBlock(). + bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); + const uint256 hash(pblock->GetHash()); + { + LOCK(cs_main); + // Also always process if we requested the block explicitly, as we may + // need it even though it is not a candidate for a new best tip. + forceProcessing |= MarkBlockAsReceived(hash); + // mapBlockSource is only used for sending reject messages and DoS scores, + // so the race between here and cs_main in ProcessNewBlock is fine. + mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); + } + bool fNewBlock = false; + ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); + } + + + else if (strCommand == NetMsgType::GETADDR) + { + // This asymmetric behavior for inbound and outbound connections was introduced + // to prevent a fingerprinting attack: an attacker can send specific fake addresses + // to users' AddrMan and later request them by sending getaddr messages. + // Making nodes which are behind NAT and can only make outgoing connections ignore + // the getaddr message mitigates the attack. + if (!pfrom->fInbound) { + LogPrint("net", "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); + return true; + } + + // Only send one GetAddr response per connection to reduce resource waste + // and discourage addr stamping of INV announcements. + if (pfrom->fSentAddr) { + LogPrint("net", "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); + return true; + } + pfrom->fSentAddr = true; + + pfrom->vAddrToSend.clear(); + vector<CAddress> vAddr = connman.GetAddresses(); + FastRandomContext insecure_rand; + BOOST_FOREACH(const CAddress &addr, vAddr) + pfrom->PushAddress(addr, insecure_rand); + } + + + else if (strCommand == NetMsgType::MEMPOOL) + { + if (!(pfrom->GetLocalServices() & NODE_BLOOM) && !pfrom->fWhitelisted) + { + LogPrint("net", "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); + pfrom->fDisconnect = true; + return true; + } + + if (connman.OutboundTargetReached(false) && !pfrom->fWhitelisted) + { + LogPrint("net", "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); + pfrom->fDisconnect = true; + return true; + } + + LOCK(pfrom->cs_inventory); + pfrom->fSendMempool = true; + } + + + else if (strCommand == NetMsgType::PING) + { + if (pfrom->nVersion > BIP0031_VERSION) + { + uint64_t nonce = 0; + vRecv >> nonce; + // Echo the message back with the nonce. This allows for two useful features: + // + // 1) A remote node can quickly check if the connection is operational + // 2) Remote nodes can measure the latency of the network thread. If this node + // is overloaded it won't respond to pings quickly and the remote node can + // avoid sending us more work, like chain download requests. + // + // The nonce stops the remote getting confused between different pings: without + // it, if the remote node sends a ping once per second and this node takes 5 + // seconds to respond to each, the 5th ping the remote sends would appear to + // return very quickly. + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + } + } + + + else if (strCommand == NetMsgType::PONG) + { + int64_t pingUsecEnd = nTimeReceived; + uint64_t nonce = 0; + size_t nAvail = vRecv.in_avail(); + bool bPingFinished = false; + std::string sProblem; + + if (nAvail >= sizeof(nonce)) { + vRecv >> nonce; + + // Only process pong message if there is an outstanding ping (old ping without nonce should never pong) + if (pfrom->nPingNonceSent != 0) { + if (nonce == pfrom->nPingNonceSent) { + // Matching pong received, this ping is no longer outstanding + bPingFinished = true; + int64_t pingUsecTime = pingUsecEnd - pfrom->nPingUsecStart; + if (pingUsecTime > 0) { + // Successful ping time measurement, replace previous + pfrom->nPingUsecTime = pingUsecTime; + pfrom->nMinPingUsecTime = std::min(pfrom->nMinPingUsecTime, pingUsecTime); + } else { + // This should never happen + sProblem = "Timing mishap"; + } + } else { + // Nonce mismatches are normal when pings are overlapping + sProblem = "Nonce mismatch"; + if (nonce == 0) { + // This is most likely a bug in another implementation somewhere; cancel this ping + bPingFinished = true; + sProblem = "Nonce zero"; + } + } + } else { + sProblem = "Unsolicited pong without ping"; + } + } else { + // This is most likely a bug in another implementation somewhere; cancel this ping + bPingFinished = true; + sProblem = "Short payload"; + } + + if (!(sProblem.empty())) { + LogPrint("net", "pong peer=%d: %s, %x expected, %x received, %u bytes\n", + pfrom->id, + sProblem, + pfrom->nPingNonceSent, + nonce, + nAvail); + } + if (bPingFinished) { + pfrom->nPingNonceSent = 0; + } + } + + + else if (strCommand == NetMsgType::FILTERLOAD) + { + CBloomFilter filter; + vRecv >> filter; + + if (!filter.IsWithinSizeConstraints()) + { + // There is no excuse for sending a too-large filter + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 100); + } + else + { + LOCK(pfrom->cs_filter); + delete pfrom->pfilter; + pfrom->pfilter = new CBloomFilter(filter); + pfrom->pfilter->UpdateEmptyFull(); + pfrom->fRelayTxes = true; + } + } + + + else if (strCommand == NetMsgType::FILTERADD) + { + vector<unsigned char> vData; + vRecv >> vData; + + // Nodes must NEVER send a data item > 520 bytes (the max size for a script data object, + // and thus, the maximum size any matched object can have) in a filteradd message + bool bad = false; + if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { + bad = true; + } else { + LOCK(pfrom->cs_filter); + if (pfrom->pfilter) { + pfrom->pfilter->insert(vData); + } else { + bad = true; + } + } + if (bad) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 100); + } + } + + + else if (strCommand == NetMsgType::FILTERCLEAR) + { + LOCK(pfrom->cs_filter); + if (pfrom->GetLocalServices() & NODE_BLOOM) { + delete pfrom->pfilter; + pfrom->pfilter = new CBloomFilter(); + } + pfrom->fRelayTxes = true; + } + + + else if (strCommand == NetMsgType::REJECT) + { + if (fDebug) { + try { + string strMsg; unsigned char ccode; string strReason; + vRecv >> LIMITED_STRING(strMsg, CMessageHeader::COMMAND_SIZE) >> ccode >> LIMITED_STRING(strReason, MAX_REJECT_MESSAGE_LENGTH); + + ostringstream ss; + ss << strMsg << " code " << itostr(ccode) << ": " << strReason; + + if (strMsg == NetMsgType::BLOCK || strMsg == NetMsgType::TX) + { + uint256 hash; + vRecv >> hash; + ss << ": hash " << hash.ToString(); + } + LogPrint("net", "Reject %s\n", SanitizeString(ss.str())); + } catch (const std::ios_base::failure&) { + // Avoid feedback loops by preventing reject messages from triggering a new reject message. + LogPrint("net", "Unparseable reject message received\n"); + } + } + } + + else if (strCommand == NetMsgType::FEEFILTER) { + CAmount newFeeFilter = 0; + vRecv >> newFeeFilter; + if (MoneyRange(newFeeFilter)) { + { + LOCK(pfrom->cs_feeFilter); + pfrom->minFeeFilter = newFeeFilter; + } + LogPrint("net", "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); + } + } + + else if (strCommand == NetMsgType::NOTFOUND) { + // We do not care about the NOTFOUND message, but logging an Unknown Command + // message would be undesirable as we transmit it ourselves. + } + + else { + // Ignore unknown commands for extensibility + LogPrint("net", "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); + } + + + + return true; +} + +// requires LOCK(cs_vRecvMsg) +bool ProcessMessages(CNode* pfrom, CConnman& connman) +{ + const CChainParams& chainparams = Params(); + unsigned int nMaxSendBufferSize = connman.GetSendBufferSize(); + //if (fDebug) + // LogPrintf("%s(%u messages)\n", __func__, pfrom->vRecvMsg.size()); + + // + // Message format + // (4) message start + // (12) command + // (4) size + // (4) checksum + // (x) data + // + bool fOk = true; + + if (!pfrom->vRecvGetData.empty()) + ProcessGetData(pfrom, chainparams.GetConsensus(), connman); + + // this maintains the order of responses + if (!pfrom->vRecvGetData.empty()) return fOk; + + std::deque<CNetMessage>::iterator it = pfrom->vRecvMsg.begin(); + while (!pfrom->fDisconnect && it != pfrom->vRecvMsg.end()) { + // Don't bother if send buffer is too full to respond anyway + if (pfrom->nSendSize >= nMaxSendBufferSize) + break; + + // get next message + CNetMessage& msg = *it; + + //if (fDebug) + // LogPrintf("%s(message %u msgsz, %u bytes, complete:%s)\n", __func__, + // msg.hdr.nMessageSize, msg.vRecv.size(), + // msg.complete() ? "Y" : "N"); + + // end, if an incomplete message is found + if (!msg.complete()) + break; + + // at this point, any failure means we can delete the current message + it++; + + // Scan for message start + if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { + LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); + fOk = false; + break; + } + + // Read header + CMessageHeader& hdr = msg.hdr; + if (!hdr.IsValid(chainparams.MessageStart())) + { + LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); + continue; + } + string strCommand = hdr.GetCommand(); + + // Message size + unsigned int nMessageSize = hdr.nMessageSize; + + // Checksum + CDataStream& vRecv = msg.vRecv; + const uint256& hash = msg.GetMessageHash(); + if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + { + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, + SanitizeString(strCommand), nMessageSize, + HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), + HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + continue; + } + + // Process message + bool fRet = false; + try + { + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman); + boost::this_thread::interruption_point(); + } + catch (const std::ios_base::failure& e) + { + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, string("error parsing message"))); + if (strstr(e.what(), "end of data")) + { + // Allow exceptions from under-length message on vRecv + LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + } + else if (strstr(e.what(), "size too large")) + { + // Allow exceptions from over-long size + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + } + else if (strstr(e.what(), "non-canonical ReadCompactSize()")) + { + // Allow exceptions from non-canonical encoding + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + } + else + { + PrintExceptionContinue(&e, "ProcessMessages()"); + } + } + catch (const boost::thread_interrupted&) { + throw; + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "ProcessMessages()"); + } catch (...) { + PrintExceptionContinue(NULL, "ProcessMessages()"); + } + + if (!fRet) + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); + + break; + } + + // In case the connection got shut down, its receive buffer was wiped + if (!pfrom->fDisconnect) + pfrom->vRecvMsg.erase(pfrom->vRecvMsg.begin(), it); + + return fOk; +} + +class CompareInvMempoolOrder +{ + CTxMemPool *mp; +public: + CompareInvMempoolOrder(CTxMemPool *_mempool) + { + mp = _mempool; + } + + bool operator()(std::set<uint256>::iterator a, std::set<uint256>::iterator b) + { + /* As std::make_heap produces a max-heap, we want the entries with the + * fewest ancestors/highest fee to sort later. */ + return mp->CompareDepthAndScore(*b, *a); + } +}; + +bool SendMessages(CNode* pto, CConnman& connman) +{ + const Consensus::Params& consensusParams = Params().GetConsensus(); + { + // Don't send anything until we get its version message + if (pto->nVersion == 0 || pto->fDisconnect) + return true; + + // If we get here, the outgoing message serialization version is set and can't change. + CNetMsgMaker msgMaker(pto->GetSendVersion()); + + // + // Message: ping + // + bool pingSend = false; + if (pto->fPingQueued) { + // RPC ping request by user + pingSend = true; + } + if (pto->nPingNonceSent == 0 && pto->nPingUsecStart + PING_INTERVAL * 1000000 < GetTimeMicros()) { + // Ping automatically sent as a latency probe & keepalive. + pingSend = true; + } + if (pingSend) { + uint64_t nonce = 0; + while (nonce == 0) { + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + } + pto->fPingQueued = false; + pto->nPingUsecStart = GetTimeMicros(); + if (pto->nVersion > BIP0031_VERSION) { + pto->nPingNonceSent = nonce; + connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); + } else { + // Peer is too old to support ping command with nonce, pong will never arrive. + pto->nPingNonceSent = 0; + connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); + } + } + + TRY_LOCK(cs_main, lockMain); // Acquire cs_main for IsInitialBlockDownload() and CNodeState() + if (!lockMain) + return true; + + CNodeState &state = *State(pto->GetId()); + + BOOST_FOREACH(const CBlockReject& reject, state.rejects) + connman.PushMessage(pto, msgMaker.Make(NetMsgType::REJECT, (string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock)); + state.rejects.clear(); + + if (state.fShouldBan) { + state.fShouldBan = false; + if (pto->fWhitelisted) + LogPrintf("Warning: not punishing whitelisted peer %s!\n", pto->addr.ToString()); + else { + pto->fDisconnect = true; + if (pto->addr.IsLocal()) + LogPrintf("Warning: not banning local peer %s!\n", pto->addr.ToString()); + else + { + connman.Ban(pto->addr, BanReasonNodeMisbehaving); + } + return true; + } + } + + // Address refresh broadcast + int64_t nNow = GetTimeMicros(); + if (!IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { + AdvertiseLocal(pto); + pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); + } + + // + // Message: addr + // + if (pto->nNextAddrSend < nNow) { + pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); + vector<CAddress> vAddr; + vAddr.reserve(pto->vAddrToSend.size()); + BOOST_FOREACH(const CAddress& addr, pto->vAddrToSend) + { + if (!pto->addrKnown.contains(addr.GetKey())) + { + pto->addrKnown.insert(addr.GetKey()); + vAddr.push_back(addr); + // receiver rejects addr messages larger than 1000 + if (vAddr.size() >= 1000) + { + connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + vAddr.clear(); + } + } + } + pto->vAddrToSend.clear(); + if (!vAddr.empty()) + connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + // we only send the big addr message once + if (pto->vAddrToSend.capacity() > 40) + pto->vAddrToSend.shrink_to_fit(); + } + + // Start block sync + if (pindexBestHeader == NULL) + pindexBestHeader = chainActive.Tip(); + bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); // Download if this is a nice peer, or we have no nice peers and this one might do. + if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { + // Only actively request headers from a single peer, unless we're close to today. + if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { + state.fSyncStarted = true; + nSyncStarted++; + const CBlockIndex *pindexStart = pindexBestHeader; + /* If possible, start at the block preceding the currently + best known header. This ensures that we always get a + non-empty list of headers back as long as the peer + is up-to-date. With a non-empty response, we can initialise + the peer's known best block. This wouldn't be possible + if we requested starting at pindexBestHeader and + got back an empty response. */ + if (pindexStart->pprev) + pindexStart = pindexStart->pprev; + LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); + } + } + + // Resend wallet transactions that haven't gotten in a block yet + // Except during reindex, importing and IBD, when old wallet + // transactions become unconfirmed and spams other nodes. + if (!fReindex && !fImporting && !IsInitialBlockDownload()) + { + GetMainSignals().Broadcast(nTimeBestReceived, &connman); + } + + // + // Try sending block announcements via headers + // + { + // If we have less than MAX_BLOCKS_TO_ANNOUNCE in our + // list of block hashes we're relaying, and our peer wants + // headers announcements, then find the first header + // not yet known to our peer but would connect, and send. + // If no header would connect, or if we have too many + // blocks, or if the peer doesn't want headers, just + // add all to the inv queue. + LOCK(pto->cs_inventory); + vector<CBlock> vHeaders; + bool fRevertToInv = ((!state.fPreferHeaders && + (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || + pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); + CBlockIndex *pBestIndex = NULL; // last header queued for delivery + ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date + + if (!fRevertToInv) { + bool fFoundStartingHeader = false; + // Try to find first header that our peer doesn't have, and + // then send all headers past that one. If we come across any + // headers that aren't on chainActive, give up. + BOOST_FOREACH(const uint256 &hash, pto->vBlockHashesToAnnounce) { + BlockMap::iterator mi = mapBlockIndex.find(hash); + assert(mi != mapBlockIndex.end()); + CBlockIndex *pindex = mi->second; + if (chainActive[pindex->nHeight] != pindex) { + // Bail out if we reorged away from this block + fRevertToInv = true; + break; + } + if (pBestIndex != NULL && pindex->pprev != pBestIndex) { + // This means that the list of blocks to announce don't + // connect to each other. + // This shouldn't really be possible to hit during + // regular operation (because reorgs should take us to + // a chain that has some block not on the prior chain, + // which should be caught by the prior check), but one + // way this could happen is by using invalidateblock / + // reconsiderblock repeatedly on the tip, causing it to + // be added multiple times to vBlockHashesToAnnounce. + // Robustly deal with this rare situation by reverting + // to an inv. + fRevertToInv = true; + break; + } + pBestIndex = pindex; + if (fFoundStartingHeader) { + // add this to the headers message + vHeaders.push_back(pindex->GetBlockHeader()); + } else if (PeerHasHeader(&state, pindex)) { + continue; // keep looking for the first new block + } else if (pindex->pprev == NULL || PeerHasHeader(&state, pindex->pprev)) { + // Peer doesn't have this header but they do have the prior one. + // Start sending headers. + fFoundStartingHeader = true; + vHeaders.push_back(pindex->GetBlockHeader()); + } else { + // Peer doesn't have this header or the prior one -- nothing will + // connect, so bail out. + fRevertToInv = true; + break; + } + } + } + if (!fRevertToInv && !vHeaders.empty()) { + if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { + // We only send up to 1 block as header-and-ids, as otherwise + // probably means we're doing an initial-ish-sync or they're slow + LogPrint("net", "%s sending header-and-ids %s to peer %d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + //TODO: Shouldn't need to reload block from disk, but requires refactor + CBlock block; + bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); + assert(ret); + CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); + int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; + connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + state.pindexBestHeaderSent = pBestIndex; + } else if (state.fPreferHeaders) { + if (vHeaders.size() > 1) { + LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, + vHeaders.size(), + vHeaders.front().GetHash().ToString(), + vHeaders.back().GetHash().ToString(), pto->id); + } else { + LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + } + connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + state.pindexBestHeaderSent = pBestIndex; + } else + fRevertToInv = true; + } + if (fRevertToInv) { + // If falling back to using an inv, just try to inv the tip. + // The last entry in vBlockHashesToAnnounce was our tip at some point + // in the past. + if (!pto->vBlockHashesToAnnounce.empty()) { + const uint256 &hashToAnnounce = pto->vBlockHashesToAnnounce.back(); + BlockMap::iterator mi = mapBlockIndex.find(hashToAnnounce); + assert(mi != mapBlockIndex.end()); + CBlockIndex *pindex = mi->second; + + // Warn if we're announcing a block that is not on the main chain. + // This should be very rare and could be optimized out. + // Just log for now. + if (chainActive[pindex->nHeight] != pindex) { + LogPrint("net", "Announcing block %s not on main chain (tip=%s)\n", + hashToAnnounce.ToString(), chainActive.Tip()->GetBlockHash().ToString()); + } + + // If the peer's chain has this block, don't inv it back. + if (!PeerHasHeader(&state, pindex)) { + pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); + LogPrint("net", "%s: sending inv peer=%d hash=%s\n", __func__, + pto->id, hashToAnnounce.ToString()); + } + } + } + pto->vBlockHashesToAnnounce.clear(); + } + + // + // Message: inventory + // + vector<CInv> vInv; + { + LOCK(pto->cs_inventory); + vInv.reserve(std::max<size_t>(pto->vInventoryBlockToSend.size(), INVENTORY_BROADCAST_MAX)); + + // Add blocks + BOOST_FOREACH(const uint256& hash, pto->vInventoryBlockToSend) { + vInv.push_back(CInv(MSG_BLOCK, hash)); + if (vInv.size() == MAX_INV_SZ) { + connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + vInv.clear(); + } + } + pto->vInventoryBlockToSend.clear(); + + // Check whether periodic sends should happen + bool fSendTrickle = pto->fWhitelisted; + if (pto->nNextInvSend < nNow) { + fSendTrickle = true; + // Use half the delay for outbound peers, as there is less privacy concern for them. + pto->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> !pto->fInbound); + } + + // Time to send but the peer has requested we not relay transactions. + if (fSendTrickle) { + LOCK(pto->cs_filter); + if (!pto->fRelayTxes) pto->setInventoryTxToSend.clear(); + } + + // Respond to BIP35 mempool requests + if (fSendTrickle && pto->fSendMempool) { + auto vtxinfo = mempool.infoAll(); + pto->fSendMempool = false; + CAmount filterrate = 0; + { + LOCK(pto->cs_feeFilter); + filterrate = pto->minFeeFilter; + } + + LOCK(pto->cs_filter); + + for (const auto& txinfo : vtxinfo) { + const uint256& hash = txinfo.tx->GetHash(); + CInv inv(MSG_TX, hash); + pto->setInventoryTxToSend.erase(hash); + if (filterrate) { + if (txinfo.feeRate.GetFeePerK() < filterrate) + continue; + } + if (pto->pfilter) { + if (!pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + } + pto->filterInventoryKnown.insert(hash); + vInv.push_back(inv); + if (vInv.size() == MAX_INV_SZ) { + connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + vInv.clear(); + } + } + pto->timeLastMempoolReq = GetTime(); + } + + // Determine transactions to relay + if (fSendTrickle) { + // Produce a vector with all candidates for sending + vector<std::set<uint256>::iterator> vInvTx; + vInvTx.reserve(pto->setInventoryTxToSend.size()); + for (std::set<uint256>::iterator it = pto->setInventoryTxToSend.begin(); it != pto->setInventoryTxToSend.end(); it++) { + vInvTx.push_back(it); + } + CAmount filterrate = 0; + { + LOCK(pto->cs_feeFilter); + filterrate = pto->minFeeFilter; + } + // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. + // A heap is used so that not all items need sorting if only a few are being sent. + CompareInvMempoolOrder compareInvMempoolOrder(&mempool); + std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); + // No reason to drain out at many times the network's capacity, + // especially since we have many peers and some will draw much shorter delays. + unsigned int nRelayedTransactions = 0; + LOCK(pto->cs_filter); + while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { + // Fetch the top element from the heap + std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); + std::set<uint256>::iterator it = vInvTx.back(); + vInvTx.pop_back(); + uint256 hash = *it; + // Remove it from the to-be-sent set + pto->setInventoryTxToSend.erase(it); + // Check if not in the filter already + if (pto->filterInventoryKnown.contains(hash)) { + continue; + } + // Not in the mempool anymore? don't bother sending it. + auto txinfo = mempool.info(hash); + if (!txinfo.tx) { + continue; + } + if (filterrate && txinfo.feeRate.GetFeePerK() < filterrate) { + continue; + } + if (pto->pfilter && !pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + // Send + vInv.push_back(CInv(MSG_TX, hash)); + nRelayedTransactions++; + { + // Expire old relay messages + while (!vRelayExpiration.empty() && vRelayExpiration.front().first < nNow) + { + mapRelay.erase(vRelayExpiration.front().second); + vRelayExpiration.pop_front(); + } + + auto ret = mapRelay.insert(std::make_pair(hash, std::move(txinfo.tx))); + if (ret.second) { + vRelayExpiration.push_back(std::make_pair(nNow + 15 * 60 * 1000000, ret.first)); + } + } + if (vInv.size() == MAX_INV_SZ) { + connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + vInv.clear(); + } + pto->filterInventoryKnown.insert(hash); + } + } + } + if (!vInv.empty()) + connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + + // Detect whether we're stalling + nNow = GetTimeMicros(); + if (state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) { + // Stalling only triggers when the block download window cannot move. During normal steady state, + // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection + // should only happen during initial block download. + LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); + pto->fDisconnect = true; + return true; + } + // In case there is a block that has been in flight from this peer for 2 + 0.5 * N times the block interval + // (with N the number of peers from which we're downloading validated blocks), disconnect due to timeout. + // We compensate for other peers to prevent killing off peers due to our own downstream link + // being saturated. We only count validated in-flight blocks so peers can't advertise non-existing block hashes + // to unreasonably increase our timeout. + if (state.vBlocksInFlight.size() > 0) { + QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); + int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); + if (nNow > state.nDownloadingSince + consensusParams.nPowTargetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { + LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); + pto->fDisconnect = true; + return true; + } + } + + // + // Message: getdata (blocks) + // + vector<CInv> vGetData; + if (!pto->fClient && (fFetch || !IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + vector<CBlockIndex*> vToDownload; + NodeId staller = -1; + FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); + BOOST_FOREACH(CBlockIndex *pindex, vToDownload) { + uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); + LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), + pindex->nHeight, pto->id); + } + if (state.nBlocksInFlight == 0 && staller != -1) { + if (State(staller)->nStallingSince == 0) { + State(staller)->nStallingSince = nNow; + LogPrint("net", "Stall started peer=%d\n", staller); + } + } + } + + // + // Message: getdata (non-blocks) + // + while (!pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) + { + const CInv& inv = (*pto->mapAskFor.begin()).second; + if (!AlreadyHave(inv)) + { + if (fDebug) + LogPrint("net", "Requesting %s peer=%d\n", inv.ToString(), pto->id); + vGetData.push_back(inv); + if (vGetData.size() >= 1000) + { + connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + vGetData.clear(); + } + } else { + //If we're not going to ask, don't expect a response. + pto->setAskFor.erase(inv.hash); + } + pto->mapAskFor.erase(pto->mapAskFor.begin()); + } + if (!vGetData.empty()) + connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + + // + // Message: feefilter + // + // We don't want white listed peers to filter txs to us if we have -whitelistforcerelay + if (pto->nVersion >= FEEFILTER_VERSION && GetBoolArg("-feefilter", DEFAULT_FEEFILTER) && + !(pto->fWhitelisted && GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY))) { + CAmount currentFilter = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); + int64_t timeNow = GetTimeMicros(); + if (timeNow > pto->nextSendTimeFeeFilter) { + static CFeeRate default_feerate(DEFAULT_MIN_RELAY_TX_FEE); + static FeeFilterRounder filterRounder(default_feerate); + CAmount filterToSend = filterRounder.round(currentFilter); + // If we don't allow free transactions, then we always have a fee filter of at least minRelayTxFee + if (GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) <= 0) + filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); + if (filterToSend != pto->lastSentFeeFilter) { + connman.PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); + pto->lastSentFeeFilter = filterToSend; + } + pto->nextSendTimeFeeFilter = PoissonNextSend(timeNow, AVG_FEEFILTER_BROADCAST_INTERVAL); + } + // If the fee filter has changed substantially and it's still more than MAX_FEEFILTER_CHANGE_DELAY + // until scheduled broadcast, then move the broadcast to within MAX_FEEFILTER_CHANGE_DELAY. + else if (timeNow + MAX_FEEFILTER_CHANGE_DELAY * 1000000 < pto->nextSendTimeFeeFilter && + (currentFilter < 3 * pto->lastSentFeeFilter / 4 || currentFilter > 4 * pto->lastSentFeeFilter / 3)) { + pto->nextSendTimeFeeFilter = timeNow + GetRandInt(MAX_FEEFILTER_CHANGE_DELAY) * 1000000; + } + } + } + return true; +} + +class CNetProcessingCleanup +{ +public: + CNetProcessingCleanup() {} + ~CNetProcessingCleanup() { + // orphan transactions + mapOrphanTransactions.clear(); + mapOrphanTransactionsByPrev.clear(); + } +} instance_of_cnetprocessingcleanup; diff --git a/src/net_processing.h b/src/net_processing.h new file mode 100644 index 0000000000..130433cc7c --- /dev/null +++ b/src/net_processing.h @@ -0,0 +1,51 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NET_PROCESSING_H +#define BITCOIN_NET_PROCESSING_H + +#include "net.h" +#include "validationinterface.h" + +/** Register with a network node to receive its signals */ +void RegisterNodeSignals(CNodeSignals& nodeSignals); +/** Unregister a network node */ +void UnregisterNodeSignals(CNodeSignals& nodeSignals); + +class PeerLogicValidation : public CValidationInterface { +private: + CConnman* connman; + +public: + PeerLogicValidation(CConnman* connmanIn); + + virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock); + virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); + virtual void BlockChecked(const CBlock& block, const CValidationState& state); +}; + +struct CNodeStateStats { + int nMisbehavior; + int nSyncHeight; + int nCommonHeight; + std::vector<int> vHeightInFlight; +}; + +/** Get statistics from node state */ +bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); +/** Increase a node's misbehavior score. */ +void Misbehaving(NodeId nodeid, int howmuch); + +/** Process protocol messages received from a given node */ +bool ProcessMessages(CNode* pfrom, CConnman& connman); +/** + * Send queued protocol messages to be sent to a give node. + * + * @param[in] pto The node which we are sending messages to. + * @param[in] connman The connection manager for that node. + */ +bool SendMessages(CNode* pto, CConnman& connman); + +#endif // BITCOIN_NET_PROCESSING_H diff --git a/src/netbase.cpp b/src/netbase.cpp index 9fe34108f5..da94fd4d13 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -94,30 +94,9 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign } } -#ifdef HAVE_GETADDRINFO_A - struct in_addr ipv4_addr; -#ifdef HAVE_INET_PTON - if (inet_pton(AF_INET, pszName, &ipv4_addr) > 0) { - vIP.push_back(CNetAddr(ipv4_addr)); - return true; - } - - struct in6_addr ipv6_addr; - if (inet_pton(AF_INET6, pszName, &ipv6_addr) > 0) { - vIP.push_back(CNetAddr(ipv6_addr)); - return true; - } -#else - ipv4_addr.s_addr = inet_addr(pszName); - if (ipv4_addr.s_addr != INADDR_NONE) { - vIP.push_back(CNetAddr(ipv4_addr)); - return true; - } -#endif -#endif - struct addrinfo aiHint; memset(&aiHint, 0, sizeof(struct addrinfo)); + aiHint.ai_socktype = SOCK_STREAM; aiHint.ai_protocol = IPPROTO_TCP; aiHint.ai_family = AF_UNSPEC; @@ -126,33 +105,8 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign #else aiHint.ai_flags = fAllowLookup ? AI_ADDRCONFIG : AI_NUMERICHOST; #endif - struct addrinfo *aiRes = NULL; -#ifdef HAVE_GETADDRINFO_A - struct gaicb gcb, *query = &gcb; - memset(query, 0, sizeof(struct gaicb)); - gcb.ar_name = pszName; - gcb.ar_request = &aiHint; - int nErr = getaddrinfo_a(GAI_NOWAIT, &query, 1, NULL); - if (nErr) - return false; - - do { - // Should set the timeout limit to a reasonable value to avoid - // generating unnecessary checking call during the polling loop, - // while it can still response to stop request quick enough. - // 2 seconds looks fine in our situation. - struct timespec ts = { 2, 0 }; - gai_suspend(&query, 1, &ts); - boost::this_thread::interruption_point(); - - nErr = gai_error(query); - if (0 == nErr) - aiRes = query->ar_result; - } while (nErr == EAI_INPROGRESS); -#else int nErr = getaddrinfo(pszName, NULL, &aiHint, &aiRes); -#endif if (nErr) return false; @@ -338,7 +292,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5Init.push_back(0x01); // # METHODS vSocks5Init.push_back(0x00); // X'00' NO AUTHENTICATION REQUIRED } - ssize_t ret = send(hSocket, (const char*)begin_ptr(vSocks5Init), vSocks5Init.size(), MSG_NOSIGNAL); + ssize_t ret = send(hSocket, (const char*)vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5Init.size()) { CloseSocket(hSocket); return error("Error sending to proxy"); @@ -363,7 +317,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); vAuth.push_back(auth->password.size()); vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); - ret = send(hSocket, (const char*)begin_ptr(vAuth), vAuth.size(), MSG_NOSIGNAL); + ret = send(hSocket, (const char*)vAuth.data(), vAuth.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vAuth.size()) { CloseSocket(hSocket); return error("Error sending authentication to proxy"); @@ -393,7 +347,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); vSocks5.push_back((port >> 8) & 0xFF); vSocks5.push_back((port >> 0) & 0xFF); - ret = send(hSocket, (const char*)begin_ptr(vSocks5), vSocks5.size(), MSG_NOSIGNAL); + ret = send(hSocket, (const char*)vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5.size()) { CloseSocket(hSocket); return error("Error sending to proxy"); diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 7113390cdf..9eb831bc17 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -404,7 +404,8 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) { // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget <= 1 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) return CFeeRate(0); double median = feeStats.EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); @@ -423,6 +424,10 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) return CFeeRate(0); + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + confTarget = 2; + double median = -1; while (median < 0 && (unsigned int)confTarget <= feeStats.GetMaxConfirms()) { median = feeStats.EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 3ad1ff7bae..0c71a079fb 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -7,7 +7,7 @@ #include "policy/policy.h" -#include "main.h" +#include "validation.h" #include "tinyformat.h" #include "util.h" #include "utilstrencodings.h" diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 91f4d29488..11d7eace55 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -62,9 +62,9 @@ uint256 CMutableTransaction::GetHash() const return SerializeHash(*this, SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); } -void CTransaction::UpdateHash() const +uint256 CTransaction::ComputeHash() const { - *const_cast<uint256*>(&hash) = SerializeHash(*this, SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); + return SerializeHash(*this, SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); } uint256 CTransaction::GetWitnessHash() const @@ -72,25 +72,10 @@ uint256 CTransaction::GetWitnessHash() const return SerializeHash(*this, SER_GETHASH, 0); } -CTransaction::CTransaction() : nVersion(CTransaction::CURRENT_VERSION), vin(), vout(), nLockTime(0) { } - -CTransaction::CTransaction(const CMutableTransaction &tx) : nVersion(tx.nVersion), vin(tx.vin), vout(tx.vout), wit(tx.wit), nLockTime(tx.nLockTime) { - UpdateHash(); -} - -CTransaction::CTransaction(CMutableTransaction &&tx) : nVersion(tx.nVersion), vin(std::move(tx.vin)), vout(std::move(tx.vout)), wit(std::move(tx.wit)), nLockTime(tx.nLockTime) { - UpdateHash(); -} - -CTransaction& CTransaction::operator=(const CTransaction &tx) { - *const_cast<int*>(&nVersion) = tx.nVersion; - *const_cast<std::vector<CTxIn>*>(&vin) = tx.vin; - *const_cast<std::vector<CTxOut>*>(&vout) = tx.vout; - *const_cast<CTxWitness*>(&wit) = tx.wit; - *const_cast<unsigned int*>(&nLockTime) = tx.nLockTime; - *const_cast<uint256*>(&hash) = tx.hash; - return *this; -} +/* For backward compatibility, the hash is initialized to 0. TODO: remove the need for this default constructor entirely. */ +CTransaction::CTransaction() : nVersion(CTransaction::CURRENT_VERSION), vin(), vout(), nLockTime(0), hash() {} +CTransaction::CTransaction(const CMutableTransaction &tx) : nVersion(tx.nVersion), vin(tx.vin), vout(tx.vout), wit(tx.wit), nLockTime(tx.nLockTime), hash(ComputeHash()) {} +CTransaction::CTransaction(CMutableTransaction &&tx) : nVersion(tx.nVersion), vin(std::move(tx.vin)), vout(std::move(tx.vout)), wit(std::move(tx.wit)), nLockTime(tx.nLockTime), hash(ComputeHash()) {} CAmount CTransaction::GetValueOut() const { diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 0fa85a1519..b594cab458 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -286,76 +286,84 @@ struct CMutableTransaction; * - CTxWitness wit; * - uint32_t nLockTime */ -template<typename Stream, typename Operation, typename TxType> -inline void SerializeTransaction(TxType& tx, Stream& s, Operation ser_action) { +template<typename Stream, typename TxType> +inline void UnserializeTransaction(TxType& tx, Stream& s) { const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); - READWRITE(*const_cast<int32_t*>(&tx.nVersion)); + s >> tx.nVersion; unsigned char flags = 0; - if (ser_action.ForRead()) { - const_cast<std::vector<CTxIn>*>(&tx.vin)->clear(); - const_cast<std::vector<CTxOut>*>(&tx.vout)->clear(); - const_cast<CTxWitness*>(&tx.wit)->SetNull(); - /* Try to read the vin. In case the dummy is there, this will be read as an empty vector. */ - READWRITE(*const_cast<std::vector<CTxIn>*>(&tx.vin)); - if (tx.vin.size() == 0 && fAllowWitness) { - /* We read a dummy or an empty vin. */ - READWRITE(flags); - if (flags != 0) { - READWRITE(*const_cast<std::vector<CTxIn>*>(&tx.vin)); - READWRITE(*const_cast<std::vector<CTxOut>*>(&tx.vout)); - } - } else { - /* We read a non-empty vin. Assume a normal vout follows. */ - READWRITE(*const_cast<std::vector<CTxOut>*>(&tx.vout)); - } - if ((flags & 1) && fAllowWitness) { - /* The witness flag is present, and we support witnesses. */ - flags ^= 1; - const_cast<CTxWitness*>(&tx.wit)->vtxinwit.resize(tx.vin.size()); - READWRITE(tx.wit); - } - if (flags) { - /* Unknown flag in the serialization */ - throw std::ios_base::failure("Unknown transaction optional data"); + tx.vin.clear(); + tx.vout.clear(); + tx.wit.SetNull(); + /* Try to read the vin. In case the dummy is there, this will be read as an empty vector. */ + s >> tx.vin; + if (tx.vin.size() == 0 && fAllowWitness) { + /* We read a dummy or an empty vin. */ + s >> flags; + if (flags != 0) { + s >> tx.vin; + s >> tx.vout; } } else { - // Consistency check - assert(tx.wit.vtxinwit.size() <= tx.vin.size()); - if (fAllowWitness) { - /* Check whether witnesses need to be serialized. */ - if (!tx.wit.IsNull()) { - flags |= 1; - } - } - if (flags) { - /* Use extended format in case witnesses are to be serialized. */ - std::vector<CTxIn> vinDummy; - READWRITE(vinDummy); - READWRITE(flags); + /* We read a non-empty vin. Assume a normal vout follows. */ + s >> tx.vout; + } + if ((flags & 1) && fAllowWitness) { + /* The witness flag is present, and we support witnesses. */ + flags ^= 1; + tx.wit.vtxinwit.resize(tx.vin.size()); + s >> tx.wit; + } + if (flags) { + /* Unknown flag in the serialization */ + throw std::ios_base::failure("Unknown transaction optional data"); + } + s >> tx.nLockTime; +} + +template<typename Stream, typename TxType> +inline void SerializeTransaction(const TxType& tx, Stream& s) { + const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); + + s << tx.nVersion; + unsigned char flags = 0; + // Consistency check + assert(tx.wit.vtxinwit.size() <= tx.vin.size()); + if (fAllowWitness) { + /* Check whether witnesses need to be serialized. */ + if (!tx.wit.IsNull()) { + flags |= 1; } - READWRITE(*const_cast<std::vector<CTxIn>*>(&tx.vin)); - READWRITE(*const_cast<std::vector<CTxOut>*>(&tx.vout)); - if (flags & 1) { - const_cast<CTxWitness*>(&tx.wit)->vtxinwit.resize(tx.vin.size()); - READWRITE(tx.wit); + } + if (flags) { + /* Use extended format in case witnesses are to be serialized. */ + std::vector<CTxIn> vinDummy; + s << vinDummy; + s << flags; + } + s << tx.vin; + s << tx.vout; + if (flags & 1) { + for (size_t i = 0; i < tx.vin.size(); i++) { + if (i < tx.wit.vtxinwit.size()) { + s << tx.wit.vtxinwit[i]; + } else { + s << CTxInWitness(); + } } } - READWRITE(*const_cast<uint32_t*>(&tx.nLockTime)); + s << tx.nLockTime; } + /** The basic transaction that is broadcasted on the network and contained in * blocks. A transaction can contain multiple inputs and outputs. */ class CTransaction { -private: - /** Memory only. */ - const uint256 hash; - public: // Default transaction version. - static const int32_t CURRENT_VERSION=1; + static const int32_t CURRENT_VERSION=2; // Changing the default transaction version requires a two step process: first // adapting relay policy by bumping MAX_STANDARD_VERSION, and then later date @@ -374,6 +382,13 @@ public: CTxWitness wit; // Not const: can change without invalidating the txid cache const uint32_t nLockTime; +private: + /** Memory only. */ + const uint256 hash; + + uint256 ComputeHash() const; + +public: /** Construct a CTransaction that qualifies as IsNull() */ CTransaction(); @@ -381,18 +396,13 @@ public: CTransaction(const CMutableTransaction &tx); CTransaction(CMutableTransaction &&tx); - CTransaction& operator=(const CTransaction& tx); - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - SerializeTransaction(*this, s, ser_action); - if (ser_action.ForRead()) { - UpdateHash(); - } + template <typename Stream> + inline void Serialize(Stream& s) const { + SerializeTransaction(*this, s); } + /** This deserializing constructor is provided instead of an Unserialize method. + * Unserialize is not possible, since it would require overwriting const fields. */ template <typename Stream> CTransaction(deserialize_type, Stream& s) : CTransaction(CMutableTransaction(deserialize, s)) {} @@ -417,7 +427,7 @@ public: // Compute modified tx size for priority calculation (optionally given tx size) unsigned int CalculateModifiedSize(unsigned int nTxSize=0) const; - + /** * Get the total transaction size in bytes, including witness data. * "Total Size" defined in BIP141 and BIP144. @@ -441,8 +451,6 @@ public: } std::string ToString() const; - - void UpdateHash() const; }; /** A mutable version of CTransaction. */ @@ -457,11 +465,15 @@ struct CMutableTransaction CMutableTransaction(); CMutableTransaction(const CTransaction& tx); - ADD_SERIALIZE_METHODS; + template <typename Stream> + inline void Serialize(Stream& s) const { + SerializeTransaction(*this, s); + } - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - SerializeTransaction(*this, s, ser_action); + + template <typename Stream> + inline void Unserialize(Stream& s) { + UnserializeTransaction(*this, s); } template <typename Stream> diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 4f48e21a22..cfd0955a74 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -30,6 +30,7 @@ #include "scheduler.h" #include "ui_interface.h" #include "util.h" +#include "warnings.h" #ifdef ENABLE_WALLET #include "wallet/wallet.h" @@ -260,7 +261,7 @@ BitcoinCore::BitcoinCore(): void BitcoinCore::handleRunawayException(const std::exception *e) { PrintExceptionContinue(e, "Runaway exception"); - Q_EMIT runawayException(QString::fromStdString(strMiscWarning)); + Q_EMIT runawayException(QString::fromStdString(GetWarnings("gui"))); } void BitcoinCore::initialize() @@ -691,10 +692,10 @@ int main(int argc, char *argv[]) app.exec(); } catch (const std::exception& e) { PrintExceptionContinue(&e, "Runaway exception"); - app.handleRunawayException(QString::fromStdString(strMiscWarning)); + app.handleRunawayException(QString::fromStdString(GetWarnings("gui"))); } catch (...) { PrintExceptionContinue(NULL, "Runaway exception"); - app.handleRunawayException(QString::fromStdString(strMiscWarning)); + app.handleRunawayException(QString::fromStdString(GetWarnings("gui"))); } return app.getReturnValue(); } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 6112a1d256..651ff84293 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -46,7 +46,6 @@ #include <QMenuBar> #include <QMessageBox> #include <QMimeData> -#include <QProgressBar> #include <QProgressDialog> #include <QSettings> #include <QShortcut> @@ -199,8 +198,8 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *_platformStyle, const NetworkStyle * unitDisplayControl = new UnitDisplayStatusBarControl(platformStyle); labelWalletEncryptionIcon = new QLabel(); labelWalletHDStatusIcon = new QLabel(); - connectionsControl = new NetworkToggleStatusBarControl(); - labelBlocksIcon = new QLabel(); + connectionsControl = new GUIUtil::ClickableLabel(); + labelBlocksIcon = new GUIUtil::ClickableLabel(); if(enableWallet) { frameBlocksLayout->addStretch(); @@ -244,10 +243,15 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *_platformStyle, const NetworkStyle * // Subscribe to notifications from core subscribeToCoreSignals(); + connect(connectionsControl, SIGNAL(clicked(QPoint)), this, SLOT(toggleNetworkActive())); + modalOverlay = new ModalOverlay(this->centralWidget()); #ifdef ENABLE_WALLET - if(enableWallet) + if(enableWallet) { connect(walletFrame, SIGNAL(requestedSyncWarningInfo()), this, SLOT(showModalOverlay())); + connect(labelBlocksIcon, SIGNAL(clicked(QPoint)), this, SLOT(showModalOverlay())); + connect(progressBar, SIGNAL(clicked(QPoint)), this, SLOT(showModalOverlay())); + } #endif } @@ -490,7 +494,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) } #endif // ENABLE_WALLET unitDisplayControl->setOptionsModel(_clientModel->getOptionsModel()); - connectionsControl->setClientModel(_clientModel); OptionsModel* optionsModel = _clientModel->getOptionsModel(); if(optionsModel) @@ -517,7 +520,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) walletFrame->setClientModel(nullptr); #endif // ENABLE_WALLET unitDisplayControl->setOptionsModel(nullptr); - connectionsControl->setClientModel(nullptr); } } @@ -1136,8 +1138,8 @@ void BitcoinGUI::setTrayIconVisible(bool fHideTrayIcon) void BitcoinGUI::showModalOverlay() { - if (modalOverlay) - modalOverlay->showHide(false, true); + if (modalOverlay && (progressBar->isVisible() || modalOverlay->isLayerVisible())) + modalOverlay->toggleVisibility(); } static bool ThreadSafeMessageBox(BitcoinGUI *gui, const std::string& message, const std::string& caption, unsigned int style) @@ -1171,6 +1173,13 @@ void BitcoinGUI::unsubscribeFromCoreSignals() uiInterface.ThreadSafeQuestion.disconnect(boost::bind(ThreadSafeMessageBox, this, _1, _3, _4)); } +void BitcoinGUI::toggleNetworkActive() +{ + if (clientModel) { + clientModel->setNetworkActive(!clientModel->getNetworkActive()); + } +} + UnitDisplayStatusBarControl::UnitDisplayStatusBarControl(const PlatformStyle *platformStyle) : optionsModel(0), menu(0) @@ -1244,16 +1253,3 @@ void UnitDisplayStatusBarControl::onMenuSelection(QAction* action) optionsModel->setDisplayUnit(action->data()); } } - -void NetworkToggleStatusBarControl::mousePressEvent(QMouseEvent *event) -{ - if (clientModel) { - clientModel->setNetworkActive(!clientModel->getNetworkActive()); - } -} - -/** Lets the control know about the Client Model */ -void NetworkToggleStatusBarControl::setClientModel(ClientModel *_clientModel) -{ - this->clientModel = _clientModel; -} diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 1b02e77fc4..59540bfe6b 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -26,7 +26,6 @@ class PlatformStyle; class RPCConsole; class SendCoinsRecipient; class UnitDisplayStatusBarControl; -class NetworkToggleStatusBarControl; class WalletFrame; class WalletModel; class HelpMessageDialog; @@ -86,7 +85,7 @@ private: UnitDisplayStatusBarControl *unitDisplayControl; QLabel *labelWalletEncryptionIcon; QLabel *labelWalletHDStatusIcon; - NetworkToggleStatusBarControl *connectionsControl; + QLabel *connectionsControl; QLabel *labelBlocksIcon; QLabel *progressBarLabel; QProgressBar *progressBar; @@ -238,6 +237,9 @@ private Q_SLOTS: /** When hideTrayIcon setting is changed in OptionsModel hide or show the icon accordingly. */ void setTrayIconVisible(bool); + /** Toggle networking */ + void toggleNetworkActive(); + void showModalOverlay(); }; @@ -270,17 +272,4 @@ private Q_SLOTS: void onMenuSelection(QAction* action); }; -class NetworkToggleStatusBarControl : public QLabel -{ - Q_OBJECT - -public: - void setClientModel(ClientModel *clientModel); -protected: - void mousePressEvent(QMouseEvent *event); - -private: - ClientModel *clientModel; -}; - #endif // BITCOIN_QT_BITCOINGUI_H diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index a4bb2f77fe..e63a9cc7b8 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -12,6 +12,7 @@ #include "chainparams.h" #include "checkpoints.h" #include "clientversion.h" +#include "validation.h" #include "net.h" #include "txmempool.h" #include "ui_interface.h" diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index d77db39b3c..4d34bf95e9 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -15,7 +15,7 @@ #include "wallet/coincontrol.h" #include "init.h" -#include "main.h" // For minRelayTxFee +#include "validation.h" // For minRelayTxFee #include "wallet/wallet.h" #include <boost/assign/list_of.hpp> // for 'map_list_of()' @@ -470,21 +470,21 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nQuantity++; // Amount - nAmount += out.tx->vout[out.i].nValue; + nAmount += out.tx->tx->vout[out.i].nValue; // Priority - dPriorityInputs += (double)out.tx->vout[out.i].nValue * (out.nDepth+1); + dPriorityInputs += (double)out.tx->tx->vout[out.i].nValue * (out.nDepth+1); // Bytes CTxDestination address; int witnessversion = 0; std::vector<unsigned char> witnessprogram; - if (out.tx->vout[out.i].scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) + if (out.tx->tx->vout[out.i].scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { nBytesInputs += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); fWitness = true; } - else if(ExtractDestination(out.tx->vout[out.i].scriptPubKey, address)) + else if(ExtractDestination(out.tx->tx->vout[out.i].scriptPubKey, address)) { CPubKey pubkey; CKeyID *keyid = boost::get<CKeyID>(&address); @@ -677,7 +677,7 @@ void CoinControlDialog::updateView() CAmount nSum = 0; int nChildren = 0; BOOST_FOREACH(const COutput& out, coins.second) { - nSum += out.tx->vout[out.i].nValue; + nSum += out.tx->tx->vout[out.i].nValue; nChildren++; CCoinControlWidgetItem *itemOutput; @@ -689,7 +689,7 @@ void CoinControlDialog::updateView() // address CTxDestination outputAddress; QString sAddress = ""; - if(ExtractDestination(out.tx->vout[out.i].scriptPubKey, outputAddress)) + if(ExtractDestination(out.tx->tx->vout[out.i].scriptPubKey, outputAddress)) { sAddress = QString::fromStdString(CBitcoinAddress(outputAddress).ToString()); @@ -714,8 +714,8 @@ void CoinControlDialog::updateView() } // amount - itemOutput->setText(COLUMN_AMOUNT, BitcoinUnits::format(nDisplayUnit, out.tx->vout[out.i].nValue)); - itemOutput->setData(COLUMN_AMOUNT, Qt::UserRole, QVariant((qlonglong)out.tx->vout[out.i].nValue)); // padding so that sorting works correctly + itemOutput->setText(COLUMN_AMOUNT, BitcoinUnits::format(nDisplayUnit, out.tx->tx->vout[out.i].nValue)); + itemOutput->setData(COLUMN_AMOUNT, Qt::UserRole, QVariant((qlonglong)out.tx->tx->vout[out.i].nValue)); // padding so that sorting works correctly // date itemOutput->setText(COLUMN_DATE, GUIUtil::dateTimeStr(out.tx->GetTxTime())); diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 33db9f8938..0a8afa2e76 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -1064,7 +1064,7 @@ <number>0</number> </property> <property name="maximum"> - <number>24</number> + <number>23</number> </property> <property name="pageStep"> <number>1</number> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 130cfc6e7d..8132e4fe0d 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -11,7 +11,7 @@ #include "primitives/transaction.h" #include "init.h" -#include "main.h" // For minRelayTxFee +#include "validation.h" // For minRelayTxFee #include "protocol.h" #include "script/script.h" #include "script/standard.h" @@ -55,6 +55,7 @@ #include <QSettings> #include <QTextDocument> // for Qt::mightBeRichText #include <QThread> +#include <QMouseEvent> #if QT_VERSION < 0x050000 #include <QUrl> @@ -116,7 +117,7 @@ static std::string DummyAddress(const CChainParams ¶ms) std::vector<unsigned char> sourcedata = params.Base58Prefix(CChainParams::PUBKEY_ADDRESS); sourcedata.insert(sourcedata.end(), dummydata, dummydata + sizeof(dummydata)); for(int i=0; i<256; ++i) { // Try every trailing byte - std::string s = EncodeBase58(begin_ptr(sourcedata), end_ptr(sourcedata)); + std::string s = EncodeBase58(sourcedata.data(), sourcedata.data() + sourcedata.size()); if (!CBitcoinAddress(s).IsValid()) return s; sourcedata[sourcedata.size()-1] += 1; @@ -986,4 +987,15 @@ QString formateNiceTimeOffset(qint64 secs) } return timeBehindText; } + +void ClickableLabel::mouseReleaseEvent(QMouseEvent *event) +{ + Q_EMIT clicked(event->pos()); +} + +void ClickableProgressBar::mouseReleaseEvent(QMouseEvent *event) +{ + Q_EMIT clicked(event->pos()); +} + } // namespace GUIUtil diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 8f1f3fbb2c..4ea2aa36e7 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -14,6 +14,7 @@ #include <QProgressBar> #include <QString> #include <QTableView> +#include <QLabel> #include <boost/filesystem.hpp> @@ -201,18 +202,44 @@ namespace GUIUtil QString formateNiceTimeOffset(qint64 secs); + class ClickableLabel : public QLabel + { + Q_OBJECT + + Q_SIGNALS: + /** Emitted when the label is clicked. The relative mouse coordinates of the click are + * passed to the signal. + */ + void clicked(const QPoint& point); + protected: + void mouseReleaseEvent(QMouseEvent *event); + }; + + class ClickableProgressBar : public QProgressBar + { + Q_OBJECT + + Q_SIGNALS: + /** Emitted when the progressbar is clicked. The relative mouse coordinates of the click are + * passed to the signal. + */ + void clicked(const QPoint& point); + protected: + void mouseReleaseEvent(QMouseEvent *event); + }; + #if defined(Q_OS_MAC) && QT_VERSION >= 0x050000 // workaround for Qt OSX Bug: // https://bugreports.qt-project.org/browse/QTBUG-15631 // QProgressBar uses around 10% CPU even when app is in background - class ProgressBar : public QProgressBar + class ProgressBar : public ClickableProgressBar { bool event(QEvent *e) { return (e->type() != QEvent::StyleAnimationUpdate) ? QProgressBar::event(e) : false; } }; #else - typedef QProgressBar ProgressBar; + typedef ClickableProgressBar ProgressBar; #endif } // namespace GUIUtil diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 1a843a07ac..3e8282e63d 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -137,6 +137,13 @@ void ModalOverlay::tipUpdate(int count, const QDateTime& blockDate, double nVeri } } +void ModalOverlay::toggleVisibility() +{ + showHide(layerIsVisible, true); + if (!layerIsVisible) + userClosed = true; +} + void ModalOverlay::showHide(bool hide, bool userRequested) { if ( (layerIsVisible && !hide) || (!layerIsVisible && hide) || (!hide && userClosed && !userRequested)) diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index 66c0aa78cf..70d37b87ad 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -25,9 +25,11 @@ public Q_SLOTS: void tipUpdate(int count, const QDateTime& blockDate, double nVerificationProgress); void setKnownBestHeight(int count, const QDateTime& blockDate); + void toggleVisibility(); // will show or hide the modal layer void showHide(bool hide = false, bool userRequested = false); void closeClicked(); + bool isLayerVisible() { return layerIsVisible; } protected: bool eventFilter(QObject * obj, QEvent * ev); diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 588059d0c5..2d5e9ff686 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -13,7 +13,7 @@ #include "guiutil.h" #include "optionsmodel.h" -#include "main.h" // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS +#include "validation.h" // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS #include "netbase.h" #include "txdb.h" // for -dbcache defaults diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index f82e153b67..d48c4d91e7 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -13,7 +13,7 @@ #include "amount.h" #include "init.h" -#include "main.h" // For DEFAULT_SCRIPTCHECK_THREADS +#include "validation.h" // For DEFAULT_SCRIPTCHECK_THREADS #include "net.h" #include "netbase.h" #include "txdb.h" // for -dbcache defaults diff --git a/src/qt/paymentrequestplus.cpp b/src/qt/paymentrequestplus.cpp index 20e1f79ffa..82be4d831f 100644 --- a/src/qt/paymentrequestplus.cpp +++ b/src/qt/paymentrequestplus.cpp @@ -159,14 +159,24 @@ bool PaymentRequestPlus::getMerchant(X509_STORE* certStore, QString& merchant) c std::string data_to_verify; // Everything but the signature rcopy.SerializeToString(&data_to_verify); - EVP_MD_CTX ctx; +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + if (!ctx) throw SSLVerifyError("Error allocating OpenSSL context."); +#else + EVP_MD_CTX _ctx; + EVP_MD_CTX *ctx; + ctx = &_ctx; +#endif EVP_PKEY *pubkey = X509_get_pubkey(signing_cert); - EVP_MD_CTX_init(&ctx); - if (!EVP_VerifyInit_ex(&ctx, digestAlgorithm, NULL) || - !EVP_VerifyUpdate(&ctx, data_to_verify.data(), data_to_verify.size()) || - !EVP_VerifyFinal(&ctx, (const unsigned char*)paymentRequest.signature().data(), (unsigned int)paymentRequest.signature().size(), pubkey)) { + EVP_MD_CTX_init(ctx); + if (!EVP_VerifyInit_ex(ctx, digestAlgorithm, NULL) || + !EVP_VerifyUpdate(ctx, data_to_verify.data(), data_to_verify.size()) || + !EVP_VerifyFinal(ctx, (const unsigned char*)paymentRequest.signature().data(), (unsigned int)paymentRequest.signature().size(), pubkey)) { throw SSLVerifyError("Bad signature, invalid payment request."); } +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + EVP_MD_CTX_free(ctx); +#endif // OpenSSL API for getting human printable strings from certs is baroque. int textlen = X509_NAME_get_text_by_NID(certname, NID_commonName, NULL, 0); diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 229752cad2..d5f0156d6c 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -10,7 +10,7 @@ #include "base58.h" #include "chainparams.h" -#include "main.h" // For minRelayTxFee +#include "validation.h" // For minRelayTxFee #include "ui_interface.h" #include "util.h" #include "wallet/wallet.h" diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index c0b4900285..dd4bd55396 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -8,6 +8,7 @@ #include "guiconstants.h" #include "guiutil.h" +#include "validation.h" // for cs_main #include "sync.h" #include <QDebug> diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index af34b147b1..3e4f0a68c4 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_PEERTABLEMODEL_H #define BITCOIN_QT_PEERTABLEMODEL_H -#include "main.h" // For CNodeStateStats +#include "net_processing.h" // For CNodeStateStats #include "net.h" #include <QAbstractTableModel> diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 520d229901..cfe197de2a 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -137,6 +137,8 @@ bool RPCConsole::RPCExecuteCommandLine(std::string &strResult, const std::string enum CmdParseState { STATE_EATING_SPACES, + STATE_EATING_SPACES_IN_ARG, + STATE_EATING_SPACES_IN_BRACKETS, STATE_ARGUMENT, STATE_SINGLEQUOTED, STATE_DOUBLEQUOTED, @@ -220,6 +222,8 @@ bool RPCConsole::RPCExecuteCommandLine(std::string &strResult, const std::string break; } case STATE_ARGUMENT: // In or after argument + case STATE_EATING_SPACES_IN_ARG: + case STATE_EATING_SPACES_IN_BRACKETS: case STATE_EATING_SPACES: // Handle runs of whitespace switch(ch) { @@ -227,19 +231,20 @@ bool RPCConsole::RPCExecuteCommandLine(std::string &strResult, const std::string case '\'': state = STATE_SINGLEQUOTED; break; case '\\': state = STATE_ESCAPE_OUTER; break; case '(': case ')': case '\n': + if (state == STATE_EATING_SPACES_IN_ARG) + throw std::runtime_error("Invalid Syntax"); if (state == STATE_ARGUMENT) { if (ch == '(' && stack.size() && stack.back().size() > 0) stack.push_back(std::vector<std::string>()); - if (curarg.size()) - { - // don't allow commands after executed commands on baselevel - if (!stack.size()) - throw std::runtime_error("Invalid Syntax"); - stack.back().push_back(curarg); - } + + // don't allow commands after executed commands on baselevel + if (!stack.size()) + throw std::runtime_error("Invalid Syntax"); + + stack.back().push_back(curarg); curarg.clear(); - state = STATE_EATING_SPACES; + state = STATE_EATING_SPACES_IN_BRACKETS; } if ((ch == ')' || ch == '\n') && stack.size() > 0) { @@ -256,12 +261,19 @@ bool RPCConsole::RPCExecuteCommandLine(std::string &strResult, const std::string } break; case ' ': case ',': case '\t': - if(state == STATE_ARGUMENT) // Space ends argument + if(state == STATE_EATING_SPACES_IN_ARG && curarg.empty() && ch == ',') + throw std::runtime_error("Invalid Syntax"); + + else if(state == STATE_ARGUMENT) // Space ends argument { - if (curarg.size()) - stack.back().push_back(curarg); + stack.back().push_back(curarg); curarg.clear(); } + if ((state == STATE_EATING_SPACES_IN_BRACKETS || state == STATE_ARGUMENT) && ch == ',') + { + state = STATE_EATING_SPACES_IN_ARG; + break; + } state = STATE_EATING_SPACES; break; default: curarg += ch; state = STATE_ARGUMENT; @@ -515,7 +527,7 @@ void RPCConsole::setClientModel(ClientModel *model) // peer table signal handling - update peer details when new nodes are added to the model connect(model->getPeerTableModel(), SIGNAL(layoutChanged()), this, SLOT(peerLayoutChanged())); // peer table signal handling - cache selected node ids - connect(model->getPeerTableModel(), SIGNAL(layoutAboutToChange()), this, SLOT(peerLayoutAboutToChange())); + connect(model->getPeerTableModel(), SIGNAL(layoutAboutToBeChanged()), this, SLOT(peerLayoutAboutToChange())); // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); @@ -650,13 +662,18 @@ void RPCConsole::clear(bool clearHistory) "td.message { font-family: %1; font-size: %2; white-space:pre-wrap; } " "td.cmd-request { color: #006060; } " "td.cmd-error { color: red; } " + ".secwarning { color: red; }" "b { color: #006060; } " ).arg(fixedFontInfo.family(), QString("%1pt").arg(consoleFontSize)) ); message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + tr("Use up and down arrows to navigate history, and <b>Ctrl-L</b> to clear screen.") + "<br>" + - tr("Type <b>help</b> for an overview of available commands.")), true); + tr("Type <b>help</b> for an overview of available commands.")) + + "<br><span class=\"secwarning\">" + + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramification of a command.") + + "</span>", + true); } void RPCConsole::keyPressEvent(QKeyEvent *event) @@ -778,7 +795,6 @@ void RPCConsole::startExecutor() connect(this, SIGNAL(stopExecutor()), &thread, SLOT(quit())); // - queue executor for deletion (in execution thread) connect(&thread, SIGNAL(finished()), executor, SLOT(deleteLater()), Qt::DirectConnection); - connect(&thread, SIGNAL(finished()), this, SLOT(test()), Qt::DirectConnection); // Default implementation of QThread::run() simply spins up an event loop in the thread, // which is what we want. @@ -1008,11 +1024,11 @@ void RPCConsole::disconnectSelectedNode() return; // Get selected peer addresses - QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, 0); + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data(PeerTableModel::NetNodeId).toInt(); + NodeId id = nodes.at(i).data().toInt(); // Find the node, disconnect it and clear the selected node if(g_connman->DisconnectNode(id)) clearSelectedNode(); @@ -1025,11 +1041,11 @@ void RPCConsole::banSelectedNode(int bantime) return; // Get selected peer addresses - QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, 0); + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data(PeerTableModel::NetNodeId).toInt(); + NodeId id = nodes.at(i).data().toInt(); // Get currently selected peer address int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); @@ -1052,11 +1068,11 @@ void RPCConsole::unbanSelectedNode() return; // Get selected ban addresses - QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->banlistWidget, 0); + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->banlistWidget, BanTableModel::Address); for(int i = 0; i < nodes.count(); i++) { // Get currently selected ban address - QString strNode = nodes.at(i).data(BanTableModel::Address).toString(); + QString strNode = nodes.at(i).data().toString(); CSubNet possibleSubnet; LookupSubNet(strNode.toStdString().c_str(), possibleSubnet); @@ -1090,8 +1106,3 @@ void RPCConsole::setTabFocus(enum TabTypes tabType) { ui->tabWidget->setCurrentIndex(tabType); } - -void RPCConsole::on_toggleNetworkActiveButton_clicked() -{ - clientModel->setNetworkActive(!clientModel->getNetworkActive()); -} diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 344d5ecb98..ab8c3dc914 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -62,8 +62,6 @@ protected: private Q_SLOTS: void on_lineEdit_returnPressed(); void on_tabWidget_currentChanged(int index); - /** toggle network activity */ - void on_toggleNetworkActiveButton_clicked(); /** open the debug.log from the current datadir */ void on_openDebugLogfileButton_clicked(); /** change the time range of the network traffic graph */ diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 57b2179435..13210a1adc 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -17,7 +17,7 @@ #include "base58.h" #include "wallet/coincontrol.h" -#include "main.h" // mempool and minRelayTxFee +#include "validation.h" // mempool and minRelayTxFee #include "ui_interface.h" #include "txmempool.h" #include "wallet/wallet.h" @@ -175,7 +175,7 @@ void SendCoinsDialog::setModel(WalletModel *_model) // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; if (settings.value("nSmartFeeSliderPosition").toInt() == 0) - ui->sliderSmartFee->setValue(ui->sliderSmartFee->maximum() - model->getDefaultConfirmTarget() + 1); + ui->sliderSmartFee->setValue(ui->sliderSmartFee->maximum() - model->getDefaultConfirmTarget() + 2); else ui->sliderSmartFee->setValue(settings.value("nSmartFeeSliderPosition").toInt()); } @@ -241,7 +241,7 @@ void SendCoinsDialog::on_sendButton_clicked() if (model->getOptionsModel()->getCoinControlFeatures()) ctrl = *CoinControlDialog::coinControl; if (ui->radioSmartFee->isChecked()) - ctrl.nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; + ctrl.nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; else ctrl.nConfirmTarget = 0; @@ -601,7 +601,7 @@ void SendCoinsDialog::updateGlobalFeeVariables() { if (ui->radioSmartFee->isChecked()) { - int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; + int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; payTxFee = CFeeRate(0); // set nMinimumTotalFee to 0 to not accidentally pay a custom fee @@ -646,7 +646,7 @@ void SendCoinsDialog::updateSmartFeeLabel() if(!model || !model->getOptionsModel()) return; - int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; + int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; int estimateFoundAtBlocks = nBlocksToConfirm; CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); if (feeRate <= CFeeRate(0)) // not enough data => minfee diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 3e42f3a7b0..e28a6b47af 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -12,7 +12,7 @@ #include "base58.h" #include "init.h" -#include "main.h" // For strMessageMagic +#include "validation.h" // For strMessageMagic #include "wallet/wallet.h" #include <string> diff --git a/src/qt/test/compattests.cpp b/src/qt/test/compattests.cpp new file mode 100644 index 0000000000..2a7284b5b2 --- /dev/null +++ b/src/qt/test/compattests.cpp @@ -0,0 +1,23 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "paymentrequestplus.h" // this includes protobuf's port.h which defines its own bswap macos + +#include "compattests.h" + +#include "compat/byteswap.h" + +void CompatTests::bswapTests() +{ + // Sibling in bitcoin/src/test/bswap_tests.cpp + uint16_t u1 = 0x1234; + uint32_t u2 = 0x56789abc; + uint64_t u3 = 0xdef0123456789abc; + uint16_t e1 = 0x3412; + uint32_t e2 = 0xbc9a7856; + uint64_t e3 = 0xbc9a78563412f0de; + QVERIFY(bswap_16(u1) == e1); + QVERIFY(bswap_32(u2) == e2); + QVERIFY(bswap_64(u3) == e3); +} diff --git a/src/qt/test/compattests.h b/src/qt/test/compattests.h new file mode 100644 index 0000000000..35dede7743 --- /dev/null +++ b/src/qt/test/compattests.h @@ -0,0 +1,19 @@ +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_TEST_COMPATTESTS_H +#define BITCOIN_QT_TEST_COMPATTESTS_H + +#include <QObject> +#include <QTest> + +class CompatTests : public QObject +{ + Q_OBJECT + +private Q_SLOTS: + void bswapTests(); +}; + +#endif // BITCOIN_QT_TEST_COMPATTESTS_H diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 3dae33bafb..26ac5e2114 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -6,7 +6,7 @@ #include "chainparams.h" #include "consensus/validation.h" -#include "main.h" +#include "validation.h" #include "rpc/register.h" #include "rpc/server.h" #include "rpcconsole.h" @@ -15,9 +15,23 @@ #include "util.h" #include <QDir> +#include <QtGlobal> #include <boost/filesystem.hpp> +static UniValue rpcNestedTest_rpc(const JSONRPCRequest& request) +{ + if (request.fHelp) { + return "help message"; + } + return request.params.write(0, 0); +} + +static const CRPCCommand vRPCCommands[] = +{ + { "test", "rpcNestedTest", &rpcNestedTest_rpc, true }, +}; + void RPCNestedTests::rpcNestedTests() { UniValue jsonRPCError; @@ -26,6 +40,7 @@ void RPCNestedTests::rpcNestedTests() // could be moved to a more generic place when we add more tests on QT level const CChainParams& chainparams = Params(); RegisterAllCoreRPCCommands(tableRPC); + tableRPC.appendCommand("rpcNestedTest", &vRPCCommands[0]); ClearDatadirCache(); std::string path = QDir::tempPath().toStdString() + "/" + strprintf("test_bitcoin_qt_%lu_%i", (unsigned long)GetTime(), (int)(GetRand(100000))); QDir dir(QString::fromStdString(path)); @@ -63,16 +78,6 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo "); //whitespace at the end will be tolerated QVERIFY(result.substr(0,1) == "{"); -#if QT_VERSION >= 0x050300 - // do the QVERIFY_EXCEPTION_THROWN checks only with Qt5.3 and higher (QVERIFY_EXCEPTION_THROWN was introduced in Qt5.3) - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax - (RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo(")); //tolerate non closing brackets if we have no arguments - (RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo()()()")); //tolerate non command brackts - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo(True)"), UniValue); //invalid argument - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "a(getblockchaininfo(True))"), UniValue); //method not found -#endif - (RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo()[\"chain\"]")); //Quote path identifier are allowed, but look after a child contaning the quotes in the key QVERIFY(result == "null"); @@ -85,6 +90,40 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCExecuteCommandLine(result, "getblock(getbestblockhash())[tx][0]"); QVERIFY(result == "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest"); + QVERIFY(result == "[]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest ''"); + QVERIFY(result == "[\"\"]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest \"\""); + QVERIFY(result == "[\"\"]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest '' abc"); + QVERIFY(result == "[\"\",\"abc\"]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest abc '' abc"); + QVERIFY(result == "[\"abc\",\"\",\"abc\"]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest abc abc"); + QVERIFY(result == "[\"abc\",\"abc\"]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest abc\t\tabc"); + QVERIFY(result == "[\"abc\",\"abc\"]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest(abc )"); + QVERIFY(result == "[\"abc\"]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest( abc )"); + QVERIFY(result == "[\"abc\"]"); + RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest( abc , cba )"); + QVERIFY(result == "[\"abc\",\"cba\"]"); + +#if QT_VERSION >= 0x050300 + // do the QVERIFY_EXCEPTION_THROWN checks only with Qt5.3 and higher (QVERIFY_EXCEPTION_THROWN was introduced in Qt5.3) + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax + (RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo(")); //tolerate non closing brackets if we have no arguments + (RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo()()()")); //tolerate non command brackts + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "getblockchaininfo(True)"), UniValue); //invalid argument + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "a(getblockchaininfo(True))"), UniValue); //method not found + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest abc,,abc"), std::runtime_error); //don't tollerate empty arguments when using , + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest(abc,,abc)"), std::runtime_error); //don't tollerate empty arguments when using , + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest(abc,,)"), std::runtime_error); //don't tollerate empty arguments when using , +#endif + delete pcoinsTip; delete pcoinsdbview; delete pblocktree; diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index dbaab54fb6..d44d711315 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -11,6 +11,7 @@ #include "rpcnestedtests.h" #include "util.h" #include "uritests.h" +#include "compattests.h" #ifdef ENABLE_WALLET #include "paymentservertests.h" @@ -61,6 +62,9 @@ int main(int argc, char *argv[]) RPCNestedTests test3; if (QTest::qExec(&test3) != 0) fInvalid = true; + CompatTests test4; + if (QTest::qExec(&test4) != 0) + fInvalid = true; ECC_Stop(); return fInvalid; diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 65144e7865..e7d03a6119 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -11,7 +11,7 @@ #include "base58.h" #include "consensus/consensus.h" -#include "main.h" +#include "validation.h" #include "script/script.h" #include "timedata.h" #include "util.h" @@ -26,10 +26,10 @@ QString TransactionDesc::FormatTxStatus(const CWalletTx& wtx) AssertLockHeld(cs_main); if (!CheckFinalTx(wtx)) { - if (wtx.nLockTime < LOCKTIME_THRESHOLD) - return tr("Open for %n more block(s)", "", wtx.nLockTime - chainActive.Height()); + if (wtx.tx->nLockTime < LOCKTIME_THRESHOLD) + return tr("Open for %n more block(s)", "", wtx.tx->nLockTime - chainActive.Height()); else - return tr("Open until %1").arg(GUIUtil::dateTimeStr(wtx.nLockTime)); + return tr("Open until %1").arg(GUIUtil::dateTimeStr(wtx.tx->nLockTime)); } else { @@ -133,7 +133,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco // Coinbase // CAmount nUnmatured = 0; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) nUnmatured += wallet->GetCredit(txout, ISMINE_ALL); strHTML += "<b>" + tr("Credit") + ":</b> "; if (wtx.IsInMainChain()) @@ -152,14 +152,14 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco else { isminetype fAllFromMe = ISMINE_SPENDABLE; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { isminetype mine = wallet->IsMine(txin); if(fAllFromMe > mine) fAllFromMe = mine; } isminetype fAllToMe = ISMINE_SPENDABLE; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { isminetype mine = wallet->IsMine(txout); if(fAllToMe > mine) fAllToMe = mine; @@ -173,7 +173,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco // // Debit // - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { // Ignore change isminetype toSelf = wallet->IsMine(txout); @@ -212,7 +212,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco strHTML += "<b>" + tr("Total credit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, nValue) + "<br>"; } - CAmount nTxFee = nDebit - wtx.GetValueOut(); + CAmount nTxFee = nDebit - wtx.tx->GetValueOut(); if (nTxFee > 0) strHTML += "<b>" + tr("Transaction fee") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, -nTxFee) + "<br>"; } @@ -221,10 +221,10 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco // // Mixed debit transaction // - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) if (wallet->IsMine(txin)) strHTML += "<b>" + tr("Debit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, -wallet->GetDebit(txin, ISMINE_ALL)) + "<br>"; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) if (wallet->IsMine(txout)) strHTML += "<b>" + tr("Credit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, wallet->GetCredit(txout, ISMINE_ALL)) + "<br>"; } @@ -241,7 +241,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco strHTML += "<br><b>" + tr("Comment") + ":</b><br>" + GUIUtil::HtmlEscape(wtx.mapValue["comment"], true) + "<br>"; strHTML += "<b>" + tr("Transaction ID") + ":</b> " + rec->getTxID() + "<br>"; - strHTML += "<b>" + tr("Transaction total size") + ":</b> " + QString::number(wtx.GetTotalSize()) + " bytes<br>"; + strHTML += "<b>" + tr("Transaction total size") + ":</b> " + QString::number(wtx.tx->GetTotalSize()) + " bytes<br>"; strHTML += "<b>" + tr("Output index") + ":</b> " + QString::number(rec->getOutputIndex()) + "<br>"; // Message from normal bitcoin:URI (bitcoin:123...?message=example) @@ -276,20 +276,20 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco if (fDebug) { strHTML += "<hr><br>" + tr("Debug information") + "<br><br>"; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) if(wallet->IsMine(txin)) strHTML += "<b>" + tr("Debit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, -wallet->GetDebit(txin, ISMINE_ALL)) + "<br>"; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) if(wallet->IsMine(txout)) strHTML += "<b>" + tr("Credit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, wallet->GetCredit(txout, ISMINE_ALL)) + "<br>"; strHTML += "<br><b>" + tr("Transaction") + ":</b><br>"; - strHTML += GUIUtil::HtmlEscape(wtx.ToString(), true); + strHTML += GUIUtil::HtmlEscape(wtx.tx->ToString(), true); strHTML += "<br><b>" + tr("Inputs") + ":</b>"; strHTML += "<ul>"; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { COutPoint prevout = txin.prevout; diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 4fe47181f6..a1d1422317 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -6,7 +6,7 @@ #include "base58.h" #include "consensus/consensus.h" -#include "main.h" +#include "validation.h" #include "timedata.h" #include "wallet/wallet.h" @@ -47,7 +47,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet * // // Credit // - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { isminetype mine = wallet->IsMine(txout); if(mine) @@ -83,7 +83,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet * { bool involvesWatchAddress = false; isminetype fAllFromMe = ISMINE_SPENDABLE; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { isminetype mine = wallet->IsMine(txin); if(mine & ISMINE_WATCH_ONLY) involvesWatchAddress = true; @@ -91,7 +91,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet * } isminetype fAllToMe = ISMINE_SPENDABLE; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { isminetype mine = wallet->IsMine(txout); if(mine & ISMINE_WATCH_ONLY) involvesWatchAddress = true; @@ -112,11 +112,11 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet * // // Debit // - CAmount nTxFee = nDebit - wtx.GetValueOut(); + CAmount nTxFee = nDebit - wtx.tx->GetValueOut(); - for (unsigned int nOut = 0; nOut < wtx.vout.size(); nOut++) + for (unsigned int nOut = 0; nOut < wtx.tx->vout.size(); nOut++) { - const CTxOut& txout = wtx.vout[nOut]; + const CTxOut& txout = wtx.tx->vout[nOut]; TransactionRecord sub(hash, nTime); sub.idx = parts.size(); sub.involvesWatchAddress = involvesWatchAddress; @@ -190,15 +190,15 @@ void TransactionRecord::updateStatus(const CWalletTx &wtx) if (!CheckFinalTx(wtx)) { - if (wtx.nLockTime < LOCKTIME_THRESHOLD) + if (wtx.tx->nLockTime < LOCKTIME_THRESHOLD) { status.status = TransactionStatus::OpenUntilBlock; - status.open_for = wtx.nLockTime - chainActive.Height(); + status.open_for = wtx.tx->nLockTime - chainActive.Height(); } else { status.status = TransactionStatus::OpenUntilDate; - status.open_for = wtx.nLockTime; + status.open_for = wtx.tx->nLockTime; } } // For generated transactions, determine maturity diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 52261ff04b..da0742aa6a 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -14,7 +14,7 @@ #include "walletmodel.h" #include "core_io.h" -#include "main.h" +#include "validation.h" #include "sync.h" #include "uint256.h" #include "util.h" diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 3490d1c1cc..afc72fae69 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -14,9 +14,11 @@ #include "base58.h" #include "keystore.h" -#include "main.h" +#include "validation.h" +#include "net.h" // for g_connman #include "sync.h" #include "ui_interface.h" +#include "util.h" // for GetBoolArg #include "wallet/wallet.h" #include "wallet/walletdb.h" // for BackupWallet @@ -65,7 +67,7 @@ CAmount WalletModel::getBalance(const CCoinControl *coinControl) const wallet->AvailableCoins(vCoins, true, coinControl); BOOST_FOREACH(const COutput& out, vCoins) if(out.fSpendable) - nBalance += out.tx->vout[out.i].nValue; + nBalance += out.tx->tx->vout[out.i].nValue; return nBalance; } @@ -333,9 +335,8 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran if(!wallet->CommitTransaction(*newTx, *keyChange, g_connman.get(), state)) return SendCoinsReturn(TransactionCommitFailed, QString::fromStdString(state.GetRejectReason())); - CTransaction* t = (CTransaction*)newTx; CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << *t; + ssTx << *newTx->tx; transaction_array.append(&(ssTx[0]), ssTx.size()); } @@ -607,7 +608,7 @@ void WalletModel::listCoins(std::map<QString, std::vector<COutput> >& mapCoins) int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); if (nDepth < 0) continue; COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true, true); - if (outpoint.n < out.tx->vout.size() && wallet->IsMine(out.tx->vout[outpoint.n]) == ISMINE_SPENDABLE) + if (outpoint.n < out.tx->tx->vout.size() && wallet->IsMine(out.tx->tx->vout[outpoint.n]) == ISMINE_SPENDABLE) vCoins.push_back(out); } @@ -615,14 +616,14 @@ void WalletModel::listCoins(std::map<QString, std::vector<COutput> >& mapCoins) { COutput cout = out; - while (wallet->IsChange(cout.tx->vout[cout.i]) && cout.tx->vin.size() > 0 && wallet->IsMine(cout.tx->vin[0])) + while (wallet->IsChange(cout.tx->tx->vout[cout.i]) && cout.tx->tx->vin.size() > 0 && wallet->IsMine(cout.tx->tx->vin[0])) { - if (!wallet->mapWallet.count(cout.tx->vin[0].prevout.hash)) break; - cout = COutput(&wallet->mapWallet[cout.tx->vin[0].prevout.hash], cout.tx->vin[0].prevout.n, 0, true, true); + if (!wallet->mapWallet.count(cout.tx->tx->vin[0].prevout.hash)) break; + cout = COutput(&wallet->mapWallet[cout.tx->tx->vin[0].prevout.hash], cout.tx->tx->vin[0].prevout.n, 0, true, true); } CTxDestination address; - if(!out.fSpendable || !ExtractDestination(cout.tx->vout[cout.i].scriptPubKey, address)) + if(!out.fSpendable || !ExtractDestination(cout.tx->tx->vout[cout.i].scriptPubKey, address)) continue; mapCoins[QString::fromStdString(CBitcoinAddress(address).ToString())].push_back(out); } diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index fdec6a1c86..f9a95bed42 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -64,7 +64,7 @@ void WalletModelTransaction::reassignAmounts(int nChangePosRet) if (out.amount() <= 0) continue; if (i == nChangePosRet) i++; - subtotal += walletTransaction->vout[i].nValue; + subtotal += walletTransaction->tx->vout[i].nValue; i++; } rcp.amount = subtotal; @@ -73,7 +73,7 @@ void WalletModelTransaction::reassignAmounts(int nChangePosRet) { if (i == nChangePosRet) i++; - rcp.amount = walletTransaction->vout[i].nValue; + rcp.amount = walletTransaction->tx->vout[i].nValue; i++; } } diff --git a/src/random.cpp b/src/random.cpp index aa027e49c4..c2605b45bd 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -11,7 +11,6 @@ #include "compat.h" // for Windows API #include <wincrypt.h> #endif -#include "serialize.h" // for begin_ptr(vec) #include "util.h" // for LogPrint() #include "utilstrencodings.h" // for GetTime() @@ -72,15 +71,15 @@ static void RandAddSeedPerfmon() const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data while (true) { nSize = vData.size(); - ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", NULL, NULL, begin_ptr(vData), &nSize); + ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", NULL, NULL, vData.data(), &nSize); if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) break; vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially } RegCloseKey(HKEY_PERFORMANCE_DATA); if (ret == ERROR_SUCCESS) { - RAND_add(begin_ptr(vData), nSize, nSize / 100.0); - memory_cleanse(begin_ptr(vData), nSize); + RAND_add(vData.data(), nSize, nSize / 100.0); + memory_cleanse(vData.data(), nSize); LogPrint("rand", "%s: %lu bytes\n", __func__, nSize); } else { static bool warned = false; // Warn only once diff --git a/src/rest.cpp b/src/rest.cpp index 90cca6f480..6b6b7401f6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -7,7 +7,7 @@ #include "chainparams.h" #include "primitives/block.h" #include "primitives/transaction.h" -#include "main.h" +#include "validation.h" #include "httpserver.h" #include "rpc/server.h" #include "streams.h" @@ -228,7 +228,7 @@ static bool rest_block(HTTPRequest* req, return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; switch (rf) { @@ -363,12 +363,12 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) if (!ParseHashStr(hashStr, hash)) return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); - CTransaction tx; + CTransactionRef tx; uint256 hashBlock = uint256(); if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; switch (rf) { @@ -388,7 +388,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) case RF_JSON: { UniValue objTx(UniValue::VOBJ); - TxToJSON(tx, hashBlock, objTx); + TxToJSON(*tx, hashBlock, objTx); string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 733d14d24e..1139c2aa5c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -9,7 +9,7 @@ #include "checkpoints.h" #include "coins.h" #include "consensus/validation.h" -#include "main.h" +#include "validation.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" @@ -751,7 +751,7 @@ UniValue getblock(const JSONRPCRequest& request) if (!fVerbose) { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); return strHex; @@ -1319,7 +1319,7 @@ UniValue invalidateblock(const JSONRPCRequest& request) } if (state.IsValid()) { - ActivateBestChain(state, Params(), NULL); + ActivateBestChain(state, Params()); } if (!state.IsValid()) { @@ -1357,7 +1357,7 @@ UniValue reconsiderblock(const JSONRPCRequest& request) } CValidationState state; - ActivateBestChain(state, Params(), NULL); + ActivateBestChain(state, Params()); if (!state.IsValid()) { throw JSONRPCError(RPC_DATABASE_ERROR, state.GetRejectReason()); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index f3cd1fbf0b..73797e2019 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -12,7 +12,7 @@ #include "consensus/validation.h" #include "core_io.h" #include "init.h" -#include "main.h" +#include "validation.h" #include "miner.h" #include "net.h" #include "pow.h" @@ -131,7 +131,8 @@ UniValue generateBlocks(boost::shared_ptr<CReserveScript> coinbaseScript, int nG if (pblock->nNonce == nInnerLoopCount) { continue; } - if (!ProcessNewBlock(Params(), pblock, true, NULL, NULL)) + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); + if (!ProcessNewBlock(Params(), shared_pblock, true, NULL)) throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); ++nHeight; blockHashes.push_back(pblock->GetHash().GetHex()); @@ -728,7 +729,8 @@ UniValue submitblock(const JSONRPCRequest& request) + HelpExampleRpc("submitblock", "\"mydata\"") ); - CBlock block; + std::shared_ptr<CBlock> blockptr = std::make_shared<CBlock>(); + CBlock& block = *blockptr; if (!DecodeHexBlk(block, request.params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); @@ -758,7 +760,7 @@ UniValue submitblock(const JSONRPCRequest& request) submitblock_StateCatcher sc(block.GetHash()); RegisterValidationInterface(&sc); - bool fAccepted = ProcessNewBlock(Params(), &block, true, NULL, NULL); + bool fAccepted = ProcessNewBlock(Params(), blockptr, true, NULL); UnregisterValidationInterface(&sc); if (fBlockPresent) { @@ -785,6 +787,8 @@ UniValue estimatefee(const JSONRPCRequest& request) "\n" "A negative value is returned if not enough transactions and blocks\n" "have been observed to make an estimate.\n" + "-1 is always returned for nblocks == 1 as it is impossible to calculate\n" + "a fee that is high enough to get reliably included in the next block.\n" "\nExample:\n" + HelpExampleCli("estimatefee", "6") ); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 3193985803..2aaee7f3f5 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -6,7 +6,7 @@ #include "base58.h" #include "clientversion.h" #include "init.h" -#include "main.h" +#include "validation.h" #include "net.h" #include "netbase.h" #include "rpc/server.h" diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index f57ba76d3a..53c0f993dc 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -6,8 +6,9 @@ #include "chainparams.h" #include "clientversion.h" -#include "main.h" +#include "validation.h" #include "net.h" +#include "net_processing.h" #include "netbase.h" #include "protocol.h" #include "sync.h" diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 370c021ea6..1229a00c5f 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -10,7 +10,7 @@ #include "core_io.h" #include "init.h" #include "keystore.h" -#include "main.h" +#include "validation.h" #include "merkleblock.h" #include "net.h" #include "policy/policy.h" @@ -218,19 +218,19 @@ UniValue getrawtransaction(const JSONRPCRequest& request) } } - CTransaction tx; + CTransactionRef tx; uint256 hashBlock; if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); - string strHex = EncodeHexTx(tx); + string strHex = EncodeHexTx(*tx, RPCSerializationFlags()); if (!fVerbose) return strHex; UniValue result(UniValue::VOBJ); result.push_back(Pair("hex", strHex)); - TxToJSON(tx, hashBlock, result); + TxToJSON(*tx, hashBlock, result); return result; } @@ -289,7 +289,7 @@ UniValue gettxoutproof(const JSONRPCRequest& request) if (pblockindex == NULL) { - CTransaction tx; + CTransactionRef tx; if (!GetTransaction(oneTxid, tx, Params().GetConsensus(), hashBlock, false) || hashBlock.IsNull()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); if (!mapBlockIndex.count(hashBlock)) @@ -520,13 +520,13 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) LOCK(cs_main); RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VSTR)); - CTransaction tx; + CMutableTransaction mtx; - if (!DecodeHexTx(tx, request.params[0].get_str(), true)) + if (!DecodeHexTx(mtx, request.params[0].get_str(), true)) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); UniValue result(UniValue::VOBJ); - TxToJSON(tx, uint256(), result); + TxToJSON(CTransaction(std::move(mtx)), uint256(), result); return result; } @@ -883,9 +883,10 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VSTR)(UniValue::VBOOL)); // parse hex string from parameter - CTransaction tx; - if (!DecodeHexTx(tx, request.params[0].get_str())) + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, request.params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); + CTransaction tx(std::move(mtx)); uint256 hashTx = tx.GetHash(); bool fLimitFree = false; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 164e0f00e2..2122819398 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -497,4 +497,12 @@ void RPCRunLater(const std::string& name, boost::function<void(void)> func, int6 deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); } +int RPCSerializationFlags() +{ + int flag = 0; + if (GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) + flag |= SERIALIZE_TRANSACTION_NO_WITNESS; + return flag; +} + CRPCTable tableRPC; diff --git a/src/rpc/server.h b/src/rpc/server.h index c59886222c..4fac68a51f 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -19,6 +19,8 @@ #include <univalue.h> +static const unsigned int DEFAULT_RPC_SERIALIZE_VERSION = 1; + class CRPCCommand; namespace RPCServer @@ -198,4 +200,7 @@ void StopRPC(); std::string JSONRPCExecBatch(const UniValue& vReq); void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); +// Retrieves any serialization flags requested in command line argument +int RPCSerializationFlags(); + #endif // BITCOIN_RPCSERVER_H diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 069ac55bfb..036d6ca7bf 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -85,8 +85,7 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP } try { TxInputStream stream(SER_NETWORK, PROTOCOL_VERSION, txTo, txToLen); - CTransaction tx; - stream >> tx; + CTransaction tx(deserialize, stream); if (nIn >= tx.vin.size()) return set_error(err, bitcoinconsensus_ERR_TX_INDEX); if (GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION) != txToLen) diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index a6403f9363..1410d0b73f 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -856,15 +856,15 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, un valtype& vch = stacktop(-1); valtype vchHash((opcode == OP_RIPEMD160 || opcode == OP_SHA1 || opcode == OP_HASH160) ? 20 : 32); if (opcode == OP_RIPEMD160) - CRIPEMD160().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CRIPEMD160().Write(vch.data(), vch.size()).Finalize(vchHash.data()); else if (opcode == OP_SHA1) - CSHA1().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CSHA1().Write(vch.data(), vch.size()).Finalize(vchHash.data()); else if (opcode == OP_SHA256) - CSHA256().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CSHA256().Write(vch.data(), vch.size()).Finalize(vchHash.data()); else if (opcode == OP_HASH160) - CHash160().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CHash160().Write(vch.data(), vch.size()).Finalize(vchHash.data()); else if (opcode == OP_HASH256) - CHash256().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CHash256().Write(vch.data(), vch.size()).Finalize(vchHash.data()); popstack(stack); stack.push_back(vchHash); } diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index bdc0bfdc1c..b78d7b607f 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -11,20 +11,29 @@ #include "uint256.h" #include "util.h" +#include "cuckoocache.h" #include <boost/thread.hpp> -#include <boost/unordered_set.hpp> namespace { /** * We're hashing a nonce into the entries themselves, so we don't need extra * blinding in the set hash computation. + * + * This may exhibit platform endian dependent behavior but because these are + * nonced hashes (random) and this state is only ever used locally it is safe. + * All that matters is local consistency. */ -class CSignatureCacheHasher +class SignatureCacheHasher { public: - size_t operator()(const uint256& key) const { - return key.GetCheapHash(); + template <uint8_t hash_select> + uint32_t operator()(const uint256& key) const + { + static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); + uint32_t u; + std::memcpy(&u, key.begin()+4*hash_select, 4); + return u; } }; @@ -38,11 +47,10 @@ class CSignatureCache private: //! Entries are SHA256(nonce || signature hash || public key || signature): uint256 nonce; - typedef boost::unordered_set<uint256, CSignatureCacheHasher> map_type; + typedef CuckooCache::cache<uint256, SignatureCacheHasher> map_type; map_type setValid; boost::shared_mutex cs_sigcache; - public: CSignatureCache() { @@ -56,58 +64,51 @@ public: } bool - Get(const uint256& entry) + Get(const uint256& entry, const bool erase) { boost::shared_lock<boost::shared_mutex> lock(cs_sigcache); - return setValid.count(entry); + return setValid.contains(entry, erase); } - void Erase(const uint256& entry) + void Set(uint256& entry) { boost::unique_lock<boost::shared_mutex> lock(cs_sigcache); - setValid.erase(entry); + setValid.insert(entry); } - - void Set(const uint256& entry) + uint32_t setup_bytes(size_t n) { - size_t nMaxCacheSize = GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); - if (nMaxCacheSize <= 0) return; - - boost::unique_lock<boost::shared_mutex> lock(cs_sigcache); - while (memusage::DynamicUsage(setValid) > nMaxCacheSize) - { - map_type::size_type s = GetRand(setValid.bucket_count()); - map_type::local_iterator it = setValid.begin(s); - if (it != setValid.end(s)) { - setValid.erase(*it); - } - } - - setValid.insert(entry); + return setValid.setup_bytes(n); } }; +/* In previous versions of this code, signatureCache was a local static variable + * in CachingTransactionSignatureChecker::VerifySignature. We initialize + * signatureCache outside of VerifySignature to avoid the atomic operation per + * call overhead associated with local static variables even though + * signatureCache could be made local to VerifySignature. +*/ +static CSignatureCache signatureCache; } -bool CachingTransactionSignatureChecker::VerifySignature(const std::vector<unsigned char>& vchSig, const CPubKey& pubkey, const uint256& sighash) const +// To be called once in AppInit2/TestingSetup to initialize the signatureCache +void InitSignatureCache() { - static CSignatureCache signatureCache; + size_t nMaxCacheSize = GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); + if (nMaxCacheSize <= 0) return; + size_t nElems = signatureCache.setup_bytes(nMaxCacheSize); + LogPrintf("Using %zu MiB out of %zu requested for signature cache, able to store %zu elements\n", + (nElems*sizeof(uint256)) >>20, nMaxCacheSize>>20, nElems); +} +bool CachingTransactionSignatureChecker::VerifySignature(const std::vector<unsigned char>& vchSig, const CPubKey& pubkey, const uint256& sighash) const +{ uint256 entry; signatureCache.ComputeEntry(entry, sighash, vchSig, pubkey); - - if (signatureCache.Get(entry)) { - if (!store) { - signatureCache.Erase(entry); - } + if (signatureCache.Get(entry, !store)) return true; - } - if (!TransactionSignatureChecker::VerifySignature(vchSig, pubkey, sighash)) return false; - - if (store) { + if (store) signatureCache.Set(entry); - } return true; } diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 44551ec2bc..5243fc0a42 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -10,9 +10,10 @@ #include <vector> -// DoS prevention: limit cache size to less than 40MB (over 500000 -// entries on 64-bit systems). -static const unsigned int DEFAULT_MAX_SIG_CACHE_SIZE = 40; +// DoS prevention: limit cache size to 32MB (over 1000000 entries on 64-bit +// systems). Due to how we count cache size, actual memory usage is slightly +// more (~32.25 MB) +static const unsigned int DEFAULT_MAX_SIG_CACHE_SIZE = 32; class CPubKey; @@ -27,4 +28,6 @@ public: bool VerifySignature(const std::vector<unsigned char>& vchSig, const CPubKey& vchPubKey, const uint256& sighash) const; }; +void InitSignatureCache(); + #endif // BITCOIN_SCRIPT_SIGCACHE_H diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index efb277d347..87fea161ba 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -6,6 +6,7 @@ bench_schnorr_verify bench_recover bench_internal tests +exhaustive_tests gen_context *.exe *.so diff --git a/src/secp256k1/.travis.yml b/src/secp256k1/.travis.yml index 2c5c63adad..2439529242 100644 --- a/src/secp256k1/.travis.yml +++ b/src/secp256k1/.travis.yml @@ -11,7 +11,7 @@ cache: - src/java/guava/ env: global: - - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no STATICPRECOMPUTATION=yes ASM=no BUILD=check EXTRAFLAGS= HOST= ECDH=no schnorr=no RECOVERY=no EXPERIMENTAL=no + - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no STATICPRECOMPUTATION=yes ASM=no BUILD=check EXTRAFLAGS= HOST= ECDH=no RECOVERY=no EXPERIMENTAL=no - GUAVA_URL=https://search.maven.org/remotecontent?filepath=com/google/guava/guava/18.0/guava-18.0.jar GUAVA_JAR=src/java/guava/guava-18.0.jar matrix: - SCALAR=32bit RECOVERY=yes @@ -22,15 +22,14 @@ env: - FIELD=64bit ENDOMORPHISM=yes ECDH=yes EXPERIMENTAL=yes - FIELD=64bit ASM=x86_64 - FIELD=64bit ENDOMORPHISM=yes ASM=x86_64 - - FIELD=32bit SCHNORR=yes EXPERIMENTAL=yes - FIELD=32bit ENDOMORPHISM=yes - BIGNUM=no - - BIGNUM=no ENDOMORPHISM=yes SCHNORR=yes RECOVERY=yes EXPERIMENTAL=yes + - BIGNUM=no ENDOMORPHISM=yes RECOVERY=yes EXPERIMENTAL=yes - BIGNUM=no STATICPRECOMPUTATION=no - BUILD=distcheck - EXTRAFLAGS=CPPFLAGS=-DDETERMINISTIC - EXTRAFLAGS=CFLAGS=-O0 - - BUILD=check-java ECDH=yes SCHNORR=yes EXPERIMENTAL=yes + - BUILD=check-java ECDH=yes EXPERIMENTAL=yes matrix: fast_finish: true include: @@ -66,5 +65,5 @@ before_script: ./autogen.sh script: - if [ -n "$HOST" ]; then export USE_HOST="--host=$HOST"; fi - if [ "x$HOST" = "xi686-linux-gnu" ]; then export CC="$CC -m32"; fi - - ./configure --enable-experimental=$EXPERIMENTAL --enable-endomorphism=$ENDOMORPHISM --with-field=$FIELD --with-bignum=$BIGNUM --with-scalar=$SCALAR --enable-ecmult-static-precomputation=$STATICPRECOMPUTATION --enable-module-ecdh=$ECDH --enable-module-schnorr=$SCHNORR --enable-module-recovery=$RECOVERY $EXTRAFLAGS $USE_HOST && make -j2 $BUILD + - ./configure --enable-experimental=$EXPERIMENTAL --enable-endomorphism=$ENDOMORPHISM --with-field=$FIELD --with-bignum=$BIGNUM --with-scalar=$SCALAR --enable-ecmult-static-precomputation=$STATICPRECOMPUTATION --enable-module-ecdh=$ECDH --enable-module-recovery=$RECOVERY $EXTRAFLAGS $USE_HOST && make -j2 $BUILD os: linux diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 3d130bdcbd..e5657f7f31 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -12,9 +12,11 @@ noinst_HEADERS = noinst_HEADERS += src/scalar.h noinst_HEADERS += src/scalar_4x64.h noinst_HEADERS += src/scalar_8x32.h +noinst_HEADERS += src/scalar_low.h noinst_HEADERS += src/scalar_impl.h noinst_HEADERS += src/scalar_4x64_impl.h noinst_HEADERS += src/scalar_8x32_impl.h +noinst_HEADERS += src/scalar_low_impl.h noinst_HEADERS += src/group.h noinst_HEADERS += src/group_impl.h noinst_HEADERS += src/num_gmp.h @@ -87,13 +89,23 @@ bench_internal_LDADD = $(SECP_LIBS) $(COMMON_LIB) bench_internal_CPPFLAGS = -DSECP256K1_BUILD $(SECP_INCLUDES) endif +TESTS = if USE_TESTS noinst_PROGRAMS += tests tests_SOURCES = src/tests.c tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) tests_LDADD = $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) tests_LDFLAGS = -static -TESTS = tests +TESTS += tests +endif + +if USE_EXHAUSTIVE_TESTS +noinst_PROGRAMS += exhaustive_tests +exhaustive_tests_SOURCES = src/tests_exhaustive.c +exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src $(SECP_INCLUDES) +exhaustive_tests_LDADD = $(SECP_LIBS) +exhaustive_tests_LDFLAGS = -static +TESTS += exhaustive_tests endif JAVAROOT=src/java @@ -154,10 +166,6 @@ if ENABLE_MODULE_ECDH include src/modules/ecdh/Makefile.am.include endif -if ENABLE_MODULE_SCHNORR -include src/modules/schnorr/Makefile.am.include -endif - if ENABLE_MODULE_RECOVERY include src/modules/recovery/Makefile.am.include endif diff --git a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 index b25d8adb92..b74acb8c13 100644 --- a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 +++ b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 @@ -46,6 +46,10 @@ if test x"$has_libcrypto" = x"yes" && test x"$has_openssl_ec" = x; then ECDSA_sign(0, NULL, 0, NULL, NULL, eckey); ECDSA_verify(0, NULL, 0, NULL, 0, eckey); EC_KEY_free(eckey); + ECDSA_SIG *sig_openssl; + sig_openssl = ECDSA_SIG_new(); + (void)sig_openssl->r; + ECDSA_SIG_free(sig_openssl); ]])],[has_openssl_ec=yes],[has_openssl_ec=no]) AC_MSG_RESULT([$has_openssl_ec]) fi diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 0743c36690..ec50ffe3a2 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -104,6 +104,11 @@ AC_ARG_ENABLE(experimental, [use_experimental=$enableval], [use_experimental=no]) +AC_ARG_ENABLE(exhaustive_tests, + AS_HELP_STRING([--enable-exhaustive-tests],[compile exhaustive tests (default is yes)]), + [use_exhaustive_tests=$enableval], + [use_exhaustive_tests=yes]) + AC_ARG_ENABLE(endomorphism, AS_HELP_STRING([--enable-endomorphism],[enable endomorphism (default is no)]), [use_endomorphism=$enableval], @@ -119,11 +124,6 @@ AC_ARG_ENABLE(module_ecdh, [enable_module_ecdh=$enableval], [enable_module_ecdh=no]) -AC_ARG_ENABLE(module_schnorr, - AS_HELP_STRING([--enable-module-schnorr],[enable Schnorr signature module (experimental)]), - [enable_module_schnorr=$enableval], - [enable_module_schnorr=no]) - AC_ARG_ENABLE(module_recovery, AS_HELP_STRING([--enable-module-recovery],[enable ECDSA pubkey recovery module (default is no)]), [enable_module_recovery=$enableval], @@ -381,9 +381,6 @@ fi if test x"$use_jni" != x"no"; then AX_JNI_INCLUDE_DIR have_jni_dependencies=yes - if test x"$enable_module_schnorr" = x"no"; then - have_jni_dependencies=no - fi if test x"$enable_module_ecdh" = x"no"; then have_jni_dependencies=no fi @@ -392,7 +389,7 @@ if test x"$use_jni" != x"no"; then fi if test "x$have_jni_dependencies" = "xno"; then if test x"$use_jni" = x"yes"; then - AC_MSG_ERROR([jni support explicitly requested but headers/dependencies were not found. Enable ECDH and Schnorr and try again.]) + AC_MSG_ERROR([jni support explicitly requested but headers/dependencies were not found. Enable ECDH and try again.]) fi AC_MSG_WARN([jni headers/dependencies not found. jni support disabled]) use_jni=no @@ -413,7 +410,7 @@ if test x"$use_endomorphism" = x"yes"; then AC_DEFINE(USE_ENDOMORPHISM, 1, [Define this symbol to use endomorphism optimization]) fi -if test x"$use_ecmult_static_precomputation" = x"yes"; then +if test x"$set_precomp" = x"yes"; then AC_DEFINE(USE_ECMULT_STATIC_PRECOMPUTATION, 1, [Define this symbol to use a statically generated ecmult table]) fi @@ -421,10 +418,6 @@ if test x"$enable_module_ecdh" = x"yes"; then AC_DEFINE(ENABLE_MODULE_ECDH, 1, [Define this symbol to enable the ECDH module]) fi -if test x"$enable_module_schnorr" = x"yes"; then - AC_DEFINE(ENABLE_MODULE_SCHNORR, 1, [Define this symbol to enable the Schnorr signature module]) -fi - if test x"$enable_module_recovery" = x"yes"; then AC_DEFINE(ENABLE_MODULE_RECOVERY, 1, [Define this symbol to enable the ECDSA pubkey recovery module]) fi @@ -442,7 +435,6 @@ AC_MSG_NOTICE([Using bignum implementation: $set_bignum]) AC_MSG_NOTICE([Using scalar implementation: $set_scalar]) AC_MSG_NOTICE([Using endomorphism optimizations: $use_endomorphism]) AC_MSG_NOTICE([Building ECDH module: $enable_module_ecdh]) -AC_MSG_NOTICE([Building Schnorr signatures module: $enable_module_schnorr]) AC_MSG_NOTICE([Building ECDSA pubkey recovery module: $enable_module_recovery]) AC_MSG_NOTICE([Using jni: $use_jni]) @@ -451,12 +443,8 @@ if test x"$enable_experimental" = x"yes"; then AC_MSG_NOTICE([WARNING: experimental build]) AC_MSG_NOTICE([Experimental features do not have stable APIs or properties, and may not be safe for production use.]) AC_MSG_NOTICE([Building ECDH module: $enable_module_ecdh]) - AC_MSG_NOTICE([Building Schnorr signatures module: $enable_module_schnorr]) AC_MSG_NOTICE([******]) else - if test x"$enable_module_schnorr" = x"yes"; then - AC_MSG_ERROR([Schnorr signature module is experimental. Use --enable-experimental to allow.]) - fi if test x"$enable_module_ecdh" = x"yes"; then AC_MSG_ERROR([ECDH module is experimental. Use --enable-experimental to allow.]) fi @@ -473,10 +461,10 @@ AC_SUBST(SECP_LIBS) AC_SUBST(SECP_TEST_LIBS) AC_SUBST(SECP_TEST_INCLUDES) AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) +AM_CONDITIONAL([USE_EXHAUSTIVE_TESTS], [test x"$use_exhaustive_tests" != x"no"]) AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) -AM_CONDITIONAL([USE_ECMULT_STATIC_PRECOMPUTATION], [test x"$use_ecmult_static_precomputation" = x"yes"]) +AM_CONDITIONAL([USE_ECMULT_STATIC_PRECOMPUTATION], [test x"$set_precomp" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_ECDH], [test x"$enable_module_ecdh" = x"yes"]) -AM_CONDITIONAL([ENABLE_MODULE_SCHNORR], [test x"$enable_module_schnorr" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_RECOVERY], [test x"$enable_module_recovery" = x"yes"]) AM_CONDITIONAL([USE_JNI], [test x"$use_jni" == x"yes"]) AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$use_external_asm" = x"yes"]) diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index 7145dbcc54..f268e309d0 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -47,11 +47,8 @@ typedef struct secp256k1_context_struct secp256k1_context; * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is * however guaranteed to be 64 bytes in size, and can be safely copied/moved. - * If you need to convert to a format suitable for storage or transmission, use - * secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. - * - * Furthermore, it is guaranteed that identical public keys (ignoring - * compression) will have identical representation, so they can be memcmp'ed. + * If you need to convert to a format suitable for storage, transmission, or + * comparison, use secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. */ typedef struct { unsigned char data[64]; @@ -62,12 +59,9 @@ typedef struct { * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is * however guaranteed to be 64 bytes in size, and can be safely copied/moved. - * If you need to convert to a format suitable for storage or transmission, use - * the secp256k1_ecdsa_signature_serialize_* and + * If you need to convert to a format suitable for storage, transmission, or + * comparison, use the secp256k1_ecdsa_signature_serialize_* and * secp256k1_ecdsa_signature_serialize_* functions. - * - * Furthermore, it is guaranteed to identical signatures will have identical - * representation, so they can be memcmp'ed. */ typedef struct { unsigned char data[64]; diff --git a/src/secp256k1/include/secp256k1_schnorr.h b/src/secp256k1/include/secp256k1_schnorr.h deleted file mode 100644 index dc32fec1ea..0000000000 --- a/src/secp256k1/include/secp256k1_schnorr.h +++ /dev/null @@ -1,173 +0,0 @@ -#ifndef _SECP256K1_SCHNORR_ -# define _SECP256K1_SCHNORR_ - -# include "secp256k1.h" - -# ifdef __cplusplus -extern "C" { -# endif - -/** Create a signature using a custom EC-Schnorr-SHA256 construction. It - * produces non-malleable 64-byte signatures which support public key recovery - * batch validation, and multiparty signing. - * Returns: 1: signature created - * 0: the nonce generation function failed, or the private key was - * invalid. - * Args: ctx: pointer to a context object, initialized for signing - * (cannot be NULL) - * Out: sig64: pointer to a 64-byte array where the signature will be - * placed (cannot be NULL) - * In: msg32: the 32-byte message hash being signed (cannot be NULL) - * seckey: pointer to a 32-byte secret key (cannot be NULL) - * noncefp:pointer to a nonce generation function. If NULL, - * secp256k1_nonce_function_default is used - * ndata: pointer to arbitrary data used by the nonce generation - * function (can be NULL) - */ -SECP256K1_API int secp256k1_schnorr_sign( - const secp256k1_context* ctx, - unsigned char *sig64, - const unsigned char *msg32, - const unsigned char *seckey, - secp256k1_nonce_function noncefp, - const void *ndata -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); - -/** Verify a signature created by secp256k1_schnorr_sign. - * Returns: 1: correct signature - * 0: incorrect signature - * Args: ctx: a secp256k1 context object, initialized for verification. - * In: sig64: the 64-byte signature being verified (cannot be NULL) - * msg32: the 32-byte message hash being verified (cannot be NULL) - * pubkey: the public key to verify with (cannot be NULL) - */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorr_verify( - const secp256k1_context* ctx, - const unsigned char *sig64, - const unsigned char *msg32, - const secp256k1_pubkey *pubkey -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); - -/** Recover an EC public key from a Schnorr signature created using - * secp256k1_schnorr_sign. - * Returns: 1: public key successfully recovered (which guarantees a correct - * signature). - * 0: otherwise. - * Args: ctx: pointer to a context object, initialized for - * verification (cannot be NULL) - * Out: pubkey: pointer to a pubkey to set to the recovered public key - * (cannot be NULL). - * In: sig64: signature as 64 byte array (cannot be NULL) - * msg32: the 32-byte message hash assumed to be signed (cannot - * be NULL) - */ -SECP256K1_API int secp256k1_schnorr_recover( - const secp256k1_context* ctx, - secp256k1_pubkey *pubkey, - const unsigned char *sig64, - const unsigned char *msg32 -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); - -/** Generate a nonce pair deterministically for use with - * secp256k1_schnorr_partial_sign. - * Returns: 1: valid nonce pair was generated. - * 0: otherwise (nonce generation function failed) - * Args: ctx: pointer to a context object, initialized for signing - * (cannot be NULL) - * Out: pubnonce: public side of the nonce (cannot be NULL) - * privnonce32: private side of the nonce (32 byte) (cannot be NULL) - * In: msg32: the 32-byte message hash assumed to be signed (cannot - * be NULL) - * sec32: the 32-byte private key (cannot be NULL) - * noncefp: pointer to a nonce generation function. If NULL, - * secp256k1_nonce_function_default is used - * noncedata: pointer to arbitrary data used by the nonce generation - * function (can be NULL) - * - * Do not use the output as a private/public key pair for signing/validation. - */ -SECP256K1_API int secp256k1_schnorr_generate_nonce_pair( - const secp256k1_context* ctx, - secp256k1_pubkey *pubnonce, - unsigned char *privnonce32, - const unsigned char *msg32, - const unsigned char *sec32, - secp256k1_nonce_function noncefp, - const void* noncedata -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); - -/** Produce a partial Schnorr signature, which can be combined using - * secp256k1_schnorr_partial_combine, to end up with a full signature that is - * verifiable using secp256k1_schnorr_verify. - * Returns: 1: signature created successfully. - * 0: no valid signature exists with this combination of keys, nonces - * and message (chance around 1 in 2^128) - * -1: invalid private key, nonce, or public nonces. - * Args: ctx: pointer to context object, initialized for signing (cannot - * be NULL) - * Out: sig64: pointer to 64-byte array to put partial signature in - * In: msg32: pointer to 32-byte message to sign - * sec32: pointer to 32-byte private key - * pubnonce_others: pointer to pubkey containing the sum of the other's - * nonces (see secp256k1_ec_pubkey_combine) - * secnonce32: pointer to 32-byte array containing our nonce - * - * The intended procedure for creating a multiparty signature is: - * - Each signer S[i] with private key x[i] and public key Q[i] runs - * secp256k1_schnorr_generate_nonce_pair to produce a pair (k[i],R[i]) of - * private/public nonces. - * - All signers communicate their public nonces to each other (revealing your - * private nonce can lead to discovery of your private key, so it should be - * considered secret). - * - All signers combine all the public nonces they received (excluding their - * own) using secp256k1_ec_pubkey_combine to obtain an - * Rall[i] = sum(R[0..i-1,i+1..n]). - * - All signers produce a partial signature using - * secp256k1_schnorr_partial_sign, passing in their own private key x[i], - * their own private nonce k[i], and the sum of the others' public nonces - * Rall[i]. - * - All signers communicate their partial signatures to each other. - * - Someone combines all partial signatures using - * secp256k1_schnorr_partial_combine, to obtain a full signature. - * - The resulting signature is validatable using secp256k1_schnorr_verify, with - * public key equal to the result of secp256k1_ec_pubkey_combine of the - * signers' public keys (sum(Q[0..n])). - * - * Note that secp256k1_schnorr_partial_combine and secp256k1_ec_pubkey_combine - * function take their arguments in any order, and it is possible to - * pre-combine several inputs already with one call, and add more inputs later - * by calling the function again (they are commutative and associative). - */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorr_partial_sign( - const secp256k1_context* ctx, - unsigned char *sig64, - const unsigned char *msg32, - const unsigned char *sec32, - const secp256k1_pubkey *pubnonce_others, - const unsigned char *secnonce32 -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(6); - -/** Combine multiple Schnorr partial signatures. - * Returns: 1: the passed signatures were successfully combined. - * 0: the resulting signature is not valid (chance of 1 in 2^256) - * -1: some inputs were invalid, or the signatures were not created - * using the same set of nonces - * Args: ctx: pointer to a context object - * Out: sig64: pointer to a 64-byte array to place the combined signature - * (cannot be NULL) - * In: sig64sin: pointer to an array of n pointers to 64-byte input - * signatures - * n: the number of signatures to combine (at least 1) - */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorr_partial_combine( - const secp256k1_context* ctx, - unsigned char *sig64, - const unsigned char * const * sig64sin, - size_t n -) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); - -# ifdef __cplusplus -} -# endif - -#endif diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index d110b4bb1d..9a42e519bd 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -203,7 +203,9 @@ static int secp256k1_ecdsa_sig_serialize(unsigned char *sig, size_t *size, const static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const secp256k1_scalar *sigr, const secp256k1_scalar *sigs, const secp256k1_ge *pubkey, const secp256k1_scalar *message) { unsigned char c[32]; secp256k1_scalar sn, u1, u2; +#if !defined(EXHAUSTIVE_TEST_ORDER) secp256k1_fe xr; +#endif secp256k1_gej pubkeyj; secp256k1_gej pr; @@ -219,6 +221,21 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const if (secp256k1_gej_is_infinity(&pr)) { return 0; } + +#if defined(EXHAUSTIVE_TEST_ORDER) +{ + secp256k1_scalar computed_r; + int overflow = 0; + secp256k1_ge pr_ge; + secp256k1_ge_set_gej(&pr_ge, &pr); + secp256k1_fe_normalize(&pr_ge.x); + + secp256k1_fe_get_b32(c, &pr_ge.x); + secp256k1_scalar_set_b32(&computed_r, c, &overflow); + /* we fully expect overflow */ + return secp256k1_scalar_eq(sigr, &computed_r); +} +#else secp256k1_scalar_get_b32(c, sigr); secp256k1_fe_set_b32(&xr, c); @@ -252,6 +269,7 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const return 1; } return 0; +#endif } static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context *ctx, secp256k1_scalar *sigr, secp256k1_scalar *sigs, const secp256k1_scalar *seckey, const secp256k1_scalar *message, const secp256k1_scalar *nonce, int *recid) { diff --git a/src/secp256k1/src/ecmult_const_impl.h b/src/secp256k1/src/ecmult_const_impl.h index 7a6a25318c..0db314c48e 100644 --- a/src/secp256k1/src/ecmult_const_impl.h +++ b/src/secp256k1/src/ecmult_const_impl.h @@ -78,7 +78,7 @@ static int secp256k1_wnaf_const(int *wnaf, secp256k1_scalar s, int w) { /* Negative numbers will be negated to keep their bit representation below the maximum width */ flip = secp256k1_scalar_is_high(&s); /* We add 1 to even numbers, 2 to odd ones, noting that negation flips parity */ - bit = flip ^ (s.d[0] & 1); + bit = flip ^ !secp256k1_scalar_is_even(&s); /* We check for negative one, since adding 2 to it will cause an overflow */ secp256k1_scalar_negate(&neg_s, &s); not_neg_one = !secp256k1_scalar_is_one(&neg_s); diff --git a/src/secp256k1/src/ecmult_gen_impl.h b/src/secp256k1/src/ecmult_gen_impl.h index b63c4d8662..35f2546077 100644 --- a/src/secp256k1/src/ecmult_gen_impl.h +++ b/src/secp256k1/src/ecmult_gen_impl.h @@ -77,7 +77,7 @@ static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context *ctx secp256k1_gej_add_var(&numsbase, &numsbase, &nums_gej, NULL); } } - secp256k1_ge_set_all_gej_var(1024, prec, precj, cb); + secp256k1_ge_set_all_gej_var(prec, precj, 1024, cb); } for (j = 0; j < 64; j++) { for (i = 0; i < 16; i++) { diff --git a/src/secp256k1/src/ecmult_impl.h b/src/secp256k1/src/ecmult_impl.h index 81ae08e100..4e40104ad4 100644 --- a/src/secp256k1/src/ecmult_impl.h +++ b/src/secp256k1/src/ecmult_impl.h @@ -7,15 +7,29 @@ #ifndef _SECP256K1_ECMULT_IMPL_H_ #define _SECP256K1_ECMULT_IMPL_H_ +#include <string.h> + #include "group.h" #include "scalar.h" #include "ecmult.h" -#include <string.h> - +#if defined(EXHAUSTIVE_TEST_ORDER) +/* We need to lower these values for exhaustive tests because + * the tables cannot have infinities in them (this breaks the + * affine-isomorphism stuff which tracks z-ratios) */ +# if EXHAUSTIVE_TEST_ORDER > 128 +# define WINDOW_A 5 +# define WINDOW_G 8 +# elif EXHAUSTIVE_TEST_ORDER > 8 +# define WINDOW_A 4 +# define WINDOW_G 4 +# else +# define WINDOW_A 2 +# define WINDOW_G 2 +# endif +#else /* optimal for 128-bit and 256-bit exponents. */ #define WINDOW_A 5 - /** larger numbers may result in slightly better performance, at the cost of exponentially larger precomputed tables. */ #ifdef USE_ENDOMORPHISM @@ -25,6 +39,7 @@ /** One table for window size 16: 1.375 MiB. */ #define WINDOW_G 16 #endif +#endif /** The number of entries a table with precomputed multiples needs to have. */ #define ECMULT_TABLE_SIZE(w) (1 << ((w)-2)) @@ -103,7 +118,7 @@ static void secp256k1_ecmult_odd_multiples_table_storage_var(int n, secp256k1_ge /* Compute the odd multiples in Jacobian form. */ secp256k1_ecmult_odd_multiples_table(n, prej, zr, a); /* Convert them in batch to affine coordinates. */ - secp256k1_ge_set_table_gej_var(n, prea, prej, zr); + secp256k1_ge_set_table_gej_var(prea, prej, zr, n); /* Convert them to compact storage form. */ for (i = 0; i < n; i++) { secp256k1_ge_to_storage(&pre[i], &prea[i]); diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index c5ba074244..bbb1ee866c 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -30,6 +30,8 @@ #error "Please select field implementation" #endif +#include "util.h" + /** Normalize a field element. */ static void secp256k1_fe_normalize(secp256k1_fe *r); @@ -50,6 +52,9 @@ static int secp256k1_fe_normalizes_to_zero_var(secp256k1_fe *r); /** Set a field element equal to a small integer. Resulting field element is normalized. */ static void secp256k1_fe_set_int(secp256k1_fe *r, int a); +/** Sets a field element equal to zero, initializing all fields. */ +static void secp256k1_fe_clear(secp256k1_fe *a); + /** Verify whether a field element is zero. Requires the input to be normalized. */ static int secp256k1_fe_is_zero(const secp256k1_fe *a); @@ -110,7 +115,7 @@ static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *a); /** Calculate the (modular) inverses of a batch of field elements. Requires the inputs' magnitudes to be * at most 8. The output magnitudes are 1 (but not guaranteed to be normalized). The inputs and * outputs must not overlap in memory. */ -static void secp256k1_fe_inv_all_var(size_t len, secp256k1_fe *r, const secp256k1_fe *a); +static void secp256k1_fe_inv_all_var(secp256k1_fe *r, const secp256k1_fe *a, size_t len); /** Convert a field element to the storage type. */ static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a); diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 52cd902eb3..5127b279bc 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -260,7 +260,7 @@ static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *a) { #endif } -static void secp256k1_fe_inv_all_var(size_t len, secp256k1_fe *r, const secp256k1_fe *a) { +static void secp256k1_fe_inv_all_var(secp256k1_fe *r, const secp256k1_fe *a, size_t len) { secp256k1_fe u; size_t i; if (len < 1) { diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index d515716744..4957b248fe 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -65,12 +65,12 @@ static void secp256k1_ge_neg(secp256k1_ge *r, const secp256k1_ge *a); static void secp256k1_ge_set_gej(secp256k1_ge *r, secp256k1_gej *a); /** Set a batch of group elements equal to the inputs given in jacobian coordinates */ -static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_callback *cb); +static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a, size_t len, const secp256k1_callback *cb); /** Set a batch of group elements equal to the inputs given in jacobian * coordinates (with known z-ratios). zr must contain the known z-ratios such * that mul(a[i].z, zr[i+1]) == a[i+1].z. zr[0] is ignored. */ -static void secp256k1_ge_set_table_gej_var(size_t len, secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zr); +static void secp256k1_ge_set_table_gej_var(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zr, size_t len); /** Bring a batch inputs given in jacobian coordinates (with known z-ratios) to * the same global z "denominator". zr must contain the known z-ratios such diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 3e9c4c410d..2e192b62fd 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -11,6 +11,53 @@ #include "field.h" #include "group.h" +/* These points can be generated in sage as follows: + * + * 0. Setup a worksheet with the following parameters. + * b = 4 # whatever CURVE_B will be set to + * F = FiniteField (0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F) + * C = EllipticCurve ([F (0), F (b)]) + * + * 1. Determine all the small orders available to you. (If there are + * no satisfactory ones, go back and change b.) + * print C.order().factor(limit=1000) + * + * 2. Choose an order as one of the prime factors listed in the above step. + * (You can also multiply some to get a composite order, though the + * tests will crash trying to invert scalars during signing.) We take a + * random point and scale it to drop its order to the desired value. + * There is some probability this won't work; just try again. + * order = 199 + * P = C.random_point() + * P = (int(P.order()) / int(order)) * P + * assert(P.order() == order) + * + * 3. Print the values. You'll need to use a vim macro or something to + * split the hex output into 4-byte chunks. + * print "%x %x" % P.xy() + */ +#if defined(EXHAUSTIVE_TEST_ORDER) +# if EXHAUSTIVE_TEST_ORDER == 199 +const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST( + 0xFA7CC9A7, 0x0737F2DB, 0xA749DD39, 0x2B4FB069, + 0x3B017A7D, 0xA808C2F1, 0xFB12940C, 0x9EA66C18, + 0x78AC123A, 0x5ED8AEF3, 0x8732BC91, 0x1F3A2868, + 0x48DF246C, 0x808DAE72, 0xCFE52572, 0x7F0501ED +); + +const int CURVE_B = 4; +# elif EXHAUSTIVE_TEST_ORDER == 13 +const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST( + 0xedc60018, 0xa51a786b, 0x2ea91f4d, 0x4c9416c0, + 0x9de54c3b, 0xa1316554, 0x6cf4345c, 0x7277ef15, + 0x54cb1b6b, 0xdc8c1273, 0x087844ea, 0x43f4603e, + 0x0eaf9a43, 0xf6effe55, 0x939f806d, 0x37adf8ac +); +const int CURVE_B = 2; +# else +# error No known generator for the specified exhaustive test group order. +# endif +#else /** Generator for secp256k1, value 'g' defined in * "Standards for Efficient Cryptography" (SEC2) 2.7.1. */ @@ -21,8 +68,11 @@ static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST( 0xFD17B448UL, 0xA6855419UL, 0x9C47D08FUL, 0xFB10D4B8UL ); +const int CURVE_B = 7; +#endif + static void secp256k1_ge_set_gej_zinv(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zi) { - secp256k1_fe zi2; + secp256k1_fe zi2; secp256k1_fe zi3; secp256k1_fe_sqr(&zi2, zi); secp256k1_fe_mul(&zi3, &zi2, zi); @@ -76,7 +126,7 @@ static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a) { r->y = a->y; } -static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_callback *cb) { +static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a, size_t len, const secp256k1_callback *cb) { secp256k1_fe *az; secp256k1_fe *azi; size_t i; @@ -89,7 +139,7 @@ static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge *r, const secp } azi = (secp256k1_fe *)checked_malloc(cb, sizeof(secp256k1_fe) * count); - secp256k1_fe_inv_all_var(count, azi, az); + secp256k1_fe_inv_all_var(azi, az, count); free(az); count = 0; @@ -102,7 +152,7 @@ static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge *r, const secp free(azi); } -static void secp256k1_ge_set_table_gej_var(size_t len, secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zr) { +static void secp256k1_ge_set_table_gej_var(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zr, size_t len) { size_t i = len - 1; secp256k1_fe zi; @@ -145,9 +195,15 @@ static void secp256k1_ge_globalz_set_table_gej(size_t len, secp256k1_ge *r, secp static void secp256k1_gej_set_infinity(secp256k1_gej *r) { r->infinity = 1; - secp256k1_fe_set_int(&r->x, 0); - secp256k1_fe_set_int(&r->y, 0); - secp256k1_fe_set_int(&r->z, 0); + secp256k1_fe_clear(&r->x); + secp256k1_fe_clear(&r->y); + secp256k1_fe_clear(&r->z); +} + +static void secp256k1_ge_set_infinity(secp256k1_ge *r) { + r->infinity = 1; + secp256k1_fe_clear(&r->x); + secp256k1_fe_clear(&r->y); } static void secp256k1_gej_clear(secp256k1_gej *r) { @@ -169,7 +225,7 @@ static int secp256k1_ge_set_xquad(secp256k1_ge *r, const secp256k1_fe *x) { secp256k1_fe_sqr(&x2, x); secp256k1_fe_mul(&x3, x, &x2); r->infinity = 0; - secp256k1_fe_set_int(&c, 7); + secp256k1_fe_set_int(&c, CURVE_B); secp256k1_fe_add(&c, &x3); return secp256k1_fe_sqrt(&r->y, &c); } @@ -228,7 +284,7 @@ static int secp256k1_gej_is_valid_var(const secp256k1_gej *a) { secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x); secp256k1_fe_sqr(&z2, &a->z); secp256k1_fe_sqr(&z6, &z2); secp256k1_fe_mul(&z6, &z6, &z2); - secp256k1_fe_mul_int(&z6, 7); + secp256k1_fe_mul_int(&z6, CURVE_B); secp256k1_fe_add(&x3, &z6); secp256k1_fe_normalize_weak(&x3); return secp256k1_fe_equal_var(&y2, &x3); @@ -242,7 +298,7 @@ static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) { /* y^2 = x^3 + 7 */ secp256k1_fe_sqr(&y2, &a->y); secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x); - secp256k1_fe_set_int(&c, 7); + secp256k1_fe_set_int(&c, CURVE_B); secp256k1_fe_add(&x3, &c); secp256k1_fe_normalize_weak(&x3); return secp256k1_fe_equal_var(&y2, &x3); @@ -260,7 +316,7 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s /** For secp256k1, 2Q is infinity if and only if Q is infinity. This is because if 2Q = infinity, * Q must equal -Q, or that Q.y == -(Q.y), or Q.y is 0. For a point on y^2 = x^3 + 7 to have * y=0, x^3 must be -7 mod p. However, -7 has no cube root mod p. - * + * * Having said this, if this function receives a point on a sextic twist, e.g. by * a fault attack, it is possible for y to be 0. This happens for y^2 = x^3 + 6, * since -6 does have a cube root mod p. For this point, this function will not set diff --git a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java index be67048fbe..1c67802fba 100644 --- a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java +++ b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java @@ -32,7 +32,7 @@ import static org.bitcoin.NativeSecp256k1Util.*; * <p>You can find an example library that can be used for this at https://github.com/bitcoin/secp256k1</p> * * <p>To build secp256k1 for use with bitcoinj, run - * `./configure --enable-jni --enable-experimental --enable-module-schnorr --enable-module-ecdh` + * `./configure --enable-jni --enable-experimental --enable-module-ecdh` * and `make` then copy `.libs/libsecp256k1.so` to your system library path * or point the JVM to the folder containing it with -Djava.library.path * </p> @@ -417,36 +417,6 @@ public class NativeSecp256k1 { } } - public static byte[] schnorrSign(byte[] data, byte[] sec) throws AssertFailException { - Preconditions.checkArgument(data.length == 32 && sec.length <= 32); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null) { - byteBuff = ByteBuffer.allocateDirect(32 + 32); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(data); - byteBuff.put(sec); - - byte[][] retByteArray; - - r.lock(); - try { - retByteArray = secp256k1_schnorr_sign(byteBuff, Secp256k1Context.getContext()); - } finally { - r.unlock(); - } - - byte[] sigArr = retByteArray[0]; - int retVal = new BigInteger(new byte[] { retByteArray[1][0] }).intValue(); - - assertEquals(sigArr.length, 64, "Got bad signature length."); - - return retVal == 0 ? new byte[0] : sigArr; - } - private static native long secp256k1_ctx_clone(long context); private static native int secp256k1_context_randomize(ByteBuffer byteBuff, long context); @@ -471,8 +441,6 @@ public class NativeSecp256k1 { private static native byte[][] secp256k1_ec_pubkey_parse(ByteBuffer byteBuff, long context, int inputLen); - private static native byte[][] secp256k1_schnorr_sign(ByteBuffer byteBuff, long context); - private static native byte[][] secp256k1_ecdh(ByteBuffer byteBuff, long context, int inputLen); } diff --git a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java index f18ce95810..c00d08899b 100644 --- a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java +++ b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java @@ -167,22 +167,6 @@ public class NativeSecp256k1Test { assertEquals( result, true, "testRandomize"); } - /** - * This tests signSchnorr() for a valid secretkey - */ - public static void testSchnorrSign() throws AssertFailException{ - - byte[] data = BaseEncoding.base16().lowerCase().decode("CF80CD8AED482D5D1527D7DC72FCEFF84E6326592848447D2DC0B0E87DFC9A90".toLowerCase()); //sha256hash of "testing" - byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); - - byte[] resultArr = NativeSecp256k1.schnorrSign(data, sec); - String sigString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( sigString, "C5E929AA058B982048760422D3B563749B7D0E50C5EBD8CD2FFC23214BD6A2F1B072C13880997EBA847CF20F2F90FCE07C1CA33A890A4127095A351127F8D95F" , "testSchnorrSign"); - } - - /** - * This tests signSchnorr() for a valid secretkey - */ public static void testCreateECDHSecret() throws AssertFailException{ byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); @@ -216,11 +200,6 @@ public class NativeSecp256k1Test { testSignPos(); testSignNeg(); - //Test Schnorr (partial support) //TODO - testSchnorrSign(); - //testSchnorrVerify - //testSchnorrRecovery - //Test privKeyTweakAdd() 1 testPrivKeyTweakAdd_1(); diff --git a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c index dba9524dd4..bcef7b32ce 100644 --- a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c +++ b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c @@ -5,7 +5,6 @@ #include "include/secp256k1.h" #include "include/secp256k1_ecdh.h" #include "include/secp256k1_recovery.h" -#include "include/secp256k1_schnorr.h" SECP256K1_API jlong JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ctx_1clone @@ -333,39 +332,6 @@ SECP256K1_API jlong JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdsa_1p return 0; } -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1schnorr_1sign - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - unsigned char* data = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - unsigned char* secKey = (unsigned char*) (data + 32); - - jobjectArray retArray; - jbyteArray sigArray, intsByteArray; - unsigned char intsarray[1]; - unsigned char sig[64]; - - int ret = secp256k1_schnorr_sign(ctx, sig, data, secKey, NULL, NULL); - - intsarray[0] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - sigArray = (*env)->NewByteArray(env, 64); - (*env)->SetByteArrayRegion(env, sigArray, 0, 64, (jbyte*)sig); - (*env)->SetObjectArrayElement(env, retArray, 0, sigArray); - - intsByteArray = (*env)->NewByteArray(env, 1); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 1, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; -} - SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdh (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l, jint publen) { diff --git a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h index 4125a1f523..fe613c9e9e 100644 --- a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h +++ b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h @@ -106,14 +106,6 @@ SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1e /* * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_schnorr_sign - * Signature: (Ljava/nio/ByteBuffer;JI)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1schnorr_1sign - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l); - -/* - * Class: org_bitcoin_NativeSecp256k1 * Method: secp256k1_ecdh * Signature: (Ljava/nio/ByteBuffer;JI)[[B */ diff --git a/src/secp256k1/src/modules/recovery/main_impl.h b/src/secp256k1/src/modules/recovery/main_impl.h index ec42f4bb6c..86f2f0cb2b 100644..100755 --- a/src/secp256k1/src/modules/recovery/main_impl.h +++ b/src/secp256k1/src/modules/recovery/main_impl.h @@ -138,16 +138,15 @@ int secp256k1_ecdsa_sign_recoverable(const secp256k1_context* ctx, secp256k1_ecd secp256k1_scalar_set_b32(&sec, seckey, &overflow); /* Fail if the secret key is invalid. */ if (!overflow && !secp256k1_scalar_is_zero(&sec)) { + unsigned char nonce32[32]; unsigned int count = 0; secp256k1_scalar_set_b32(&msg, msg32, NULL); while (1) { - unsigned char nonce32[32]; ret = noncefp(nonce32, msg32, seckey, NULL, (void*)noncedata, count); if (!ret) { break; } secp256k1_scalar_set_b32(&non, nonce32, &overflow); - memset(nonce32, 0, 32); if (!secp256k1_scalar_is_zero(&non) && !overflow) { if (secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &r, &s, &sec, &msg, &non, &recid)) { break; @@ -155,6 +154,7 @@ int secp256k1_ecdsa_sign_recoverable(const secp256k1_context* ctx, secp256k1_ecd } count++; } + memset(nonce32, 0, 32); secp256k1_scalar_clear(&msg); secp256k1_scalar_clear(&non); secp256k1_scalar_clear(&sec); diff --git a/src/secp256k1/src/modules/schnorr/Makefile.am.include b/src/secp256k1/src/modules/schnorr/Makefile.am.include deleted file mode 100644 index f1af8e8325..0000000000 --- a/src/secp256k1/src/modules/schnorr/Makefile.am.include +++ /dev/null @@ -1,10 +0,0 @@ -include_HEADERS += include/secp256k1_schnorr.h -noinst_HEADERS += src/modules/schnorr/main_impl.h -noinst_HEADERS += src/modules/schnorr/schnorr.h -noinst_HEADERS += src/modules/schnorr/schnorr_impl.h -noinst_HEADERS += src/modules/schnorr/tests_impl.h -if USE_BENCHMARK -noinst_PROGRAMS += bench_schnorr_verify -bench_schnorr_verify_SOURCES = src/bench_schnorr_verify.c -bench_schnorr_verify_LDADD = libsecp256k1.la $(SECP_LIBS) $(COMMON_LIB) -endif diff --git a/src/secp256k1/src/modules/schnorr/main_impl.h b/src/secp256k1/src/modules/schnorr/main_impl.h deleted file mode 100644 index fa176a1767..0000000000 --- a/src/secp256k1/src/modules/schnorr/main_impl.h +++ /dev/null @@ -1,164 +0,0 @@ -/********************************************************************** - * Copyright (c) 2014-2015 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php.* - **********************************************************************/ - -#ifndef SECP256K1_MODULE_SCHNORR_MAIN -#define SECP256K1_MODULE_SCHNORR_MAIN - -#include "include/secp256k1_schnorr.h" -#include "modules/schnorr/schnorr_impl.h" - -static void secp256k1_schnorr_msghash_sha256(unsigned char *h32, const unsigned char *r32, const unsigned char *msg32) { - secp256k1_sha256_t sha; - secp256k1_sha256_initialize(&sha); - secp256k1_sha256_write(&sha, r32, 32); - secp256k1_sha256_write(&sha, msg32, 32); - secp256k1_sha256_finalize(&sha, h32); -} - -static const unsigned char secp256k1_schnorr_algo16[17] = "Schnorr+SHA256 "; - -int secp256k1_schnorr_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const unsigned char *seckey, secp256k1_nonce_function noncefp, const void* noncedata) { - secp256k1_scalar sec, non; - int ret = 0; - int overflow = 0; - unsigned int count = 0; - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sig64 != NULL); - ARG_CHECK(seckey != NULL); - if (noncefp == NULL) { - noncefp = secp256k1_nonce_function_default; - } - - secp256k1_scalar_set_b32(&sec, seckey, NULL); - while (1) { - unsigned char nonce32[32]; - ret = noncefp(nonce32, msg32, seckey, secp256k1_schnorr_algo16, (void*)noncedata, count); - if (!ret) { - break; - } - secp256k1_scalar_set_b32(&non, nonce32, &overflow); - memset(nonce32, 0, 32); - if (!secp256k1_scalar_is_zero(&non) && !overflow) { - if (secp256k1_schnorr_sig_sign(&ctx->ecmult_gen_ctx, sig64, &sec, &non, NULL, secp256k1_schnorr_msghash_sha256, msg32)) { - break; - } - } - count++; - } - if (!ret) { - memset(sig64, 0, 64); - } - secp256k1_scalar_clear(&non); - secp256k1_scalar_clear(&sec); - return ret; -} - -int secp256k1_schnorr_verify(const secp256k1_context* ctx, const unsigned char *sig64, const unsigned char *msg32, const secp256k1_pubkey *pubkey) { - secp256k1_ge q; - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sig64 != NULL); - ARG_CHECK(pubkey != NULL); - - secp256k1_pubkey_load(ctx, &q, pubkey); - return secp256k1_schnorr_sig_verify(&ctx->ecmult_ctx, sig64, &q, secp256k1_schnorr_msghash_sha256, msg32); -} - -int secp256k1_schnorr_recover(const secp256k1_context* ctx, secp256k1_pubkey *pubkey, const unsigned char *sig64, const unsigned char *msg32) { - secp256k1_ge q; - - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sig64 != NULL); - ARG_CHECK(pubkey != NULL); - - if (secp256k1_schnorr_sig_recover(&ctx->ecmult_ctx, sig64, &q, secp256k1_schnorr_msghash_sha256, msg32)) { - secp256k1_pubkey_save(pubkey, &q); - return 1; - } else { - memset(pubkey, 0, sizeof(*pubkey)); - return 0; - } -} - -int secp256k1_schnorr_generate_nonce_pair(const secp256k1_context* ctx, secp256k1_pubkey *pubnonce, unsigned char *privnonce32, const unsigned char *sec32, const unsigned char *msg32, secp256k1_nonce_function noncefp, const void* noncedata) { - int count = 0; - int ret = 1; - secp256k1_gej Qj; - secp256k1_ge Q; - secp256k1_scalar sec; - - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sec32 != NULL); - ARG_CHECK(pubnonce != NULL); - ARG_CHECK(privnonce32 != NULL); - - if (noncefp == NULL) { - noncefp = secp256k1_nonce_function_default; - } - - do { - int overflow; - ret = noncefp(privnonce32, sec32, msg32, secp256k1_schnorr_algo16, (void*)noncedata, count++); - if (!ret) { - break; - } - secp256k1_scalar_set_b32(&sec, privnonce32, &overflow); - if (overflow || secp256k1_scalar_is_zero(&sec)) { - continue; - } - secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &Qj, &sec); - secp256k1_ge_set_gej(&Q, &Qj); - - secp256k1_pubkey_save(pubnonce, &Q); - break; - } while(1); - - secp256k1_scalar_clear(&sec); - if (!ret) { - memset(pubnonce, 0, sizeof(*pubnonce)); - } - return ret; -} - -int secp256k1_schnorr_partial_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const unsigned char *sec32, const secp256k1_pubkey *pubnonce_others, const unsigned char *secnonce32) { - int overflow = 0; - secp256k1_scalar sec, non; - secp256k1_ge pubnon; - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sig64 != NULL); - ARG_CHECK(sec32 != NULL); - ARG_CHECK(secnonce32 != NULL); - ARG_CHECK(pubnonce_others != NULL); - - secp256k1_scalar_set_b32(&sec, sec32, &overflow); - if (overflow || secp256k1_scalar_is_zero(&sec)) { - return -1; - } - secp256k1_scalar_set_b32(&non, secnonce32, &overflow); - if (overflow || secp256k1_scalar_is_zero(&non)) { - return -1; - } - secp256k1_pubkey_load(ctx, &pubnon, pubnonce_others); - return secp256k1_schnorr_sig_sign(&ctx->ecmult_gen_ctx, sig64, &sec, &non, &pubnon, secp256k1_schnorr_msghash_sha256, msg32); -} - -int secp256k1_schnorr_partial_combine(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char * const *sig64sin, size_t n) { - ARG_CHECK(sig64 != NULL); - ARG_CHECK(n >= 1); - ARG_CHECK(sig64sin != NULL); - return secp256k1_schnorr_sig_combine(sig64, n, sig64sin); -} - -#endif diff --git a/src/secp256k1/src/modules/schnorr/schnorr.h b/src/secp256k1/src/modules/schnorr/schnorr.h deleted file mode 100644 index de18147bd5..0000000000 --- a/src/secp256k1/src/modules/schnorr/schnorr.h +++ /dev/null @@ -1,20 +0,0 @@ -/*********************************************************************** - * Copyright (c) 2014-2015 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php. * - ***********************************************************************/ - -#ifndef _SECP256K1_MODULE_SCHNORR_H_ -#define _SECP256K1_MODULE_SCHNORR_H_ - -#include "scalar.h" -#include "group.h" - -typedef void (*secp256k1_schnorr_msghash)(unsigned char *h32, const unsigned char *r32, const unsigned char *msg32); - -static int secp256k1_schnorr_sig_sign(const secp256k1_ecmult_gen_context* ctx, unsigned char *sig64, const secp256k1_scalar *key, const secp256k1_scalar *nonce, const secp256k1_ge *pubnonce, secp256k1_schnorr_msghash hash, const unsigned char *msg32); -static int secp256k1_schnorr_sig_verify(const secp256k1_ecmult_context* ctx, const unsigned char *sig64, const secp256k1_ge *pubkey, secp256k1_schnorr_msghash hash, const unsigned char *msg32); -static int secp256k1_schnorr_sig_recover(const secp256k1_ecmult_context* ctx, const unsigned char *sig64, secp256k1_ge *pubkey, secp256k1_schnorr_msghash hash, const unsigned char *msg32); -static int secp256k1_schnorr_sig_combine(unsigned char *sig64, size_t n, const unsigned char * const *sig64ins); - -#endif diff --git a/src/secp256k1/src/modules/schnorr/schnorr_impl.h b/src/secp256k1/src/modules/schnorr/schnorr_impl.h deleted file mode 100644 index e13ab6db7c..0000000000 --- a/src/secp256k1/src/modules/schnorr/schnorr_impl.h +++ /dev/null @@ -1,207 +0,0 @@ -/*********************************************************************** - * Copyright (c) 2014-2015 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php. * - ***********************************************************************/ - -#ifndef _SECP256K1_SCHNORR_IMPL_H_ -#define _SECP256K1_SCHNORR_IMPL_H_ - -#include <string.h> - -#include "schnorr.h" -#include "num.h" -#include "field.h" -#include "group.h" -#include "ecmult.h" -#include "ecmult_gen.h" - -/** - * Custom Schnorr-based signature scheme. They support multiparty signing, public key - * recovery and batch validation. - * - * Rationale for verifying R's y coordinate: - * In order to support batch validation and public key recovery, the full R point must - * be known to verifiers, rather than just its x coordinate. In order to not risk - * being more strict in batch validation than normal validation, validators must be - * required to reject signatures with incorrect y coordinate. This is only possible - * by including a (relatively slow) field inverse, or a field square root. However, - * batch validation offers potentially much higher benefits than this cost. - * - * Rationale for having an implicit y coordinate oddness: - * If we commit to having the full R point known to verifiers, there are two mechanism. - * Either include its oddness in the signature, or give it an implicit fixed value. - * As the R y coordinate can be flipped by a simple negation of the nonce, we choose the - * latter, as it comes with nearly zero impact on signing or validation performance, and - * saves a byte in the signature. - * - * Signing: - * Inputs: 32-byte message m, 32-byte scalar key x (!=0), 32-byte scalar nonce k (!=0) - * - * Compute point R = k * G. Reject nonce if R's y coordinate is odd (or negate nonce). - * Compute 32-byte r, the serialization of R's x coordinate. - * Compute scalar h = Hash(r || m). Reject nonce if h == 0 or h >= order. - * Compute scalar s = k - h * x. - * The signature is (r, s). - * - * - * Verification: - * Inputs: 32-byte message m, public key point Q, signature: (32-byte r, scalar s) - * - * Signature is invalid if s >= order. - * Signature is invalid if r >= p. - * Compute scalar h = Hash(r || m). Signature is invalid if h == 0 or h >= order. - * Option 1 (faster for single verification): - * Compute point R = h * Q + s * G. Signature is invalid if R is infinity or R's y coordinate is odd. - * Signature is valid if the serialization of R's x coordinate equals r. - * Option 2 (allows batch validation and pubkey recovery): - * Decompress x coordinate r into point R, with odd y coordinate. Fail if R is not on the curve. - * Signature is valid if R + h * Q + s * G == 0. - */ - -static int secp256k1_schnorr_sig_sign(const secp256k1_ecmult_gen_context* ctx, unsigned char *sig64, const secp256k1_scalar *key, const secp256k1_scalar *nonce, const secp256k1_ge *pubnonce, secp256k1_schnorr_msghash hash, const unsigned char *msg32) { - secp256k1_gej Rj; - secp256k1_ge Ra; - unsigned char h32[32]; - secp256k1_scalar h, s; - int overflow; - secp256k1_scalar n; - - if (secp256k1_scalar_is_zero(key) || secp256k1_scalar_is_zero(nonce)) { - return 0; - } - n = *nonce; - - secp256k1_ecmult_gen(ctx, &Rj, &n); - if (pubnonce != NULL) { - secp256k1_gej_add_ge(&Rj, &Rj, pubnonce); - } - secp256k1_ge_set_gej(&Ra, &Rj); - secp256k1_fe_normalize(&Ra.y); - if (secp256k1_fe_is_odd(&Ra.y)) { - /* R's y coordinate is odd, which is not allowed (see rationale above). - Force it to be even by negating the nonce. Note that this even works - for multiparty signing, as the R point is known to all participants, - which can all decide to flip the sign in unison, resulting in the - overall R point to be negated too. */ - secp256k1_scalar_negate(&n, &n); - } - secp256k1_fe_normalize(&Ra.x); - secp256k1_fe_get_b32(sig64, &Ra.x); - hash(h32, sig64, msg32); - overflow = 0; - secp256k1_scalar_set_b32(&h, h32, &overflow); - if (overflow || secp256k1_scalar_is_zero(&h)) { - secp256k1_scalar_clear(&n); - return 0; - } - secp256k1_scalar_mul(&s, &h, key); - secp256k1_scalar_negate(&s, &s); - secp256k1_scalar_add(&s, &s, &n); - secp256k1_scalar_clear(&n); - secp256k1_scalar_get_b32(sig64 + 32, &s); - return 1; -} - -static int secp256k1_schnorr_sig_verify(const secp256k1_ecmult_context* ctx, const unsigned char *sig64, const secp256k1_ge *pubkey, secp256k1_schnorr_msghash hash, const unsigned char *msg32) { - secp256k1_gej Qj, Rj; - secp256k1_ge Ra; - secp256k1_fe Rx; - secp256k1_scalar h, s; - unsigned char hh[32]; - int overflow; - - if (secp256k1_ge_is_infinity(pubkey)) { - return 0; - } - hash(hh, sig64, msg32); - overflow = 0; - secp256k1_scalar_set_b32(&h, hh, &overflow); - if (overflow || secp256k1_scalar_is_zero(&h)) { - return 0; - } - overflow = 0; - secp256k1_scalar_set_b32(&s, sig64 + 32, &overflow); - if (overflow) { - return 0; - } - if (!secp256k1_fe_set_b32(&Rx, sig64)) { - return 0; - } - secp256k1_gej_set_ge(&Qj, pubkey); - secp256k1_ecmult(ctx, &Rj, &Qj, &h, &s); - if (secp256k1_gej_is_infinity(&Rj)) { - return 0; - } - secp256k1_ge_set_gej_var(&Ra, &Rj); - secp256k1_fe_normalize_var(&Ra.y); - if (secp256k1_fe_is_odd(&Ra.y)) { - return 0; - } - return secp256k1_fe_equal_var(&Rx, &Ra.x); -} - -static int secp256k1_schnorr_sig_recover(const secp256k1_ecmult_context* ctx, const unsigned char *sig64, secp256k1_ge *pubkey, secp256k1_schnorr_msghash hash, const unsigned char *msg32) { - secp256k1_gej Qj, Rj; - secp256k1_ge Ra; - secp256k1_fe Rx; - secp256k1_scalar h, s; - unsigned char hh[32]; - int overflow; - - hash(hh, sig64, msg32); - overflow = 0; - secp256k1_scalar_set_b32(&h, hh, &overflow); - if (overflow || secp256k1_scalar_is_zero(&h)) { - return 0; - } - overflow = 0; - secp256k1_scalar_set_b32(&s, sig64 + 32, &overflow); - if (overflow) { - return 0; - } - if (!secp256k1_fe_set_b32(&Rx, sig64)) { - return 0; - } - if (!secp256k1_ge_set_xo_var(&Ra, &Rx, 0)) { - return 0; - } - secp256k1_gej_set_ge(&Rj, &Ra); - secp256k1_scalar_inverse_var(&h, &h); - secp256k1_scalar_negate(&s, &s); - secp256k1_scalar_mul(&s, &s, &h); - secp256k1_ecmult(ctx, &Qj, &Rj, &h, &s); - if (secp256k1_gej_is_infinity(&Qj)) { - return 0; - } - secp256k1_ge_set_gej(pubkey, &Qj); - return 1; -} - -static int secp256k1_schnorr_sig_combine(unsigned char *sig64, size_t n, const unsigned char * const *sig64ins) { - secp256k1_scalar s = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); - size_t i; - for (i = 0; i < n; i++) { - secp256k1_scalar si; - int overflow; - secp256k1_scalar_set_b32(&si, sig64ins[i] + 32, &overflow); - if (overflow) { - return -1; - } - if (i) { - if (memcmp(sig64ins[i - 1], sig64ins[i], 32) != 0) { - return -1; - } - } - secp256k1_scalar_add(&s, &s, &si); - } - if (secp256k1_scalar_is_zero(&s)) { - return 0; - } - memcpy(sig64, sig64ins[0], 32); - secp256k1_scalar_get_b32(sig64 + 32, &s); - secp256k1_scalar_clear(&s); - return 1; -} - -#endif diff --git a/src/secp256k1/src/modules/schnorr/tests_impl.h b/src/secp256k1/src/modules/schnorr/tests_impl.h deleted file mode 100644 index 5bd14a03e3..0000000000 --- a/src/secp256k1/src/modules/schnorr/tests_impl.h +++ /dev/null @@ -1,175 +0,0 @@ -/********************************************************************** - * Copyright (c) 2014-2015 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php.* - **********************************************************************/ - -#ifndef SECP256K1_MODULE_SCHNORR_TESTS -#define SECP256K1_MODULE_SCHNORR_TESTS - -#include "include/secp256k1_schnorr.h" - -void test_schnorr_end_to_end(void) { - unsigned char privkey[32]; - unsigned char message[32]; - unsigned char schnorr_signature[64]; - secp256k1_pubkey pubkey, recpubkey; - - /* Generate a random key and message. */ - { - secp256k1_scalar key; - random_scalar_order_test(&key); - secp256k1_scalar_get_b32(privkey, &key); - secp256k1_rand256_test(message); - } - - /* Construct and verify corresponding public key. */ - CHECK(secp256k1_ec_seckey_verify(ctx, privkey) == 1); - CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey, privkey) == 1); - - /* Schnorr sign. */ - CHECK(secp256k1_schnorr_sign(ctx, schnorr_signature, message, privkey, NULL, NULL) == 1); - CHECK(secp256k1_schnorr_verify(ctx, schnorr_signature, message, &pubkey) == 1); - CHECK(secp256k1_schnorr_recover(ctx, &recpubkey, schnorr_signature, message) == 1); - CHECK(memcmp(&pubkey, &recpubkey, sizeof(pubkey)) == 0); - /* Destroy signature and verify again. */ - schnorr_signature[secp256k1_rand_bits(6)] += 1 + secp256k1_rand_int(255); - CHECK(secp256k1_schnorr_verify(ctx, schnorr_signature, message, &pubkey) == 0); - CHECK(secp256k1_schnorr_recover(ctx, &recpubkey, schnorr_signature, message) != 1 || - memcmp(&pubkey, &recpubkey, sizeof(pubkey)) != 0); -} - -/** Horribly broken hash function. Do not use for anything but tests. */ -void test_schnorr_hash(unsigned char *h32, const unsigned char *r32, const unsigned char *msg32) { - int i; - for (i = 0; i < 32; i++) { - h32[i] = r32[i] ^ msg32[i]; - } -} - -void test_schnorr_sign_verify(void) { - unsigned char msg32[32]; - unsigned char sig64[3][64]; - secp256k1_gej pubkeyj[3]; - secp256k1_ge pubkey[3]; - secp256k1_scalar nonce[3], key[3]; - int i = 0; - int k; - - secp256k1_rand256_test(msg32); - - for (k = 0; k < 3; k++) { - random_scalar_order_test(&key[k]); - - do { - random_scalar_order_test(&nonce[k]); - if (secp256k1_schnorr_sig_sign(&ctx->ecmult_gen_ctx, sig64[k], &key[k], &nonce[k], NULL, &test_schnorr_hash, msg32)) { - break; - } - } while(1); - - secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pubkeyj[k], &key[k]); - secp256k1_ge_set_gej_var(&pubkey[k], &pubkeyj[k]); - CHECK(secp256k1_schnorr_sig_verify(&ctx->ecmult_ctx, sig64[k], &pubkey[k], &test_schnorr_hash, msg32)); - - for (i = 0; i < 4; i++) { - int pos = secp256k1_rand_bits(6); - int mod = 1 + secp256k1_rand_int(255); - sig64[k][pos] ^= mod; - CHECK(secp256k1_schnorr_sig_verify(&ctx->ecmult_ctx, sig64[k], &pubkey[k], &test_schnorr_hash, msg32) == 0); - sig64[k][pos] ^= mod; - } - } -} - -void test_schnorr_threshold(void) { - unsigned char msg[32]; - unsigned char sec[5][32]; - secp256k1_pubkey pub[5]; - unsigned char nonce[5][32]; - secp256k1_pubkey pubnonce[5]; - unsigned char sig[5][64]; - const unsigned char* sigs[5]; - unsigned char allsig[64]; - const secp256k1_pubkey* pubs[5]; - secp256k1_pubkey allpub; - int n, i; - int damage; - int ret = 0; - - damage = secp256k1_rand_bits(1) ? (1 + secp256k1_rand_int(4)) : 0; - secp256k1_rand256_test(msg); - n = 2 + secp256k1_rand_int(4); - for (i = 0; i < n; i++) { - do { - secp256k1_rand256_test(sec[i]); - } while (!secp256k1_ec_seckey_verify(ctx, sec[i])); - CHECK(secp256k1_ec_pubkey_create(ctx, &pub[i], sec[i])); - CHECK(secp256k1_schnorr_generate_nonce_pair(ctx, &pubnonce[i], nonce[i], msg, sec[i], NULL, NULL)); - pubs[i] = &pub[i]; - } - if (damage == 1) { - nonce[secp256k1_rand_int(n)][secp256k1_rand_int(32)] ^= 1 + secp256k1_rand_int(255); - } else if (damage == 2) { - sec[secp256k1_rand_int(n)][secp256k1_rand_int(32)] ^= 1 + secp256k1_rand_int(255); - } - for (i = 0; i < n; i++) { - secp256k1_pubkey allpubnonce; - const secp256k1_pubkey *pubnonces[4]; - int j; - for (j = 0; j < i; j++) { - pubnonces[j] = &pubnonce[j]; - } - for (j = i + 1; j < n; j++) { - pubnonces[j - 1] = &pubnonce[j]; - } - CHECK(secp256k1_ec_pubkey_combine(ctx, &allpubnonce, pubnonces, n - 1)); - ret |= (secp256k1_schnorr_partial_sign(ctx, sig[i], msg, sec[i], &allpubnonce, nonce[i]) != 1) * 1; - sigs[i] = sig[i]; - } - if (damage == 3) { - sig[secp256k1_rand_int(n)][secp256k1_rand_bits(6)] ^= 1 + secp256k1_rand_int(255); - } - ret |= (secp256k1_ec_pubkey_combine(ctx, &allpub, pubs, n) != 1) * 2; - if ((ret & 1) == 0) { - ret |= (secp256k1_schnorr_partial_combine(ctx, allsig, sigs, n) != 1) * 4; - } - if (damage == 4) { - allsig[secp256k1_rand_int(32)] ^= 1 + secp256k1_rand_int(255); - } - if ((ret & 7) == 0) { - ret |= (secp256k1_schnorr_verify(ctx, allsig, msg, &allpub) != 1) * 8; - } - CHECK((ret == 0) == (damage == 0)); -} - -void test_schnorr_recovery(void) { - unsigned char msg32[32]; - unsigned char sig64[64]; - secp256k1_ge Q; - - secp256k1_rand256_test(msg32); - secp256k1_rand256_test(sig64); - secp256k1_rand256_test(sig64 + 32); - if (secp256k1_schnorr_sig_recover(&ctx->ecmult_ctx, sig64, &Q, &test_schnorr_hash, msg32) == 1) { - CHECK(secp256k1_schnorr_sig_verify(&ctx->ecmult_ctx, sig64, &Q, &test_schnorr_hash, msg32) == 1); - } -} - -void run_schnorr_tests(void) { - int i; - for (i = 0; i < 32*count; i++) { - test_schnorr_end_to_end(); - } - for (i = 0; i < 32 * count; i++) { - test_schnorr_sign_verify(); - } - for (i = 0; i < 16 * count; i++) { - test_schnorr_recovery(); - } - for (i = 0; i < 10 * count; i++) { - test_schnorr_threshold(); - } -} - -#endif diff --git a/src/secp256k1/src/scalar.h b/src/secp256k1/src/scalar.h index b590ccd6dd..27e9d8375e 100644 --- a/src/secp256k1/src/scalar.h +++ b/src/secp256k1/src/scalar.h @@ -13,7 +13,9 @@ #include "libsecp256k1-config.h" #endif -#if defined(USE_SCALAR_4X64) +#if defined(EXHAUSTIVE_TEST_ORDER) +#include "scalar_low.h" +#elif defined(USE_SCALAR_4X64) #include "scalar_4x64.h" #elif defined(USE_SCALAR_8X32) #include "scalar_8x32.h" diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index aa2703dd23..56e7bd82af 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -282,8 +282,8 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "movq 56(%%rsi), %%r14\n" /* Initialize r8,r9,r10 */ "movq 0(%%rsi), %%r8\n" - "movq $0, %%r9\n" - "movq $0, %%r10\n" + "xorq %%r9, %%r9\n" + "xorq %%r10, %%r10\n" /* (r8,r9) += n0 * c0 */ "movq %8, %%rax\n" "mulq %%r11\n" @@ -291,7 +291,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq %%rdx, %%r9\n" /* extract m0 */ "movq %%r8, %q0\n" - "movq $0, %%r8\n" + "xorq %%r8, %%r8\n" /* (r9,r10) += l1 */ "addq 8(%%rsi), %%r9\n" "adcq $0, %%r10\n" @@ -309,7 +309,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r8\n" /* extract m1 */ "movq %%r9, %q1\n" - "movq $0, %%r9\n" + "xorq %%r9, %%r9\n" /* (r10,r8,r9) += l2 */ "addq 16(%%rsi), %%r10\n" "adcq $0, %%r8\n" @@ -332,7 +332,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r9\n" /* extract m2 */ "movq %%r10, %q2\n" - "movq $0, %%r10\n" + "xorq %%r10, %%r10\n" /* (r8,r9,r10) += l3 */ "addq 24(%%rsi), %%r8\n" "adcq $0, %%r9\n" @@ -355,7 +355,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r10\n" /* extract m3 */ "movq %%r8, %q3\n" - "movq $0, %%r8\n" + "xorq %%r8, %%r8\n" /* (r9,r10,r8) += n3 * c1 */ "movq %9, %%rax\n" "mulq %%r14\n" @@ -387,8 +387,8 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "movq %q11, %%r13\n" /* Initialize (r8,r9,r10) */ "movq %q5, %%r8\n" - "movq $0, %%r9\n" - "movq $0, %%r10\n" + "xorq %%r9, %%r9\n" + "xorq %%r10, %%r10\n" /* (r8,r9) += m4 * c0 */ "movq %12, %%rax\n" "mulq %%r11\n" @@ -396,7 +396,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq %%rdx, %%r9\n" /* extract p0 */ "movq %%r8, %q0\n" - "movq $0, %%r8\n" + "xorq %%r8, %%r8\n" /* (r9,r10) += m1 */ "addq %q6, %%r9\n" "adcq $0, %%r10\n" @@ -414,7 +414,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r8\n" /* extract p1 */ "movq %%r9, %q1\n" - "movq $0, %%r9\n" + "xorq %%r9, %%r9\n" /* (r10,r8,r9) += m2 */ "addq %q7, %%r10\n" "adcq $0, %%r8\n" @@ -472,7 +472,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "movq %%rax, 0(%q6)\n" /* Move to (r8,r9) */ "movq %%rdx, %%r8\n" - "movq $0, %%r9\n" + "xorq %%r9, %%r9\n" /* (r8,r9) += p1 */ "addq %q2, %%r8\n" "adcq $0, %%r9\n" @@ -483,7 +483,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq %%rdx, %%r9\n" /* Extract r1 */ "movq %%r8, 8(%q6)\n" - "movq $0, %%r8\n" + "xorq %%r8, %%r8\n" /* (r9,r8) += p4 */ "addq %%r10, %%r9\n" "adcq $0, %%r8\n" @@ -492,7 +492,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r8\n" /* Extract r2 */ "movq %%r9, 16(%q6)\n" - "movq $0, %%r9\n" + "xorq %%r9, %%r9\n" /* (r8,r9) += p3 */ "addq %q4, %%r8\n" "adcq $0, %%r9\n" diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index c5baf4df41..f5b2376407 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -14,7 +14,9 @@ #include "libsecp256k1-config.h" #endif -#if defined(USE_SCALAR_4X64) +#if defined(EXHAUSTIVE_TEST_ORDER) +#include "scalar_low_impl.h" +#elif defined(USE_SCALAR_4X64) #include "scalar_4x64_impl.h" #elif defined(USE_SCALAR_8X32) #include "scalar_8x32_impl.h" @@ -31,17 +33,37 @@ static void secp256k1_scalar_get_num(secp256k1_num *r, const secp256k1_scalar *a /** secp256k1 curve order, see secp256k1_ecdsa_const_order_as_fe in ecdsa_impl.h */ static void secp256k1_scalar_order_get_num(secp256k1_num *r) { +#if defined(EXHAUSTIVE_TEST_ORDER) + static const unsigned char order[32] = { + 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,EXHAUSTIVE_TEST_ORDER + }; +#else static const unsigned char order[32] = { 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF, 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFE, 0xBA,0xAE,0xDC,0xE6,0xAF,0x48,0xA0,0x3B, 0xBF,0xD2,0x5E,0x8C,0xD0,0x36,0x41,0x41 }; +#endif secp256k1_num_set_bin(r, order, 32); } #endif static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar *x) { +#if defined(EXHAUSTIVE_TEST_ORDER) + int i; + *r = 0; + for (i = 0; i < EXHAUSTIVE_TEST_ORDER; i++) + if ((i * *x) % EXHAUSTIVE_TEST_ORDER == 1) + *r = i; + /* If this VERIFY_CHECK triggers we were given a noninvertible scalar (and thus + * have a composite group order; fix it in exhaustive_tests.c). */ + VERIFY_CHECK(*r != 0); +} +#else secp256k1_scalar *t; int i; /* First compute x ^ (2^N - 1) for some values of N. */ @@ -233,9 +255,9 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar } SECP256K1_INLINE static int secp256k1_scalar_is_even(const secp256k1_scalar *a) { - /* d[0] is present and is the lowest word for all representations */ return !(a->d[0] & 1); } +#endif static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_scalar *x) { #if defined(USE_SCALAR_INV_BUILTIN) @@ -259,6 +281,18 @@ static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_sc } #ifdef USE_ENDOMORPHISM +#if defined(EXHAUSTIVE_TEST_ORDER) +/** + * Find k1 and k2 given k, such that k1 + k2 * lambda == k mod n; unlike in the + * full case we don't bother making k1 and k2 be small, we just want them to be + * nontrivial to get full test coverage for the exhaustive tests. We therefore + * (arbitrarily) set k2 = k + 5 and k1 = k - k2 * lambda. + */ +static void secp256k1_scalar_split_lambda(secp256k1_scalar *r1, secp256k1_scalar *r2, const secp256k1_scalar *a) { + *r2 = (*a + 5) % EXHAUSTIVE_TEST_ORDER; + *r1 = (*a + (EXHAUSTIVE_TEST_ORDER - *r2) * EXHAUSTIVE_TEST_LAMBDA) % EXHAUSTIVE_TEST_ORDER; +} +#else /** * The Secp256k1 curve has an endomorphism, where lambda * (x, y) = (beta * x, y), where * lambda is {0x53,0x63,0xad,0x4c,0xc0,0x5c,0x30,0xe0,0xa5,0x26,0x1c,0x02,0x88,0x12,0x64,0x5a, @@ -331,5 +365,6 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar *r1, secp256k1_scalar secp256k1_scalar_add(r1, r1, a); } #endif +#endif #endif diff --git a/src/secp256k1/src/scalar_low.h b/src/secp256k1/src/scalar_low.h new file mode 100644 index 0000000000..5574c44c7a --- /dev/null +++ b/src/secp256k1/src/scalar_low.h @@ -0,0 +1,15 @@ +/********************************************************************** + * Copyright (c) 2015 Andrew Poelstra * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _SECP256K1_SCALAR_REPR_ +#define _SECP256K1_SCALAR_REPR_ + +#include <stdint.h> + +/** A scalar modulo the group order of the secp256k1 curve. */ +typedef uint32_t secp256k1_scalar; + +#endif diff --git a/src/secp256k1/src/scalar_low_impl.h b/src/secp256k1/src/scalar_low_impl.h new file mode 100644 index 0000000000..4f94441f49 --- /dev/null +++ b/src/secp256k1/src/scalar_low_impl.h @@ -0,0 +1,114 @@ +/********************************************************************** + * Copyright (c) 2015 Andrew Poelstra * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _SECP256K1_SCALAR_REPR_IMPL_H_ +#define _SECP256K1_SCALAR_REPR_IMPL_H_ + +#include "scalar.h" + +#include <string.h> + +SECP256K1_INLINE static int secp256k1_scalar_is_even(const secp256k1_scalar *a) { + return !(*a & 1); +} + +SECP256K1_INLINE static void secp256k1_scalar_clear(secp256k1_scalar *r) { *r = 0; } +SECP256K1_INLINE static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsigned int v) { *r = v; } + +SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { + if (offset < 32) + return ((*a >> offset) & ((((uint32_t)1) << count) - 1)); + else + return 0; +} + +SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits_var(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { + return secp256k1_scalar_get_bits(a, offset, count); +} + +SECP256K1_INLINE static int secp256k1_scalar_check_overflow(const secp256k1_scalar *a) { return *a >= EXHAUSTIVE_TEST_ORDER; } + +static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { + *r = (*a + *b) % EXHAUSTIVE_TEST_ORDER; + return *r < *b; +} + +static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { + if (flag && bit < 32) + *r += (1 << bit); +#ifdef VERIFY + VERIFY_CHECK(secp256k1_scalar_check_overflow(r) == 0); +#endif +} + +static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b32, int *overflow) { + const int base = 0x100 % EXHAUSTIVE_TEST_ORDER; + int i; + *r = 0; + for (i = 0; i < 32; i++) { + *r = ((*r * base) + b32[i]) % EXHAUSTIVE_TEST_ORDER; + } + /* just deny overflow, it basically always happens */ + if (overflow) *overflow = 0; +} + +static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* a) { + memset(bin, 0, 32); + bin[28] = *a >> 24; bin[29] = *a >> 16; bin[30] = *a >> 8; bin[31] = *a; +} + +SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) { + return *a == 0; +} + +static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar *a) { + if (*a == 0) { + *r = 0; + } else { + *r = EXHAUSTIVE_TEST_ORDER - *a; + } +} + +SECP256K1_INLINE static int secp256k1_scalar_is_one(const secp256k1_scalar *a) { + return *a == 1; +} + +static int secp256k1_scalar_is_high(const secp256k1_scalar *a) { + return *a > EXHAUSTIVE_TEST_ORDER / 2; +} + +static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { + if (flag) secp256k1_scalar_negate(r, r); + return flag ? -1 : 1; +} + +static void secp256k1_scalar_mul(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { + *r = (*a * *b) % EXHAUSTIVE_TEST_ORDER; +} + +static int secp256k1_scalar_shr_int(secp256k1_scalar *r, int n) { + int ret; + VERIFY_CHECK(n > 0); + VERIFY_CHECK(n < 16); + ret = *r & ((1 << n) - 1); + *r >>= n; + return ret; +} + +static void secp256k1_scalar_sqr(secp256k1_scalar *r, const secp256k1_scalar *a) { + *r = (*a * *a) % EXHAUSTIVE_TEST_ORDER; +} + +static void secp256k1_scalar_split_128(secp256k1_scalar *r1, secp256k1_scalar *r2, const secp256k1_scalar *a) { + *r1 = *a; + *r2 = 0; +} + +SECP256K1_INLINE static int secp256k1_scalar_eq(const secp256k1_scalar *a, const secp256k1_scalar *b) { + return *a == *b; +} + +#endif diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 7973d60c36..fb8b882faa 100644..100755 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -359,16 +359,15 @@ int secp256k1_ecdsa_sign(const secp256k1_context* ctx, secp256k1_ecdsa_signature secp256k1_scalar_set_b32(&sec, seckey, &overflow); /* Fail if the secret key is invalid. */ if (!overflow && !secp256k1_scalar_is_zero(&sec)) { + unsigned char nonce32[32]; unsigned int count = 0; secp256k1_scalar_set_b32(&msg, msg32, NULL); while (1) { - unsigned char nonce32[32]; ret = noncefp(nonce32, msg32, seckey, NULL, (void*)noncedata, count); if (!ret) { break; } secp256k1_scalar_set_b32(&non, nonce32, &overflow); - memset(nonce32, 0, 32); if (!overflow && !secp256k1_scalar_is_zero(&non)) { if (secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &r, &s, &sec, &msg, &non, NULL)) { break; @@ -376,6 +375,7 @@ int secp256k1_ecdsa_sign(const secp256k1_context* ctx, secp256k1_ecdsa_signature } count++; } + memset(nonce32, 0, 32); secp256k1_scalar_clear(&msg); secp256k1_scalar_clear(&non); secp256k1_scalar_clear(&sec); diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index b32cb90813..9ae7d30281 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -520,7 +520,7 @@ void test_num_mod(void) { secp256k1_num order, n; /* check that 0 mod anything is 0 */ - random_scalar_order_test(&s); + random_scalar_order_test(&s); secp256k1_scalar_get_num(&order, &s); secp256k1_scalar_set_int(&s, 0); secp256k1_scalar_get_num(&n, &s); @@ -535,7 +535,7 @@ void test_num_mod(void) { CHECK(secp256k1_num_is_zero(&n)); /* check that increasing the number past 2^256 does not break this */ - random_scalar_order_test(&s); + random_scalar_order_test(&s); secp256k1_scalar_get_num(&n, &s); /* multiply by 2^8, which'll test this case with high probability */ for (i = 0; i < 8; ++i) { @@ -568,7 +568,7 @@ void test_num_jacobi(void) { /* we first need a scalar which is not a multiple of 5 */ do { secp256k1_num fiven; - random_scalar_order_test(&sqr); + random_scalar_order_test(&sqr); secp256k1_scalar_get_num(&fiven, &five); secp256k1_scalar_get_num(&n, &sqr); secp256k1_num_mod(&n, &fiven); @@ -587,7 +587,7 @@ void test_num_jacobi(void) { /** test with secp group order as order */ secp256k1_scalar_order_get_num(&order); - random_scalar_order_test(&sqr); + random_scalar_order_test(&sqr); secp256k1_scalar_sqr(&sqr, &sqr); /* test residue */ secp256k1_scalar_get_num(&n, &sqr); @@ -1733,18 +1733,18 @@ void run_field_inv_all_var(void) { secp256k1_fe x[16], xi[16], xii[16]; int i; /* Check it's safe to call for 0 elements */ - secp256k1_fe_inv_all_var(0, xi, x); + secp256k1_fe_inv_all_var(xi, x, 0); for (i = 0; i < count; i++) { size_t j; size_t len = secp256k1_rand_int(15) + 1; for (j = 0; j < len; j++) { random_fe_non_zero(&x[j]); } - secp256k1_fe_inv_all_var(len, xi, x); + secp256k1_fe_inv_all_var(xi, x, len); for (j = 0; j < len; j++) { CHECK(check_fe_inverse(&x[j], &xi[j])); } - secp256k1_fe_inv_all_var(len, xii, xi); + secp256k1_fe_inv_all_var(xii, xi, len); for (j = 0; j < len; j++) { CHECK(check_fe_equal(&x[j], &xii[j])); } @@ -1930,7 +1930,7 @@ void test_ge(void) { zs[i] = gej[i].z; } } - secp256k1_fe_inv_all_var(4 * runs + 1, zinv, zs); + secp256k1_fe_inv_all_var(zinv, zs, 4 * runs + 1); free(zs); } @@ -2050,8 +2050,8 @@ void test_ge(void) { secp256k1_fe_mul(&zr[i + 1], &zinv[i], &gej[i + 1].z); } } - secp256k1_ge_set_table_gej_var(4 * runs + 1, ge_set_table, gej, zr); - secp256k1_ge_set_all_gej_var(4 * runs + 1, ge_set_all, gej, &ctx->error_callback); + secp256k1_ge_set_table_gej_var(ge_set_table, gej, zr, 4 * runs + 1); + secp256k1_ge_set_all_gej_var(ge_set_all, gej, 4 * runs + 1, &ctx->error_callback); for (i = 0; i < 4 * runs + 1; i++) { secp256k1_fe s; random_fe_non_zero(&s); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c new file mode 100644 index 0000000000..bda6ee475c --- /dev/null +++ b/src/secp256k1/src/tests_exhaustive.c @@ -0,0 +1,329 @@ +/*********************************************************************** + * Copyright (c) 2016 Andrew Poelstra * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#if defined HAVE_CONFIG_H +#include "libsecp256k1-config.h" +#endif + +#include <stdio.h> +#include <stdlib.h> + +#include <time.h> + +#undef USE_ECMULT_STATIC_PRECOMPUTATION + +#ifndef EXHAUSTIVE_TEST_ORDER +/* see group_impl.h for allowable values */ +#define EXHAUSTIVE_TEST_ORDER 13 +#define EXHAUSTIVE_TEST_LAMBDA 9 /* cube root of 1 mod 13 */ +#endif + +#include "include/secp256k1.h" +#include "group.h" +#include "secp256k1.c" +#include "testrand_impl.h" + +/** stolen from tests.c */ +void ge_equals_ge(const secp256k1_ge *a, const secp256k1_ge *b) { + CHECK(a->infinity == b->infinity); + if (a->infinity) { + return; + } + CHECK(secp256k1_fe_equal_var(&a->x, &b->x)); + CHECK(secp256k1_fe_equal_var(&a->y, &b->y)); +} + +void ge_equals_gej(const secp256k1_ge *a, const secp256k1_gej *b) { + secp256k1_fe z2s; + secp256k1_fe u1, u2, s1, s2; + CHECK(a->infinity == b->infinity); + if (a->infinity) { + return; + } + /* Check a.x * b.z^2 == b.x && a.y * b.z^3 == b.y, to avoid inverses. */ + secp256k1_fe_sqr(&z2s, &b->z); + secp256k1_fe_mul(&u1, &a->x, &z2s); + u2 = b->x; secp256k1_fe_normalize_weak(&u2); + secp256k1_fe_mul(&s1, &a->y, &z2s); secp256k1_fe_mul(&s1, &s1, &b->z); + s2 = b->y; secp256k1_fe_normalize_weak(&s2); + CHECK(secp256k1_fe_equal_var(&u1, &u2)); + CHECK(secp256k1_fe_equal_var(&s1, &s2)); +} + +void random_fe(secp256k1_fe *x) { + unsigned char bin[32]; + do { + secp256k1_rand256(bin); + if (secp256k1_fe_set_b32(x, bin)) { + return; + } + } while(1); +} +/** END stolen from tests.c */ + +int secp256k1_nonce_function_smallint(unsigned char *nonce32, const unsigned char *msg32, + const unsigned char *key32, const unsigned char *algo16, + void *data, unsigned int attempt) { + secp256k1_scalar s; + int *idata = data; + (void)msg32; + (void)key32; + (void)algo16; + /* Some nonces cannot be used because they'd cause s and/or r to be zero. + * The signing function has retry logic here that just re-calls the nonce + * function with an increased `attempt`. So if attempt > 0 this means we + * need to change the nonce to avoid an infinite loop. */ + if (attempt > 0) { + (*idata)++; + } + secp256k1_scalar_set_int(&s, *idata); + secp256k1_scalar_get_b32(nonce32, &s); + return 1; +} + +#ifdef USE_ENDOMORPHISM +void test_exhaustive_endomorphism(const secp256k1_ge *group, int order) { + int i; + for (i = 0; i < order; i++) { + secp256k1_ge res; + secp256k1_ge_mul_lambda(&res, &group[i]); + ge_equals_ge(&group[i * EXHAUSTIVE_TEST_LAMBDA % EXHAUSTIVE_TEST_ORDER], &res); + } +} +#endif + +void test_exhaustive_addition(const secp256k1_ge *group, const secp256k1_gej *groupj, int order) { + int i, j; + + /* Sanity-check (and check infinity functions) */ + CHECK(secp256k1_ge_is_infinity(&group[0])); + CHECK(secp256k1_gej_is_infinity(&groupj[0])); + for (i = 1; i < order; i++) { + CHECK(!secp256k1_ge_is_infinity(&group[i])); + CHECK(!secp256k1_gej_is_infinity(&groupj[i])); + } + + /* Check all addition formulae */ + for (j = 0; j < order; j++) { + secp256k1_fe fe_inv; + secp256k1_fe_inv(&fe_inv, &groupj[j].z); + for (i = 0; i < order; i++) { + secp256k1_ge zless_gej; + secp256k1_gej tmp; + /* add_var */ + secp256k1_gej_add_var(&tmp, &groupj[i], &groupj[j], NULL); + ge_equals_gej(&group[(i + j) % order], &tmp); + /* add_ge */ + if (j > 0) { + secp256k1_gej_add_ge(&tmp, &groupj[i], &group[j]); + ge_equals_gej(&group[(i + j) % order], &tmp); + } + /* add_ge_var */ + secp256k1_gej_add_ge_var(&tmp, &groupj[i], &group[j], NULL); + ge_equals_gej(&group[(i + j) % order], &tmp); + /* add_zinv_var */ + zless_gej.infinity = groupj[j].infinity; + zless_gej.x = groupj[j].x; + zless_gej.y = groupj[j].y; + secp256k1_gej_add_zinv_var(&tmp, &groupj[i], &zless_gej, &fe_inv); + ge_equals_gej(&group[(i + j) % order], &tmp); + } + } + + /* Check doubling */ + for (i = 0; i < order; i++) { + secp256k1_gej tmp; + if (i > 0) { + secp256k1_gej_double_nonzero(&tmp, &groupj[i], NULL); + ge_equals_gej(&group[(2 * i) % order], &tmp); + } + secp256k1_gej_double_var(&tmp, &groupj[i], NULL); + ge_equals_gej(&group[(2 * i) % order], &tmp); + } + + /* Check negation */ + for (i = 1; i < order; i++) { + secp256k1_ge tmp; + secp256k1_gej tmpj; + secp256k1_ge_neg(&tmp, &group[i]); + ge_equals_ge(&group[order - i], &tmp); + secp256k1_gej_neg(&tmpj, &groupj[i]); + ge_equals_gej(&group[order - i], &tmpj); + } +} + +void test_exhaustive_ecmult(const secp256k1_context *ctx, const secp256k1_ge *group, const secp256k1_gej *groupj, int order) { + int i, j, r_log; + for (r_log = 1; r_log < order; r_log++) { + for (j = 0; j < order; j++) { + for (i = 0; i < order; i++) { + secp256k1_gej tmp; + secp256k1_scalar na, ng; + secp256k1_scalar_set_int(&na, i); + secp256k1_scalar_set_int(&ng, j); + + secp256k1_ecmult(&ctx->ecmult_ctx, &tmp, &groupj[r_log], &na, &ng); + ge_equals_gej(&group[(i * r_log + j) % order], &tmp); + + if (i > 0) { + secp256k1_ecmult_const(&tmp, &group[i], &ng); + ge_equals_gej(&group[(i * j) % order], &tmp); + } + } + } + } +} + +void r_from_k(secp256k1_scalar *r, const secp256k1_ge *group, int k) { + secp256k1_fe x; + unsigned char x_bin[32]; + k %= EXHAUSTIVE_TEST_ORDER; + x = group[k].x; + secp256k1_fe_normalize(&x); + secp256k1_fe_get_b32(x_bin, &x); + secp256k1_scalar_set_b32(r, x_bin, NULL); +} + +void test_exhaustive_verify(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + int s, r, msg, key; + for (s = 1; s < order; s++) { + for (r = 1; r < order; r++) { + for (msg = 1; msg < order; msg++) { + for (key = 1; key < order; key++) { + secp256k1_ge nonconst_ge; + secp256k1_ecdsa_signature sig; + secp256k1_pubkey pk; + secp256k1_scalar sk_s, msg_s, r_s, s_s; + secp256k1_scalar s_times_k_s, msg_plus_r_times_sk_s; + int k, should_verify; + unsigned char msg32[32]; + + secp256k1_scalar_set_int(&s_s, s); + secp256k1_scalar_set_int(&r_s, r); + secp256k1_scalar_set_int(&msg_s, msg); + secp256k1_scalar_set_int(&sk_s, key); + + /* Verify by hand */ + /* Run through every k value that gives us this r and check that *one* works. + * Note there could be none, there could be multiple, ECDSA is weird. */ + should_verify = 0; + for (k = 0; k < order; k++) { + secp256k1_scalar check_x_s; + r_from_k(&check_x_s, group, k); + if (r_s == check_x_s) { + secp256k1_scalar_set_int(&s_times_k_s, k); + secp256k1_scalar_mul(&s_times_k_s, &s_times_k_s, &s_s); + secp256k1_scalar_mul(&msg_plus_r_times_sk_s, &r_s, &sk_s); + secp256k1_scalar_add(&msg_plus_r_times_sk_s, &msg_plus_r_times_sk_s, &msg_s); + should_verify |= secp256k1_scalar_eq(&s_times_k_s, &msg_plus_r_times_sk_s); + } + } + /* nb we have a "high s" rule */ + should_verify &= !secp256k1_scalar_is_high(&s_s); + + /* Verify by calling verify */ + secp256k1_ecdsa_signature_save(&sig, &r_s, &s_s); + memcpy(&nonconst_ge, &group[sk_s], sizeof(nonconst_ge)); + secp256k1_pubkey_save(&pk, &nonconst_ge); + secp256k1_scalar_get_b32(msg32, &msg_s); + CHECK(should_verify == + secp256k1_ecdsa_verify(ctx, &sig, msg32, &pk)); + } + } + } + } +} + +void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + int i, j, k; + + /* Loop */ + for (i = 1; i < order; i++) { /* message */ + for (j = 1; j < order; j++) { /* key */ + for (k = 1; k < order; k++) { /* nonce */ + secp256k1_ecdsa_signature sig; + secp256k1_scalar sk, msg, r, s, expected_r; + unsigned char sk32[32], msg32[32]; + secp256k1_scalar_set_int(&msg, i); + secp256k1_scalar_set_int(&sk, j); + secp256k1_scalar_get_b32(sk32, &sk); + secp256k1_scalar_get_b32(msg32, &msg); + + secp256k1_ecdsa_sign(ctx, &sig, msg32, sk32, secp256k1_nonce_function_smallint, &k); + + secp256k1_ecdsa_signature_load(ctx, &r, &s, &sig); + /* Note that we compute expected_r *after* signing -- this is important + * because our nonce-computing function function might change k during + * signing. */ + r_from_k(&expected_r, group, k); + CHECK(r == expected_r); + CHECK((k * s) % order == (i + r * j) % order || + (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + } + } + } + + /* We would like to verify zero-knowledge here by counting how often every + * possible (s, r) tuple appears, but because the group order is larger + * than the field order, when coercing the x-values to scalar values, some + * appear more often than others, so we are actually not zero-knowledge. + * (This effect also appears in the real code, but the difference is on the + * order of 1/2^128th the field order, so the deviation is not useful to a + * computationally bounded attacker.) + */ +} + +int main(void) { + int i; + secp256k1_gej groupj[EXHAUSTIVE_TEST_ORDER]; + secp256k1_ge group[EXHAUSTIVE_TEST_ORDER]; + + /* Build context */ + secp256k1_context *ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + + /* TODO set z = 1, then do num_tests runs with random z values */ + + /* Generate the entire group */ + secp256k1_gej_set_infinity(&groupj[0]); + secp256k1_ge_set_gej(&group[0], &groupj[0]); + for (i = 1; i < EXHAUSTIVE_TEST_ORDER; i++) { + /* Set a different random z-value for each Jacobian point */ + secp256k1_fe z; + random_fe(&z); + + secp256k1_gej_add_ge(&groupj[i], &groupj[i - 1], &secp256k1_ge_const_g); + secp256k1_ge_set_gej(&group[i], &groupj[i]); + secp256k1_gej_rescale(&groupj[i], &z); + + /* Verify against ecmult_gen */ + { + secp256k1_scalar scalar_i; + secp256k1_gej generatedj; + secp256k1_ge generated; + + secp256k1_scalar_set_int(&scalar_i, i); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &generatedj, &scalar_i); + secp256k1_ge_set_gej(&generated, &generatedj); + + CHECK(group[i].infinity == 0); + CHECK(generated.infinity == 0); + CHECK(secp256k1_fe_equal_var(&generated.x, &group[i].x)); + CHECK(secp256k1_fe_equal_var(&generated.y, &group[i].y)); + } + } + + /* Run the tests */ +#ifdef USE_ENDOMORPHISM + test_exhaustive_endomorphism(group, EXHAUSTIVE_TEST_ORDER); +#endif + test_exhaustive_addition(group, groupj, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_ecmult(ctx, group, groupj, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_sign(ctx, group, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_verify(ctx, group, EXHAUSTIVE_TEST_ORDER); + + return 0; +} + diff --git a/src/serialize.h b/src/serialize.h index e28ca548c0..2be6f005e3 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -59,34 +59,6 @@ inline T* NCONST_PTR(const T* val) return const_cast<T*>(val); } -/** - * Important: Do not use the following functions in new code, but use v.data() - * and v.data() + v.size() respectively directly. They were once introduced to - * have a compatible, safe way to get the begin and end pointer of a vector. - * However with C++11 the language has built-in functionality for this and it's - * more readable to just use that. - */ -template <typename V> -inline typename V::value_type* begin_ptr(V& v) -{ - return v.data(); -} -template <typename V> -inline const typename V::value_type* begin_ptr(const V& v) -{ - return v.data(); -} -template <typename V> -inline typename V::value_type* end_ptr(V& v) -{ - return v.data() + v.size(); -} -template <typename V> -inline const typename V::value_type* end_ptr(const V& v) -{ - return v.data() + v.size(); -} - /* * Lowest-level serialization and conversion. * @note Sizes of these types are verified in the tests @@ -390,14 +362,14 @@ public: template <class T, class TAl> explicit CFlatData(std::vector<T,TAl> &v) { - pbegin = (char*)begin_ptr(v); - pend = (char*)end_ptr(v); + pbegin = (char*)v.data(); + pend = (char*)(v.data() + v.size()); } template <unsigned int N, typename T, typename S, typename D> explicit CFlatData(prevector<N, T, S, D> &v) { - pbegin = (char*)begin_ptr(v); - pend = (char*)end_ptr(v); + pbegin = (char*)v.data(); + pend = (char*)(v.data() + v.size()); } char* begin() { return pbegin; } const char* begin() const { return pbegin; } diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 6eed636080..131d667e0b 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -6,8 +6,8 @@ #include "chainparams.h" #include "keystore.h" -#include "main.h" #include "net.h" +#include "net_processing.h" #include "pow.h" #include "script/sign.h" #include "serialize.h" @@ -29,9 +29,9 @@ extern unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans); struct COrphanTx { CTransaction tx; NodeId fromPeer; + int64_t nTimeExpire; }; extern std::map<uint256, COrphanTx> mapOrphanTransactions; -extern std::map<uint256, std::set<uint256> > mapOrphanTransactionsByPrev; CService ip(uint32_t i) { @@ -202,7 +202,6 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) BOOST_CHECK(mapOrphanTransactions.size() <= 10); LimitOrphanTxSize(0); BOOST_CHECK(mapOrphanTransactions.empty()); - BOOST_CHECK(mapOrphanTransactionsByPrev.empty()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index ac3ab4c83f..04a6506655 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58) std::vector<unsigned char> sourcedata = ParseHex(test[0].get_str()); std::string base58string = test[1].get_str(); BOOST_CHECK_MESSAGE( - EncodeBase58(begin_ptr(sourcedata), end_ptr(sourcedata)) == base58string, + EncodeBase58(sourcedata.data(), sourcedata.data() + sourcedata.size()) == base58string, strTest); } } diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index b013cda6d7..7478758f73 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -80,9 +80,9 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); - std::vector<CTransactionRef> removed; - pool.removeRecursive(*block.vtx[2], &removed); - BOOST_CHECK_EQUAL(removed.size(), 1); + size_t poolSize = pool.size(); + pool.removeRecursive(*block.vtx[2]); + BOOST_CHECK_EQUAL(pool.size(), poolSize - 1); CBlock block2; std::vector<CTransactionRef> vtx_missing; diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 25fb9ea2b7..b15ff9e44b 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -30,15 +30,15 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) CBloomFilter filter(3, 0.01, 0, BLOOM_UPDATE_ALL); filter.insert(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")); - BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "BloomFilter doesn't contain just-inserted object!"); + BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); // One bit different in first byte - BOOST_CHECK_MESSAGE(!filter.contains(ParseHex("19108ad8ed9bb6274d3980bab5a85c048f0950c8")), "BloomFilter contains something it shouldn't!"); + BOOST_CHECK_MESSAGE(!filter.contains(ParseHex("19108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter contains something it shouldn't!"); filter.insert(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")), "BloomFilter doesn't contain just-inserted object (2)!"); + BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")), "Bloom filter doesn't contain just-inserted object (2)!"); filter.insert(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), "BloomFilter doesn't contain just-inserted object (3)!"); + BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), "Bloom filter doesn't contain just-inserted object (3)!"); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; @@ -51,9 +51,9 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); - BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "BloomFilter doesn't contain just-inserted object!"); + BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); filter.clear(); - BOOST_CHECK_MESSAGE( !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "BloomFilter should be empty!"); + BOOST_CHECK_MESSAGE( !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter should be empty!"); } BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) @@ -62,15 +62,15 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) CBloomFilter filter(3, 0.01, 2147483649UL, BLOOM_UPDATE_ALL); filter.insert(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")); - BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "BloomFilter doesn't contain just-inserted object!"); + BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); // One bit different in first byte - BOOST_CHECK_MESSAGE(!filter.contains(ParseHex("19108ad8ed9bb6274d3980bab5a85c048f0950c8")), "BloomFilter contains something it shouldn't!"); + BOOST_CHECK_MESSAGE(!filter.contains(ParseHex("19108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter contains something it shouldn't!"); filter.insert(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")), "BloomFilter doesn't contain just-inserted object (2)!"); + BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")), "Bloom filter doesn't contain just-inserted object (2)!"); filter.insert(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), "BloomFilter doesn't contain just-inserted object (3)!"); + BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), "Bloom filter doesn't contain just-inserted object (3)!"); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; @@ -114,16 +114,14 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) BOOST_AUTO_TEST_CASE(bloom_match) { // Random real transaction (b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b) - CTransaction tx; CDataStream stream(ParseHex("01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e88607d722c190000000008b4830450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a0141046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c339ffffffff021bff3d11000000001976a91404943fdd508053c75000106d3bc6e2754dbcff1988ac2f15de00000000001976a914a266436d2965547608b9e15d9032a7b9d64fa43188ac00000000"), SER_DISK, CLIENT_VERSION); - stream >> tx; + CTransaction tx(deserialize, stream); // and one which spends it (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; vector<unsigned char> vch(ch, ch + sizeof(ch) -1); CDataStream spendStream(vch, SER_DISK, CLIENT_VERSION); - CTransaction spendingTx; - spendStream >> spendingTx; + CTransaction spendingTx(deserialize, spendStream); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); filter.insert(uint256S("0xb4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b")); diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp new file mode 100644 index 0000000000..7b3134d327 --- /dev/null +++ b/src/test/bswap_tests.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "compat/byteswap.h" +#include "test/test_bitcoin.h" + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(bswap_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(bswap_tests) +{ + // Sibling in bitcoin/src/qt/test/compattests.cpp + uint16_t u1 = 0x1234; + uint32_t u2 = 0x56789abc; + uint64_t u3 = 0xdef0123456789abc; + uint16_t e1 = 0x3412; + uint32_t e2 = 0xbc9a7856; + uint64_t e3 = 0xbc9a78563412f0de; + BOOST_CHECK(bswap_16(u1) == e1); + BOOST_CHECK(bswap_32(u2) == e2); + BOOST_CHECK(bswap_64(u3) == e3); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 82de302053..79dea00e46 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -8,7 +8,7 @@ #include "utilstrencodings.h" #include "test/test_bitcoin.h" #include "test/test_random.h" -#include "main.h" +#include "validation.h" #include "consensus/validation.h" #include <vector> diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp new file mode 100644 index 0000000000..1bc50d5ea9 --- /dev/null +++ b/src/test/cuckoocache_tests.cpp @@ -0,0 +1,394 @@ +// Copyright (c) 2012-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <boost/test/unit_test.hpp> +#include "cuckoocache.h" +#include "test/test_bitcoin.h" +#include "random.h" +#include <thread> +#include <boost/thread.hpp> + + +/** Test Suite for CuckooCache + * + * 1) All tests should have a deterministic result (using insecure rand + * with deterministic seeds) + * 2) Some test methods are templated to allow for easier testing + * against new versions / comparing + * 3) Results should be treated as a regression test, ie, did the behavior + * change significantly from what was expected. This can be OK, depending on + * the nature of the change, but requires updating the tests to reflect the new + * expected behavior. For example improving the hit rate may cause some tests + * using BOOST_CHECK_CLOSE to fail. + * + */ +FastRandomContext insecure_rand(true); + +BOOST_AUTO_TEST_SUITE(cuckoocache_tests); + + +/** insecure_GetRandHash fills in a uint256 from insecure_rand + */ +void insecure_GetRandHash(uint256& t) +{ + uint32_t* ptr = (uint32_t*)t.begin(); + for (uint8_t j = 0; j < 8; ++j) + *(ptr++) = insecure_rand.rand32(); +} + +/** Definition copied from /src/script/sigcache.cpp + */ +class uint256Hasher +{ +public: + template <uint8_t hash_select> + uint32_t operator()(const uint256& key) const + { + static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); + uint32_t u; + std::memcpy(&u, key.begin() + 4 * hash_select, 4); + return u; + } +}; + + +/* Test that no values not inserted into the cache are read out of it. + * + * There are no repeats in the first 200000 insecure_GetRandHash calls + */ +BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) +{ + insecure_rand = FastRandomContext(true); + CuckooCache::cache<uint256, uint256Hasher> cc{}; + cc.setup_bytes(32 << 20); + uint256 v; + for (int x = 0; x < 100000; ++x) { + insecure_GetRandHash(v); + cc.insert(v); + } + for (int x = 0; x < 100000; ++x) { + insecure_GetRandHash(v); + BOOST_CHECK(!cc.contains(v, false)); + } +}; + +/** This helper returns the hit rate when megabytes*load worth of entries are + * inserted into a megabytes sized cache + */ +template <typename Cache> +double test_cache(size_t megabytes, double load) +{ + insecure_rand = FastRandomContext(true); + std::vector<uint256> hashes; + Cache set{}; + size_t bytes = megabytes * (1 << 20); + set.setup_bytes(bytes); + uint32_t n_insert = static_cast<uint32_t>(load * (bytes / sizeof(uint256))); + hashes.resize(n_insert); + for (uint32_t i = 0; i < n_insert; ++i) { + uint32_t* ptr = (uint32_t*)hashes[i].begin(); + for (uint8_t j = 0; j < 8; ++j) + *(ptr++) = insecure_rand.rand32(); + } + /** We make a copy of the hashes because future optimizations of the + * cuckoocache may overwrite the inserted element, so the test is + * "future proofed". + */ + std::vector<uint256> hashes_insert_copy = hashes; + /** Do the insert */ + for (uint256& h : hashes_insert_copy) + set.insert(h); + /** Count the hits */ + uint32_t count = 0; + for (uint256& h : hashes) + count += set.contains(h, false); + double hit_rate = ((double)count) / ((double)n_insert); + return hit_rate; +} + +/** The normalized hit rate for a given load. + * + * The semantics are a little confusing, so please see the below + * explanation. + * + * Examples: + * + * 1) at load 0.5, we expect a perfect hit rate, so we multiply by + * 1.0 + * 2) at load 2.0, we expect to see half the entries, so a perfect hit rate + * would be 0.5. Therefore, if we see a hit rate of 0.4, 0.4*2.0 = 0.8 is the + * normalized hit rate. + * + * This is basically the right semantics, but has a bit of a glitch depending on + * how you measure around load 1.0 as after load 1.0 your normalized hit rate + * becomes effectively perfect, ignoring freshness. + */ +double normalize_hit_rate(double hits, double load) +{ + return hits * std::max(load, 1.0); +} + +/** Check the hit rate on loads ranging from 0.1 to 2.0 */ +BOOST_AUTO_TEST_CASE(cuckoocache_hit_rate_ok) +{ + /** Arbitrarily selected Hit Rate threshold that happens to work for this test + * as a lower bound on performance. + */ + double HitRateThresh = 0.98; + size_t megabytes = 32; + for (double load = 0.1; load < 2; load *= 2) { + double hits = test_cache<CuckooCache::cache<uint256, uint256Hasher>>(megabytes, load); + BOOST_CHECK(normalize_hit_rate(hits, load) > HitRateThresh); + } +} + + +/** This helper checks that erased elements are preferentially inserted onto and + * that the hit rate of "fresher" keys is reasonable*/ +template <typename Cache> +void test_cache_erase(size_t megabytes) +{ + double load = 1; + insecure_rand = FastRandomContext(true); + std::vector<uint256> hashes; + Cache set{}; + size_t bytes = megabytes * (1 << 20); + set.setup_bytes(bytes); + uint32_t n_insert = static_cast<uint32_t>(load * (bytes / sizeof(uint256))); + hashes.resize(n_insert); + for (uint32_t i = 0; i < n_insert; ++i) { + uint32_t* ptr = (uint32_t*)hashes[i].begin(); + for (uint8_t j = 0; j < 8; ++j) + *(ptr++) = insecure_rand.rand32(); + } + /** We make a copy of the hashes because future optimizations of the + * cuckoocache may overwrite the inserted element, so the test is + * "future proofed". + */ + std::vector<uint256> hashes_insert_copy = hashes; + + /** Insert the first half */ + for (uint32_t i = 0; i < (n_insert / 2); ++i) + set.insert(hashes_insert_copy[i]); + /** Erase the first quarter */ + for (uint32_t i = 0; i < (n_insert / 4); ++i) + set.contains(hashes[i], true); + /** Insert the second half */ + for (uint32_t i = (n_insert / 2); i < n_insert; ++i) + set.insert(hashes_insert_copy[i]); + + /** elements that we marked erased but that are still there */ + size_t count_erased_but_contained = 0; + /** elements that we did not erase but are older */ + size_t count_stale = 0; + /** elements that were most recently inserted */ + size_t count_fresh = 0; + + for (uint32_t i = 0; i < (n_insert / 4); ++i) + count_erased_but_contained += set.contains(hashes[i], false); + for (uint32_t i = (n_insert / 4); i < (n_insert / 2); ++i) + count_stale += set.contains(hashes[i], false); + for (uint32_t i = (n_insert / 2); i < n_insert; ++i) + count_fresh += set.contains(hashes[i], false); + + double hit_rate_erased_but_contained = double(count_erased_but_contained) / (double(n_insert) / 4.0); + double hit_rate_stale = double(count_stale) / (double(n_insert) / 4.0); + double hit_rate_fresh = double(count_fresh) / (double(n_insert) / 2.0); + + // Check that our hit_rate_fresh is perfect + BOOST_CHECK_EQUAL(hit_rate_fresh, 1.0); + // Check that we have a more than 2x better hit rate on stale elements than + // erased elements. + BOOST_CHECK(hit_rate_stale > 2 * hit_rate_erased_but_contained); +} + +BOOST_AUTO_TEST_CASE(cuckoocache_erase_ok) +{ + size_t megabytes = 32; + test_cache_erase<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); +} + +template <typename Cache> +void test_cache_erase_parallel(size_t megabytes) +{ + double load = 1; + insecure_rand = FastRandomContext(true); + std::vector<uint256> hashes; + Cache set{}; + size_t bytes = megabytes * (1 << 20); + set.setup_bytes(bytes); + uint32_t n_insert = static_cast<uint32_t>(load * (bytes / sizeof(uint256))); + hashes.resize(n_insert); + for (uint32_t i = 0; i < n_insert; ++i) { + uint32_t* ptr = (uint32_t*)hashes[i].begin(); + for (uint8_t j = 0; j < 8; ++j) + *(ptr++) = insecure_rand.rand32(); + } + /** We make a copy of the hashes because future optimizations of the + * cuckoocache may overwrite the inserted element, so the test is + * "future proofed". + */ + std::vector<uint256> hashes_insert_copy = hashes; + boost::shared_mutex mtx; + + { + /** Grab lock to make sure we release inserts */ + boost::unique_lock<boost::shared_mutex> l(mtx); + /** Insert the first half */ + for (uint32_t i = 0; i < (n_insert / 2); ++i) + set.insert(hashes_insert_copy[i]); + } + + /** Spin up 3 threads to run contains with erase. + */ + std::vector<std::thread> threads; + /** Erase the first quarter */ + for (uint32_t x = 0; x < 3; ++x) + /** Each thread is emplaced with x copy-by-value + */ + threads.emplace_back([&, x] { + boost::shared_lock<boost::shared_mutex> l(mtx); + size_t ntodo = (n_insert/4)/3; + size_t start = ntodo*x; + size_t end = ntodo*(x+1); + for (uint32_t i = start; i < end; ++i) + set.contains(hashes[i], true); + }); + + /** Wait for all threads to finish + */ + for (std::thread& t : threads) + t.join(); + /** Grab lock to make sure we observe erases */ + boost::unique_lock<boost::shared_mutex> l(mtx); + /** Insert the second half */ + for (uint32_t i = (n_insert / 2); i < n_insert; ++i) + set.insert(hashes_insert_copy[i]); + + /** elements that we marked erased but that are still there */ + size_t count_erased_but_contained = 0; + /** elements that we did not erase but are older */ + size_t count_stale = 0; + /** elements that were most recently inserted */ + size_t count_fresh = 0; + + for (uint32_t i = 0; i < (n_insert / 4); ++i) + count_erased_but_contained += set.contains(hashes[i], false); + for (uint32_t i = (n_insert / 4); i < (n_insert / 2); ++i) + count_stale += set.contains(hashes[i], false); + for (uint32_t i = (n_insert / 2); i < n_insert; ++i) + count_fresh += set.contains(hashes[i], false); + + double hit_rate_erased_but_contained = double(count_erased_but_contained) / (double(n_insert) / 4.0); + double hit_rate_stale = double(count_stale) / (double(n_insert) / 4.0); + double hit_rate_fresh = double(count_fresh) / (double(n_insert) / 2.0); + + // Check that our hit_rate_fresh is perfect + BOOST_CHECK_EQUAL(hit_rate_fresh, 1.0); + // Check that we have a more than 2x better hit rate on stale elements than + // erased elements. + BOOST_CHECK(hit_rate_stale > 2 * hit_rate_erased_but_contained); +} +BOOST_AUTO_TEST_CASE(cuckoocache_erase_parallel_ok) +{ + size_t megabytes = 32; + test_cache_erase_parallel<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); +} + + +template <typename Cache> +void test_cache_generations() +{ + // This test checks that for a simulation of network activity, the fresh hit + // rate is never below 99%, and the number of times that it is worse than + // 99.9% are less than 1% of the time. + double min_hit_rate = 0.99; + double tight_hit_rate = 0.999; + double max_rate_less_than_tight_hit_rate = 0.01; + // A cache that meets this specification is therefore shown to have a hit + // rate of at least tight_hit_rate * (1 - max_rate_less_than_tight_hit_rate) + + // min_hit_rate*max_rate_less_than_tight_hit_rate = 0.999*99%+0.99*1% == 99.89% + // hit rate with low variance. + + // We use deterministic values, but this test has also passed on many + // iterations with non-deterministic values, so it isn't "overfit" to the + // specific entropy in FastRandomContext(true) and implementation of the + // cache. + insecure_rand = FastRandomContext(true); + + // block_activity models a chunk of network activity. n_insert elements are + // adde to the cache. The first and last n/4 are stored for removal later + // and the middle n/2 are not stored. This models a network which uses half + // the signatures of recently (since the last block) added transactions + // immediately and never uses the other half. + struct block_activity { + std::vector<uint256> reads; + block_activity(uint32_t n_insert, Cache& c) : reads() + { + std::vector<uint256> inserts; + inserts.resize(n_insert); + reads.reserve(n_insert / 2); + for (uint32_t i = 0; i < n_insert; ++i) { + uint32_t* ptr = (uint32_t*)inserts[i].begin(); + for (uint8_t j = 0; j < 8; ++j) + *(ptr++) = insecure_rand.rand32(); + } + for (uint32_t i = 0; i < n_insert / 4; ++i) + reads.push_back(inserts[i]); + for (uint32_t i = n_insert - (n_insert / 4); i < n_insert; ++i) + reads.push_back(inserts[i]); + for (auto h : inserts) + c.insert(h); + } + }; + + const uint32_t BLOCK_SIZE = 10000; + // We expect window size 60 to perform reasonably given that each epoch + // stores 45% of the cache size (~472k). + const uint32_t WINDOW_SIZE = 60; + const uint32_t POP_AMOUNT = (BLOCK_SIZE / WINDOW_SIZE) / 2; + const double load = 10; + const size_t megabytes = 32; + const size_t bytes = megabytes * (1 << 20); + const uint32_t n_insert = static_cast<uint32_t>(load * (bytes / sizeof(uint256))); + + std::vector<block_activity> hashes; + Cache set{}; + set.setup_bytes(bytes); + hashes.reserve(n_insert / BLOCK_SIZE); + std::deque<block_activity> last_few; + uint32_t out_of_tight_tolerance = 0; + uint32_t total = n_insert / BLOCK_SIZE; + // we use the deque last_few to model a sliding window of blocks. at each + // step, each of the last WINDOW_SIZE block_activities checks the cache for + // POP_AMOUNT of the hashes that they inserted, and marks these erased. + for (uint32_t i = 0; i < total; ++i) { + if (last_few.size() == WINDOW_SIZE) + last_few.pop_front(); + last_few.emplace_back(BLOCK_SIZE, set); + uint32_t count = 0; + for (auto& act : last_few) + for (uint32_t k = 0; k < POP_AMOUNT; ++k) { + count += set.contains(act.reads.back(), true); + act.reads.pop_back(); + } + // We use last_few.size() rather than WINDOW_SIZE for the correct + // behavior on the first WINDOW_SIZE iterations where the deque is not + // full yet. + double hit = (double(count)) / (last_few.size() * POP_AMOUNT); + // Loose Check that hit rate is above min_hit_rate + BOOST_CHECK(hit > min_hit_rate); + // Tighter check, count number of times we are less than tight_hit_rate + // (and implicityly, greater than min_hit_rate) + out_of_tight_tolerance += hit < tight_hit_rate; + } + // Check that being out of tolerance happens less than + // max_rate_less_than_tight_hit_rate of the time + BOOST_CHECK(double(out_of_tight_tolerance) / double(total) < max_rate_less_than_tight_hit_rate); +} +BOOST_AUTO_TEST_CASE(cuckoocache_generations) +{ + test_cache_generations<CuckooCache::cache<uint256, uint256Hasher>>(); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/test/data/bitcoin-util-test.json b/src/test/data/bitcoin-util-test.json index 98be75919b..ab1a1385a3 100644 --- a/src/test/data/bitcoin-util-test.json +++ b/src/test/data/bitcoin-util-test.json @@ -1,24 +1,24 @@ [ { "exec": "./bitcoin-tx", - "args": ["-create"], - "output_cmp": "blanktx.hex", - "description": "Creates a blank transaction" + "args": ["-create", "nversion=1"], + "output_cmp": "blanktxv1.hex", + "description": "Creates a blank v1 transaction" }, { "exec": "./bitcoin-tx", "args": ["-json","-create"], - "output_cmp": "blanktx.json", + "output_cmp": "blanktxv2.json", "description": "Creates a blank transaction (output in json)" }, { "exec": "./bitcoin-tx", "args": ["-"], - "input": "blanktx.hex", - "output_cmp": "blanktx.hex", + "input": "blanktxv2.hex", + "output_cmp": "blanktxv2.hex", "description": "Creates a blank transaction when nothing is piped into bitcoin-tx" }, { "exec": "./bitcoin-tx", "args": ["-json","-"], - "input": "blanktx.hex", - "output_cmp": "blanktx.json", + "input": "blanktxv2.hex", + "output_cmp": "blanktxv2.json", "description": "Creates a blank transaction when nothing is piped into bitcoin-tx (output in json)" }, { "exec": "./bitcoin-tx", @@ -103,37 +103,48 @@ "description": "Creates a new transaction with a single empty output script (output in json)" }, { "exec": "./bitcoin-tx", - "args": ["01000000000100000000000000000000000000"], + "args": ["02000000000100000000000000000000000000"], "output_cmp": "txcreate2.hex", "description": "Parses a transation with no inputs and a single output script" }, { "exec": "./bitcoin-tx", - "args": ["-json", "01000000000100000000000000000000000000"], + "args": ["-json", "02000000000100000000000000000000000000"], "output_cmp": "txcreate2.json", "description": "Parses a transation with no inputs and a single output script (output in json)" }, { "exec": "./bitcoin-tx", "args": - ["-create", + ["-create", "nversion=1", "in=4d49a71ec9da436f71ec4ee231d04f292a29cd316f598bb7068feccabdc59485:0", "set=privatekeys:[\"5HpHagT65TZzG1PH3CSu63k8DbpvD8s5ip4nEB3kEsreAnchuDf\"]", "set=prevtxs:[{\"txid\":\"4d49a71ec9da436f71ec4ee231d04f292a29cd316f598bb7068feccabdc59485\",\"vout\":0,\"scriptPubKey\":\"76a91491b24bf9f5288532960ac687abb035127b1d28a588ac\"}]", "sign=ALL", "outaddr=0.001:193P6LtvS4nCnkDvM9uXn1gsSRqh4aDAz7"], - "output_cmp": "txcreatesign.hex", - "description": "Creates a new transaction with a single input and a single output, and then signs the transaction" + "output_cmp": "txcreatesignv1.hex", + "description": "Creates a new v1 transaction with a single input and a single output, and then signs the transaction" }, { "exec": "./bitcoin-tx", "args": ["-json", - "-create", + "-create", "nversion=1", "in=4d49a71ec9da436f71ec4ee231d04f292a29cd316f598bb7068feccabdc59485:0", "set=privatekeys:[\"5HpHagT65TZzG1PH3CSu63k8DbpvD8s5ip4nEB3kEsreAnchuDf\"]", "set=prevtxs:[{\"txid\":\"4d49a71ec9da436f71ec4ee231d04f292a29cd316f598bb7068feccabdc59485\",\"vout\":0,\"scriptPubKey\":\"76a91491b24bf9f5288532960ac687abb035127b1d28a588ac\"}]", "sign=ALL", "outaddr=0.001:193P6LtvS4nCnkDvM9uXn1gsSRqh4aDAz7"], - "output_cmp": "txcreatesign.json", - "description": "Creates a new transaction with a single input and a single output, and then signs the transaction (output in json)" + "output_cmp": "txcreatesignv1.json", + "description": "Creates a new v1 transaction with a single input and a single output, and then signs the transaction (output in json)" + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", + "in=4d49a71ec9da436f71ec4ee231d04f292a29cd316f598bb7068feccabdc59485:0", + "set=privatekeys:[\"5HpHagT65TZzG1PH3CSu63k8DbpvD8s5ip4nEB3kEsreAnchuDf\"]", + "set=prevtxs:[{\"txid\":\"4d49a71ec9da436f71ec4ee231d04f292a29cd316f598bb7068feccabdc59485\",\"vout\":0,\"scriptPubKey\":\"76a91491b24bf9f5288532960ac687abb035127b1d28a588ac\"}]", + "sign=ALL", + "outaddr=0.001:193P6LtvS4nCnkDvM9uXn1gsSRqh4aDAz7"], + "output_cmp": "txcreatesignv2.hex", + "description": "Creates a new transaction with a single input and a single output, and then signs the transaction" }, { "exec": "./bitcoin-tx", "args": @@ -163,12 +174,12 @@ { "exec": "./bitcoin-tx", "args": ["-json", - "-create", + "-create", "nversion=1", "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", "outaddr=0.18:13tuJJDR2RgArmgfv6JScSdreahzgc4T6o", "outdata=4:54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e"], "output_cmp": "txcreatedata1.json", - "description": "Creates a new transaction with one input, one address output and one data output (output in json)" + "description": "Creates a new v1 transaction with one input, one address output and one data output (output in json)" }, { "exec": "./bitcoin-tx", "args": diff --git a/src/test/data/blanktx.hex b/src/test/data/blanktxv1.hex index 36b6f00fb6..36b6f00fb6 100644 --- a/src/test/data/blanktx.hex +++ b/src/test/data/blanktxv1.hex diff --git a/src/test/data/blanktxv2.hex b/src/test/data/blanktxv2.hex new file mode 100644 index 0000000000..22d830eda1 --- /dev/null +++ b/src/test/data/blanktxv2.hex @@ -0,0 +1 @@ +02000000000000000000 diff --git a/src/test/data/blanktxv2.json b/src/test/data/blanktxv2.json new file mode 100644 index 0000000000..266919f445 --- /dev/null +++ b/src/test/data/blanktxv2.json @@ -0,0 +1,11 @@ +{ + "txid": "4ebd325a4b394cff8c57e8317ccf5a8d0e2bdf1b8526f8aad6c8e43d8240621a", + "hash": "4ebd325a4b394cff8c57e8317ccf5a8d0e2bdf1b8526f8aad6c8e43d8240621a", + "version": 2, + "locktime": 0, + "vin": [ + ], + "vout": [ + ], + "hex": "02000000000000000000" +} diff --git a/src/test/data/tx_valid.json b/src/test/data/tx_valid.json index 27acb2f16a..a3f47fcee2 100644 --- a/src/test/data/tx_valid.json +++ b/src/test/data/tx_valid.json @@ -478,7 +478,7 @@ ["1b2a9a426ba603ba357ce7773cb5805cb9c7c2b386d100d1fc9263513188e680", 0, "0x00 0x20 0xd9bbfbe56af7c4b7f960a70d7ea107156913d9e5a26b0a71429df5e097ca6537", 16777215]], "01000000000102e9b542c5176808107ff1df906f46bb1f2583b16112b95ee5380665ba7fcfc0010000000000ffffffff80e68831516392fcd100d186b3c2c7b95c80b53c77e77c35ba03a66b429a2a1b0000000000ffffffff0280969800000000001976a914de4b231626ef508c9a74a8517e6783c0546d6b2888ac80969800000000001976a9146648a8cd4531e1ec47f35916de8e259237294d1e88ac02483045022100f6a10b8604e6dc910194b79ccfc93e1bc0ec7c03453caaa8987f7d6c3413566002206216229ede9b4d6ec2d325be245c5b508ff0339bf1794078e20bfe0babc7ffe683270063ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac024730440220032521802a76ad7bf74d0e2c218b72cf0cbc867066e2e53db905ba37f130397e02207709e2188ed7f08f4c952d9d13986da504502b8c3be59617e043552f506c46ff83275163ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac00000000", "P2SH,WITNESS"], -["BIP143 example: Same as the previous example with input-output paris swapped"], +["BIP143 example: Same as the previous example with input-output pairs swapped"], [[["1b2a9a426ba603ba357ce7773cb5805cb9c7c2b386d100d1fc9263513188e680", 0, "0x00 0x20 0xd9bbfbe56af7c4b7f960a70d7ea107156913d9e5a26b0a71429df5e097ca6537", 16777215], ["01c0cf7fba650638e55eb91261b183251fbb466f90dff17f10086817c542b5e9", 0, "0x00 0x20 0xba468eea561b26301e4cf69fa34bde4ad60c81e70f059f045ca9a79931004a4d", 16777215]], "0100000000010280e68831516392fcd100d186b3c2c7b95c80b53c77e77c35ba03a66b429a2a1b0000000000ffffffffe9b542c5176808107ff1df906f46bb1f2583b16112b95ee5380665ba7fcfc0010000000000ffffffff0280969800000000001976a9146648a8cd4531e1ec47f35916de8e259237294d1e88ac80969800000000001976a914de4b231626ef508c9a74a8517e6783c0546d6b2888ac024730440220032521802a76ad7bf74d0e2c218b72cf0cbc867066e2e53db905ba37f130397e02207709e2188ed7f08f4c952d9d13986da504502b8c3be59617e043552f506c46ff83275163ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac02483045022100f6a10b8604e6dc910194b79ccfc93e1bc0ec7c03453caaa8987f7d6c3413566002206216229ede9b4d6ec2d325be245c5b508ff0339bf1794078e20bfe0babc7ffe683270063ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac00000000", "P2SH,WITNESS"], diff --git a/src/test/data/txcreate1.hex b/src/test/data/txcreate1.hex index e2981a51c9..9ec6ee3531 100644 --- a/src/test/data/txcreate1.hex +++ b/src/test/data/txcreate1.hex @@ -1 +1 @@ -01000000031f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff7cca453133921c50d5025878f7f738d1df891fd359763331935784cf6b9c82bf1200000000fffffffffccd319e04a996c96cfc0bf4c07539aa90bd0b1a700ef72fae535d6504f9a6220100000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0084d717000000001976a914f2d4db28cad6502226ee484ae24505c2885cb12d88ac00000000 +02000000031f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff7cca453133921c50d5025878f7f738d1df891fd359763331935784cf6b9c82bf1200000000fffffffffccd319e04a996c96cfc0bf4c07539aa90bd0b1a700ef72fae535d6504f9a6220100000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0084d717000000001976a914f2d4db28cad6502226ee484ae24505c2885cb12d88ac00000000 diff --git a/src/test/data/txcreate1.json b/src/test/data/txcreate1.json index 567e8026a3..f83e036f33 100644 --- a/src/test/data/txcreate1.json +++ b/src/test/data/txcreate1.json @@ -1,7 +1,7 @@ { - "txid": "f70f0d6c71416ed538e37549f430ab3665fee2437a42f10238c1bd490e782231", - "hash": "f70f0d6c71416ed538e37549f430ab3665fee2437a42f10238c1bd490e782231", - "version": 1, + "txid": "fe7d174f42dce0cffa7a527e9bc8368956057619ec817648f6138b98f2533e8f", + "hash": "fe7d174f42dce0cffa7a527e9bc8368956057619ec817648f6138b98f2533e8f", + "version": 2, "locktime": 0, "vin": [ { @@ -60,5 +60,5 @@ } } ], - "hex": "01000000031f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff7cca453133921c50d5025878f7f738d1df891fd359763331935784cf6b9c82bf1200000000fffffffffccd319e04a996c96cfc0bf4c07539aa90bd0b1a700ef72fae535d6504f9a6220100000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0084d717000000001976a914f2d4db28cad6502226ee484ae24505c2885cb12d88ac00000000" + "hex": "02000000031f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff7cca453133921c50d5025878f7f738d1df891fd359763331935784cf6b9c82bf1200000000fffffffffccd319e04a996c96cfc0bf4c07539aa90bd0b1a700ef72fae535d6504f9a6220100000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0084d717000000001976a914f2d4db28cad6502226ee484ae24505c2885cb12d88ac00000000" } diff --git a/src/test/data/txcreate2.hex b/src/test/data/txcreate2.hex index 5243c2d02e..38bb7b1046 100644 --- a/src/test/data/txcreate2.hex +++ b/src/test/data/txcreate2.hex @@ -1 +1 @@ -01000000000100000000000000000000000000 +02000000000100000000000000000000000000 diff --git a/src/test/data/txcreate2.json b/src/test/data/txcreate2.json index a70c1d302a..fb5e177db7 100644 --- a/src/test/data/txcreate2.json +++ b/src/test/data/txcreate2.json @@ -1,7 +1,7 @@ { - "txid": "cf90229625e9eb10f6be8156bf6aa5ec2eca19a42b1e05c11f3029b560a32e13", - "hash": "cf90229625e9eb10f6be8156bf6aa5ec2eca19a42b1e05c11f3029b560a32e13", - "version": 1, + "txid": "0481afb29931341d0d7861d8a2f6f26456fa042abf54a23e96440ed7946e0715", + "hash": "0481afb29931341d0d7861d8a2f6f26456fa042abf54a23e96440ed7946e0715", + "version": 2, "locktime": 0, "vin": [ ], @@ -16,5 +16,5 @@ } } ], - "hex": "01000000000100000000000000000000000000" + "hex": "02000000000100000000000000000000000000" } diff --git a/src/test/data/txcreatedata1.hex b/src/test/data/txcreatedata1.hex index eccc7604e6..cefd1a05a6 100644 --- a/src/test/data/txcreatedata1.hex +++ b/src/test/data/txcreatedata1.hex @@ -1 +1 @@ -01000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0084d71700000000526a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e00000000 +02000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0084d71700000000526a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e00000000 diff --git a/src/test/data/txcreatedata2.hex b/src/test/data/txcreatedata2.hex index 3c7644c297..d69cf58ba1 100644 --- a/src/test/data/txcreatedata2.hex +++ b/src/test/data/txcreatedata2.hex @@ -1 +1 @@ -01000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0000000000000000526a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e00000000 +02000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0000000000000000526a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e00000000 diff --git a/src/test/data/txcreatedata2.json b/src/test/data/txcreatedata2.json index 56dfe4a1b0..3c6da40f90 100644 --- a/src/test/data/txcreatedata2.json +++ b/src/test/data/txcreatedata2.json @@ -1,7 +1,7 @@ { - "txid": "4ed17118f5e932ba8c75c461787d171bc02a016d8557cb5bcf34cd416c27bb8b", - "hash": "4ed17118f5e932ba8c75c461787d171bc02a016d8557cb5bcf34cd416c27bb8b", - "version": 1, + "txid": "c14b007fa3a6c1e7765919c1d14c1cfc2b8642c3a5d3be4b1fa8c4ccfec98bb0", + "hash": "c14b007fa3a6c1e7765919c1d14c1cfc2b8642c3a5d3be4b1fa8c4ccfec98bb0", + "version": 2, "locktime": 0, "vin": [ { @@ -38,5 +38,5 @@ } } ], - "hex": "01000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0000000000000000526a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e00000000" + "hex": "02000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff0280a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac0000000000000000526a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e00000000" } diff --git a/src/test/data/txcreatedata_seq0.hex b/src/test/data/txcreatedata_seq0.hex index db02b5e4a4..54b89d2381 100644 --- a/src/test/data/txcreatedata_seq0.hex +++ b/src/test/data/txcreatedata_seq0.hex @@ -1 +1 @@ -01000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff0180a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac00000000 +02000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff0180a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac00000000 diff --git a/src/test/data/txcreatedata_seq0.json b/src/test/data/txcreatedata_seq0.json index 9bc0ed4593..d272a4c447 100644 --- a/src/test/data/txcreatedata_seq0.json +++ b/src/test/data/txcreatedata_seq0.json @@ -1,7 +1,7 @@ { - "txid": "71603ccb1cd76d73d76eb6cfd5f0b9df6d65d90d76860ee52cb461c4be7032e8", - "hash": "71603ccb1cd76d73d76eb6cfd5f0b9df6d65d90d76860ee52cb461c4be7032e8", - "version": 1, + "txid": "8df6ed527472542dd5e137c242a7c5a9f337ac34f7b257ae4af886aeaebb51b0", + "hash": "8df6ed527472542dd5e137c242a7c5a9f337ac34f7b257ae4af886aeaebb51b0", + "version": 2, "locktime": 0, "vin": [ { @@ -29,5 +29,5 @@ } } ], - "hex": "01000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff0180a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac00000000" + "hex": "02000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff0180a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac00000000" } diff --git a/src/test/data/txcreatesign.hex b/src/test/data/txcreatesignv1.hex index a46fcc88cb..a46fcc88cb 100644 --- a/src/test/data/txcreatesign.hex +++ b/src/test/data/txcreatesignv1.hex diff --git a/src/test/data/txcreatesignv1.json b/src/test/data/txcreatesignv1.json new file mode 100644 index 0000000000..ff39e71b40 --- /dev/null +++ b/src/test/data/txcreatesignv1.json @@ -0,0 +1,33 @@ +{ + "txid": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af", + "hash": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af", + "version": 1, + "locktime": 0, + "vin": [ + { + "txid": "4d49a71ec9da436f71ec4ee231d04f292a29cd316f598bb7068feccabdc59485", + "vout": 0, + "scriptSig": { + "asm": "304502210096a75056c9e2cc62b7214777b3d2a592cfda7092520126d4ebfcd6d590c99bd8022051bb746359cf98c0603f3004477eac68701132380db8facba19c89dc5ab5c5e2[ALL] 0479be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8", + "hex": "48304502210096a75056c9e2cc62b7214777b3d2a592cfda7092520126d4ebfcd6d590c99bd8022051bb746359cf98c0603f3004477eac68701132380db8facba19c89dc5ab5c5e201410479be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8" + }, + "sequence": 4294967295 + } + ], + "vout": [ + { + "value": 0.001, + "n": 0, + "scriptPubKey": { + "asm": "OP_DUP OP_HASH160 5834479edbbe0539b31ffd3a8f8ebadc2165ed01 OP_EQUALVERIFY OP_CHECKSIG", + "hex": "76a9145834479edbbe0539b31ffd3a8f8ebadc2165ed0188ac", + "reqSigs": 1, + "type": "pubkeyhash", + "addresses": [ + "193P6LtvS4nCnkDvM9uXn1gsSRqh4aDAz7" + ] + } + } + ], + "hex": "01000000018594c5bdcaec8f06b78b596f31cd292a294fd031e24eec716f43dac91ea7494d000000008b48304502210096a75056c9e2cc62b7214777b3d2a592cfda7092520126d4ebfcd6d590c99bd8022051bb746359cf98c0603f3004477eac68701132380db8facba19c89dc5ab5c5e201410479be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8ffffffff01a0860100000000001976a9145834479edbbe0539b31ffd3a8f8ebadc2165ed0188ac00000000" +} diff --git a/src/test/data/txcreatesignv2.hex b/src/test/data/txcreatesignv2.hex new file mode 100644 index 0000000000..ee425cd98c --- /dev/null +++ b/src/test/data/txcreatesignv2.hex @@ -0,0 +1 @@ +02000000018594c5bdcaec8f06b78b596f31cd292a294fd031e24eec716f43dac91ea7494d000000008a473044022079c7aa014177a2e973caf6df7c7b8f15399083b91eba370ea1e19c4caed9181e02205f8f8763505ce8e6cbdd2cd28fab3fd407a75003e7d0dc04e6bebb0a3c89e7cb01410479be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8ffffffff01a0860100000000001976a9145834479edbbe0539b31ffd3a8f8ebadc2165ed0188ac00000000 diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index fa9624f13d..8be9c580c4 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -124,7 +124,11 @@ BOOST_AUTO_TEST_CASE(siphash) } CHashWriter ss(SER_DISK, CLIENT_VERSION); - ss << CTransaction(); + CMutableTransaction tx; + // Note these tests were originally written with tx.nVersion=1 + // and the test would be affected by default tx version bumps if not fixed. + tx.nVersion = 1; + ss << tx; BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); } diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index dbfbdd934f..697fbf792c 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -3,7 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "chainparams.h" -#include "main.h" +#include "validation.h" +#include "net.h" #include "test/test_bitcoin.h" diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 1faf8b6aeb..84871600b2 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -55,17 +55,17 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) CTxMemPool testPool(CFeeRate(0)); - std::vector<CTransactionRef> removed; // Nothing in pool, remove should do nothing: - testPool.removeRecursive(txParent, &removed); - BOOST_CHECK_EQUAL(removed.size(), 0); + unsigned int poolSize = testPool.size(); + testPool.removeRecursive(txParent); + BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Just the parent: testPool.addUnchecked(txParent.GetHash(), entry.FromTx(txParent)); - testPool.removeRecursive(txParent, &removed); - BOOST_CHECK_EQUAL(removed.size(), 1); - removed.clear(); + poolSize = testPool.size(); + testPool.removeRecursive(txParent); + BOOST_CHECK_EQUAL(testPool.size(), poolSize - 1); // Parent, children, grandchildren: testPool.addUnchecked(txParent.GetHash(), entry.FromTx(txParent)); @@ -75,19 +75,21 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) testPool.addUnchecked(txGrandChild[i].GetHash(), entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: - testPool.removeRecursive(txChild[0], &removed); - BOOST_CHECK_EQUAL(removed.size(), 2); - removed.clear(); + poolSize = testPool.size(); + testPool.removeRecursive(txChild[0]); + BOOST_CHECK_EQUAL(testPool.size(), poolSize - 2); // ... make sure grandchild and child are gone: - testPool.removeRecursive(txGrandChild[0], &removed); - BOOST_CHECK_EQUAL(removed.size(), 0); - testPool.removeRecursive(txChild[0], &removed); - BOOST_CHECK_EQUAL(removed.size(), 0); + poolSize = testPool.size(); + testPool.removeRecursive(txGrandChild[0]); + BOOST_CHECK_EQUAL(testPool.size(), poolSize); + poolSize = testPool.size(); + testPool.removeRecursive(txChild[0]); + BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Remove parent, all children/grandchildren should go: - testPool.removeRecursive(txParent, &removed); - BOOST_CHECK_EQUAL(removed.size(), 5); + poolSize = testPool.size(); + testPool.removeRecursive(txParent); + BOOST_CHECK_EQUAL(testPool.size(), poolSize - 5); BOOST_CHECK_EQUAL(testPool.size(), 0); - removed.clear(); // Add children and grandchildren, but NOT the parent (simulate the parent being in a block) for (int i = 0; i < 3; i++) @@ -97,10 +99,10 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) } // Now remove the parent, as might happen if a block-re-org occurs but the parent cannot be // put into the mempool (maybe because it is non-standard): - testPool.removeRecursive(txParent, &removed); - BOOST_CHECK_EQUAL(removed.size(), 6); + poolSize = testPool.size(); + testPool.removeRecursive(txParent); + BOOST_CHECK_EQUAL(testPool.size(), poolSize - 6); BOOST_CHECK_EQUAL(testPool.size(), 0); - removed.clear(); } template<typename name> @@ -388,7 +390,12 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) pool.addUnchecked(tx6.GetHash(), entry.Fee(0LL).FromTx(tx6)); BOOST_CHECK_EQUAL(pool.size(), 6); - sortedOrder.push_back(tx6.GetHash().ToString()); + // Ties are broken by hash + if (tx3.GetHash() < tx6.GetHash()) + sortedOrder.push_back(tx6.GetHash().ToString()); + else + sortedOrder.insert(sortedOrder.end()-1,tx6.GetHash().ToString()); + CheckSort<ancestor_score>(pool, sortedOrder); CMutableTransaction tx7 = CMutableTransaction(); @@ -412,10 +419,14 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) /* after tx6 is mined, tx7 should move up in the sort */ std::vector<CTransactionRef> vtx; vtx.push_back(MakeTransactionRef(tx6)); - pool.removeForBlock(vtx, 1, NULL, false); + pool.removeForBlock(vtx, 1); sortedOrder.erase(sortedOrder.begin()+1); - sortedOrder.pop_back(); + // Ties are broken by hash + if (tx3.GetHash() < tx6.GetHash()) + sortedOrder.pop_back(); + else + sortedOrder.erase(sortedOrder.end()-2); sortedOrder.insert(sortedOrder.begin(), tx7.GetHash().ToString()); CheckSort<ancestor_score>(pool, sortedOrder); } diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 85a2e907c2..892e731a7a 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -7,7 +7,7 @@ #include "consensus/consensus.h" #include "consensus/merkle.h" #include "consensus/validation.h" -#include "main.h" +#include "validation.h" #include "miner.h" #include "pubkey.h" #include "script/standard.h" @@ -223,7 +223,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) txFirst.push_back(pblock->vtx[0]); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); pblock->nNonce = blockinfo[i].nonce; - BOOST_CHECK(ProcessNewBlock(chainparams, pblock, true, NULL, NULL)); + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); + BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, NULL)); pblock->hashPrevBlock = pblock->GetHash(); } diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 7dc8f226c9..c57feaec90 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -95,17 +95,22 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Highest feerate is 10*baseRate and gets in all blocks, // second highest feerate is 9*baseRate and gets in 9/10 blocks = 90%, // third highest feerate is 8*base rate, and gets in 8/10 blocks = 80%, - // so estimateFee(1) should return 10*baseRate. + // so estimateFee(1) would return 10*baseRate but is hardcoded to return failure // Second highest feerate has 100% chance of being included by 2 blocks, // so estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10;i++) { origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); - if (i > 1) { // Fee estimates should be monotonically decreasing + if (i > 2) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } int mult = 11-i; - BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); + if (i > 1) { + BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); + } + else { + BOOST_CHECK(origFeeEst[i-1] == CFeeRate(0).GetFeePerK()); + } } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -113,7 +118,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) while (blocknum < 250) mpool.removeForBlock(block, ++blocknum); - for (int i = 1; i < 10;i++) { + BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + for (int i = 2; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } @@ -151,7 +157,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, 265); block.clear(); - for (int i = 1; i < 10;i++) { + BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + for (int i = 2; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } @@ -172,7 +179,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); block.clear(); } - for (int i = 1; i < 10; i++) { + BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + for (int i = 2; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index 1a01593a8e..789cc70d7f 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -5,7 +5,7 @@ #include "core_io.h" #include "key.h" #include "keystore.h" -#include "main.h" +#include "validation.h" #include "policy/policy.h" #include "script/script.h" #include "script/script_error.h" diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 3169afb13a..d7f3a3a657 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -176,10 +176,10 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript int libconsensus_flags = flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL; if (libconsensus_flags == flags) { if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); } else { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(begin_ptr(scriptPubKey), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect,message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect,message); } } #endif @@ -276,7 +276,7 @@ private: //! The Witness embedded script CScript witscript; CScriptWitness scriptWitness; - CTransaction creditTx; + CTransactionRef creditTx; CMutableTransaction spendTx; bool havePush; std::vector<unsigned char> push; @@ -319,8 +319,8 @@ public: redeemscript = scriptPubKey; scriptPubKey = CScript() << OP_HASH160 << ToByteVector(CScriptID(redeemscript)) << OP_EQUAL; } - creditTx = BuildCreditingTransaction(scriptPubKey, nValue); - spendTx = BuildSpendingTransaction(CScript(), CScriptWitness(), creditTx); + creditTx = MakeTransactionRef(BuildCreditingTransaction(scriptPubKey, nValue)); + spendTx = BuildSpendingTransaction(CScript(), CScriptWitness(), *creditTx); } TestBuilder& ScriptError(ScriptError_t err) @@ -421,7 +421,7 @@ public: { TestBuilder copy = *this; // Make a copy so we can rollback the push. DoPush(); - DoTest(creditTx.vout[0].scriptPubKey, spendTx.vin[0].scriptSig, scriptWitness, flags, comment, scriptError, nValue); + DoTest(creditTx->vout[0].scriptPubKey, spendTx.vin[0].scriptSig, scriptWitness, flags, comment, scriptError, nValue); *this = copy; return *this; } @@ -447,7 +447,7 @@ public: array.push_back(wit); } array.push_back(FormatScript(spendTx.vin[0].scriptSig)); - array.push_back(FormatScript(creditTx.vout[0].scriptPubKey)); + array.push_back(FormatScript(creditTx->vout[0].scriptPubKey)); array.push_back(FormatScriptFlags(flags)); array.push_back(FormatScriptError((ScriptError_t)scriptError)); array.push_back(comment); @@ -461,7 +461,7 @@ public: const CScript& GetScriptPubKey() { - return creditTx.vout[0].scriptPubKey; + return creditTx->vout[0].scriptPubKey; } }; diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index bbadf57957..1dc86eb116 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -21,10 +21,10 @@ protected: bool boolval; std::string stringval; const char* charstrval; - CTransaction txval; + CTransactionRef txval; public: CSerializeMethodsTestSingle() = default; - CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, CTransaction txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), charstrval(charstrvalin), txval(txvalin){} + CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, CTransaction txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), charstrval(charstrvalin), txval(MakeTransactionRef(txvalin)){} ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> @@ -42,7 +42,7 @@ public: boolval == rhs.boolval && \ stringval == rhs.stringval && \ strcmp(charstrval, rhs.charstrval) == 0 && \ - txval == rhs.txval; + *txval == *rhs.txval; } }; diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 3bc8341b02..a524f5b946 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -5,7 +5,7 @@ #include "consensus/validation.h" #include "data/sighash.json.h" #include "hash.h" -#include "main.h" // For CheckTransaction +#include "validation.h" // For CheckTransaction #include "script/interpreter.h" #include "script/script.h" #include "serialize.h" @@ -184,7 +184,7 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) std::string raw_tx, raw_script, sigHashHex; int nIn, nHashType; uint256 sh; - CTransaction tx; + CTransactionRef tx; CScript scriptCode = CScript(); try { @@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) stream >> tx; CValidationState state; - BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); + BOOST_CHECK_MESSAGE(CheckTransaction(*tx, state), strTest); BOOST_CHECK(state.IsValid()); std::vector<unsigned char> raw = ParseHex(raw_script); @@ -209,7 +209,7 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) continue; } - sh = SignatureHash(scriptCode, tx, nIn, nHashType, 0, SIGVERSION_BASE); + sh = SignatureHash(scriptCode, *tx, nIn, nHashType, 0, SIGVERSION_BASE); BOOST_CHECK_MESSAGE(sh.GetHex() == sigHashHex, strTest); } } diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index e8a63ae60c..2e974cc612 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "main.h" +#include "validation.h" #include "pubkey.h" #include "key.h" #include "script/script.h" diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 6cbe314a76..2a5a78de02 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -10,8 +10,9 @@ #include "consensus/consensus.h" #include "consensus/validation.h" #include "key.h" -#include "main.h" +#include "validation.h" #include "miner.h" +#include "net_processing.h" #include "pubkey.h" #include "random.h" #include "txdb.h" @@ -19,6 +20,7 @@ #include "ui_interface.h" #include "rpc/server.h" #include "rpc/register.h" +#include "script/sigcache.h" #include "test/testutil.h" @@ -39,6 +41,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName) ECC_Start(); SetupEnvironment(); SetupNetworking(); + InitSignatureCache(); fPrintToDebugLog = false; // don't want to write to debug.log file fCheckBlockIndex = true; SelectParams(chainName); @@ -126,7 +129,8 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; - ProcessNewBlock(chainparams, &block, true, NULL, NULL); + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); + ProcessNewBlock(chainparams, shared_pblock, true, NULL); CBlock result = block; return result; diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp new file mode 100644 index 0000000000..584e6ed008 --- /dev/null +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -0,0 +1,256 @@ +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif + +#include "consensus/merkle.h" +#include "primitives/block.h" +#include "script/script.h" +#include "addrman.h" +#include "chain.h" +#include "coins.h" +#include "compressor.h" +#include "net.h" +#include "protocol.h" +#include "streams.h" +#include "undo.h" +#include "version.h" + +#include <stdint.h> +#include <unistd.h> + +#include <algorithm> +#include <vector> + +enum TEST_ID { + CBLOCK_DESERIALIZE=0, + CTRANSACTION_DESERIALIZE, + CBLOCKLOCATOR_DESERIALIZE, + CBLOCKMERKLEROOT, + CADDRMAN_DESERIALIZE, + CBLOCKHEADER_DESERIALIZE, + CBANENTRY_DESERIALIZE, + CTXUNDO_DESERIALIZE, + CBLOCKUNDO_DESERIALIZE, + CCOINS_DESERIALIZE, + CNETADDR_DESERIALIZE, + CSERVICE_DESERIALIZE, + CMESSAGEHEADER_DESERIALIZE, + CADDRESS_DESERIALIZE, + CINV_DESERIALIZE, + CBLOOMFILTER_DESERIALIZE, + CDISKBLOCKINDEX_DESERIALIZE, + CTXOUTCOMPRESSOR_DESERIALIZE, + TEST_ID_END +}; + +bool read_stdin(std::vector<char> &data) { + char buffer[1024]; + ssize_t length=0; + while((length = read(STDIN_FILENO, buffer, 1024)) > 0) { + data.insert(data.end(), buffer, buffer+length); + + if (data.size() > (1<<20)) return false; + } + return length==0; +} + +int main(int argc, char **argv) +{ + std::vector<char> buffer; + if (!read_stdin(buffer)) return 0; + + if (buffer.size() < sizeof(uint32_t)) return 0; + + uint32_t test_id = 0xffffffff; + memcpy(&test_id, &buffer[0], sizeof(uint32_t)); + buffer.erase(buffer.begin(), buffer.begin() + sizeof(uint32_t)); + + if (test_id >= TEST_ID_END) return 0; + + CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + try { + int nVersion; + ds >> nVersion; + ds.SetVersion(nVersion); + } catch (const std::ios_base::failure& e) { + return 0; + } + + switch(test_id) { + case CBLOCK_DESERIALIZE: + { + try + { + CBlock block; + ds >> block; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CTRANSACTION_DESERIALIZE: + { + try + { + CTransaction tx(deserialize, ds); + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CBLOCKLOCATOR_DESERIALIZE: + { + try + { + CBlockLocator bl; + ds >> bl; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CBLOCKMERKLEROOT: + { + try + { + CBlock block; + ds >> block; + bool mutated; + BlockMerkleRoot(block, &mutated); + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CADDRMAN_DESERIALIZE: + { + try + { + CAddrMan am; + ds >> am; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CBLOCKHEADER_DESERIALIZE: + { + try + { + CBlockHeader bh; + ds >> bh; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CBANENTRY_DESERIALIZE: + { + try + { + CBanEntry be; + ds >> be; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CTXUNDO_DESERIALIZE: + { + try + { + CTxUndo tu; + ds >> tu; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CBLOCKUNDO_DESERIALIZE: + { + try + { + CBlockUndo bu; + ds >> bu; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CCOINS_DESERIALIZE: + { + try + { + CCoins block; + ds >> block; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CNETADDR_DESERIALIZE: + { + try + { + CNetAddr na; + ds >> na; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CSERVICE_DESERIALIZE: + { + try + { + CService s; + ds >> s; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CMESSAGEHEADER_DESERIALIZE: + { + CMessageHeader::MessageStartChars pchMessageStart = {0x00, 0x00, 0x00, 0x00}; + try + { + CMessageHeader mh(pchMessageStart); + ds >> mh; + if (!mh.IsValid(pchMessageStart)) {return 0;} + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CADDRESS_DESERIALIZE: + { + try + { + CAddress a; + ds >> a; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CINV_DESERIALIZE: + { + try + { + CInv i; + ds >> i; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CBLOOMFILTER_DESERIALIZE: + { + try + { + CBloomFilter bf; + ds >> bf; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CDISKBLOCKINDEX_DESERIALIZE: + { + try + { + CDiskBlockIndex dbi; + ds >> dbi; + } catch (const std::ios_base::failure& e) {return 0;} + break; + } + case CTXOUTCOMPRESSOR_DESERIALIZE: + { + CTxOut to; + CTxOutCompressor toc(to); + try + { + ds >> toc; + } catch (const std::ios_base::failure& e) {return 0;} + + break; + } + default: + return 0; + } + return 0; +} + diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 34d9547f3d..ae5406ef7e 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -12,7 +12,7 @@ #include "core_io.h" #include "key.h" #include "keystore.h" -#include "main.h" // For CheckTransaction +#include "validation.h" // For CheckTransaction #include "policy/policy.h" #include "script/script.h" #include "script/sign.h" @@ -150,8 +150,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) string transaction = test[1].get_str(); CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION); - CTransaction tx; - stream >> tx; + CTransaction tx(deserialize, stream); CValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); @@ -236,8 +235,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) string transaction = test[1].get_str(); CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION ); - CTransaction tx; - stream >> tx; + CTransaction tx(deserialize, stream); CValidationState state; fValid = CheckTransaction(tx, state) && state.IsValid(); @@ -346,7 +344,7 @@ BOOST_AUTO_TEST_CASE(test_Get) BOOST_CHECK_EQUAL(coins.GetValueIn(t1), (50+21+22)*CENT); } -void CreateCreditAndSpend(const CKeyStore& keystore, const CScript& outscript, CTransaction& output, CMutableTransaction& input, bool success = true) +void CreateCreditAndSpend(const CKeyStore& keystore, const CScript& outscript, CTransactionRef& output, CMutableTransaction& input, bool success = true) { CMutableTransaction outputm; outputm.nVersion = 1; @@ -360,22 +358,22 @@ void CreateCreditAndSpend(const CKeyStore& keystore, const CScript& outscript, C CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); ssout << outputm; ssout >> output; - assert(output.vin.size() == 1); - assert(output.vin[0] == outputm.vin[0]); - assert(output.vout.size() == 1); - assert(output.vout[0] == outputm.vout[0]); - assert(output.wit.vtxinwit.size() == 0); + assert(output->vin.size() == 1); + assert(output->vin[0] == outputm.vin[0]); + assert(output->vout.size() == 1); + assert(output->vout[0] == outputm.vout[0]); + assert(output->wit.vtxinwit.size() == 0); CMutableTransaction inputm; inputm.nVersion = 1; inputm.vin.resize(1); - inputm.vin[0].prevout.hash = output.GetHash(); + inputm.vin[0].prevout.hash = output->GetHash(); inputm.vin[0].prevout.n = 0; inputm.wit.vtxinwit.resize(1); inputm.vout.resize(1); inputm.vout[0].nValue = 1; inputm.vout[0].scriptPubKey = CScript(); - bool ret = SignSignature(keystore, output, inputm, 0, SIGHASH_ALL); + bool ret = SignSignature(keystore, *output, inputm, 0, SIGHASH_ALL); assert(ret == success); CDataStream ssin(SER_NETWORK, PROTOCOL_VERSION); ssin << inputm; @@ -393,11 +391,11 @@ void CreateCreditAndSpend(const CKeyStore& keystore, const CScript& outscript, C } } -void CheckWithFlag(const CTransaction& output, const CMutableTransaction& input, int flags, bool success) +void CheckWithFlag(const CTransactionRef& output, const CMutableTransaction& input, int flags, bool success) { ScriptError error; CTransaction inputi(input); - bool ret = VerifyScript(inputi.vin[0].scriptSig, output.vout[0].scriptPubKey, inputi.wit.vtxinwit.size() > 0 ? &inputi.wit.vtxinwit[0].scriptWitness : NULL, flags, TransactionSignatureChecker(&inputi, 0, output.vout[0].nValue), &error); + bool ret = VerifyScript(inputi.vin[0].scriptSig, output->vout[0].scriptPubKey, inputi.wit.vtxinwit.size() > 0 ? &inputi.wit.vtxinwit[0].scriptWitness : NULL, flags, TransactionSignatureChecker(&inputi, 0, output->vout[0].nValue), &error); assert(ret == success); } @@ -466,10 +464,10 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { assert(hashSigned); } - CTransaction tx; CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); - WithOrVersion(&ssout, 0) << mtx; - WithOrVersion(&ssout, 0) >> tx; + auto vstream = WithOrVersion(&ssout, 0); + vstream << mtx; + CTransaction tx(deserialize, vstream); // check all inputs concurrently, with the cache PrecomputedTransactionData txdata(tx); @@ -547,7 +545,7 @@ BOOST_AUTO_TEST_CASE(test_witness) keystore2.AddCScript(GetScriptForWitness(scriptMulti)); keystore2.AddKeyPubKey(key3, pubkey3); - CTransaction output1, output2; + CTransactionRef output1, output2; CMutableTransaction input1, input2; SignatureData sigdata; @@ -639,8 +637,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CheckWithFlag(output1, input1, 0, false); CreateCreditAndSpend(keystore2, scriptMulti, output2, input2, false); CheckWithFlag(output2, input2, 0, false); - BOOST_CHECK(output1 == output2); - UpdateTransaction(input1, 0, CombineSignatures(output1.vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1.vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); + BOOST_CHECK(*output1 == *output2); + UpdateTransaction(input1, 0, CombineSignatures(output1->vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1->vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); // P2SH 2-of-2 multisig @@ -650,8 +648,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore2, GetScriptForDestination(CScriptID(scriptMulti)), output2, input2, false); CheckWithFlag(output2, input2, 0, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH, false); - BOOST_CHECK(output1 == output2); - UpdateTransaction(input1, 0, CombineSignatures(output1.vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1.vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); + BOOST_CHECK(*output1 == *output2); + UpdateTransaction(input1, 0, CombineSignatures(output1->vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1->vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); @@ -662,8 +660,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore2, GetScriptForWitness(scriptMulti), output2, input2, false); CheckWithFlag(output2, input2, 0, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); - BOOST_CHECK(output1 == output2); - UpdateTransaction(input1, 0, CombineSignatures(output1.vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1.vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); + BOOST_CHECK(*output1 == *output2); + UpdateTransaction(input1, 0, CombineSignatures(output1->vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1->vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); @@ -674,8 +672,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore2, GetScriptForDestination(CScriptID(GetScriptForWitness(scriptMulti))), output2, input2, false); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); - BOOST_CHECK(output1 == output2); - UpdateTransaction(input1, 0, CombineSignatures(output1.vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1.vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); + BOOST_CHECK(*output1 == *output2); + UpdateTransaction(input1, 0, CombineSignatures(output1->vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1->vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); } diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 76e4e7a4be..9f58f004d5 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -4,7 +4,7 @@ #include "consensus/validation.h" #include "key.h" -#include "main.h" +#include "validation.h" #include "miner.h" #include "pubkey.h" #include "txmempool.h" @@ -39,6 +39,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) spends.resize(2); for (int i = 0; i < 2; i++) { + spends[i].nVersion = 1; spends[i].vin.resize(1); spends[i].vin[0].prevout.hash = coinbaseTxns[0].GetHash(); spends[i].vin[0].prevout.n = 0; diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index c05d593ed6..bae0eff7e5 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -7,7 +7,7 @@ #include "test/test_bitcoin.h" #include "test/test_random.h" #include "chainparams.h" -#include "main.h" +#include "validation.h" #include "consensus/params.h" #include <boost/test/unit_test.hpp> diff --git a/src/timedata.cpp b/src/timedata.cpp index 25fc494121..b3cbdcf883 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -13,6 +13,7 @@ #include "ui_interface.h" #include "util.h" #include "utilstrencodings.h" +#include "warnings.h" #include <boost/foreach.hpp> @@ -103,8 +104,8 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) if (!fMatch) { fDone = true; - string strMessage = strprintf(_("Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly."), _(PACKAGE_NAME)); - strMiscWarning = strMessage; + std::string strMessage = strprintf(_("Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly."), _(PACKAGE_NAME)); + SetMiscWarning(strMessage); uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_WARNING); } } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index ffb9993f90..67a69363f0 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -501,10 +501,10 @@ static std::vector<uint8_t> ComputeResponse(const std::string &key, const std::v { CHMAC_SHA256 computeHash((const uint8_t*)key.data(), key.size()); std::vector<uint8_t> computedHash(CHMAC_SHA256::OUTPUT_SIZE, 0); - computeHash.Write(begin_ptr(cookie), cookie.size()); - computeHash.Write(begin_ptr(clientNonce), clientNonce.size()); - computeHash.Write(begin_ptr(serverNonce), serverNonce.size()); - computeHash.Finalize(begin_ptr(computedHash)); + computeHash.Write(cookie.data(), cookie.size()); + computeHash.Write(clientNonce.data(), clientNonce.size()); + computeHash.Write(serverNonce.data(), serverNonce.size()); + computeHash.Finalize(computedHash.data()); return computedHash; } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index f11a36fd40..c3da69c3f0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -8,7 +8,7 @@ #include "clientversion.h" #include "consensus/consensus.h" #include "consensus/validation.h" -#include "main.h" +#include "validation.h" #include "policy/policy.h" #include "policy/fees.h" #include "streams.h" @@ -503,7 +503,7 @@ void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants } } -void CTxMemPool::removeRecursive(const CTransaction &origTx, std::vector<CTransactionRef>* removed) +void CTxMemPool::removeRecursive(const CTransaction &origTx) { // Remove transaction from memory pool { @@ -530,11 +530,6 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, std::vector<CTransa BOOST_FOREACH(txiter it, txToRemove) { CalculateDescendants(it, setAllRemoves); } - if (removed) { - BOOST_FOREACH(txiter it, setAllRemoves) { - removed->emplace_back(it->GetSharedTx()); - } - } RemoveStaged(setAllRemoves, false); } } @@ -576,7 +571,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem RemoveStaged(setAllRemoves, false); } -void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<CTransactionRef>* removed) +void CTxMemPool::removeConflicts(const CTransaction &tx) { // Remove transactions which depend on inputs of tx, recursively LOCK(cs); @@ -586,7 +581,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<CTransactio const CTransaction &txConflict = *it->second; if (txConflict != tx) { - removeRecursive(txConflict, removed); + removeRecursive(txConflict); ClearPrioritisation(txConflict.GetHash()); } } @@ -597,7 +592,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<CTransactio * Called when a block is connected. Removes from mempool and updates the miner fee estimator. */ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight, - std::vector<CTransactionRef>* conflicts, bool fCurrentEstimate) + bool fCurrentEstimate) { LOCK(cs); std::vector<CTxMemPoolEntry> entries; @@ -617,7 +612,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne stage.insert(it); RemoveStaged(stage, true); } - removeConflicts(*tx, conflicts); + removeConflicts(*tx); ClearPrioritisation(tx->GetHash()); } // After the txs in the new block have been removed from the mempool, update policy estimates diff --git a/src/txmempool.h b/src/txmempool.h index ef445abaab..8a5787a886 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -527,11 +527,11 @@ public: bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); - void removeRecursive(const CTransaction &tx, std::vector<CTransactionRef>* removed = NULL); + void removeRecursive(const CTransaction &tx); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); - void removeConflicts(const CTransaction &tx, std::vector<CTransactionRef>* removed = NULL); + void removeConflicts(const CTransaction &tx); void removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight, - std::vector<CTransactionRef>* conflicts = NULL, bool fCurrentEstimate = true); + bool fCurrentEstimate = true); void clear(); void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); diff --git a/src/util.cpp b/src/util.cpp index 014013d214..977f8993ed 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -107,7 +107,7 @@ map<string, vector<string> > mapMultiArgs; bool fDebug = false; bool fPrintToConsole = false; bool fPrintToDebugLog = true; -string strMiscWarning; + bool fLogTimestamps = DEFAULT_LOGTIMESTAMPS; bool fLogTimeMicros = DEFAULT_LOGTIMEMICROS; bool fLogIPs = DEFAULT_LOGIPS; @@ -704,13 +704,13 @@ void ShrinkDebugFile() // Restart the file with some of the end std::vector <char> vch(200000,0); fseek(file, -((long)vch.size()), SEEK_END); - int nBytes = fread(begin_ptr(vch), 1, vch.size(), file); + int nBytes = fread(vch.data(), 1, vch.size(), file); fclose(file); file = fopen(pathLog.string().c_str(), "w"); if (file) { - fwrite(begin_ptr(vch), 1, nBytes, file); + fwrite(vch.data(), 1, nBytes, file); fclose(file); } } diff --git a/src/util.h b/src/util.h index e8aa266f28..3ec38a7c7d 100644 --- a/src/util.h +++ b/src/util.h @@ -46,7 +46,7 @@ extern std::map<std::string, std::vector<std::string> > mapMultiArgs; extern bool fDebug; extern bool fPrintToConsole; extern bool fPrintToDebugLog; -extern std::string strMiscWarning; + extern bool fLogTimestamps; extern bool fLogTimeMicros; extern bool fLogIPs; diff --git a/src/main.cpp b/src/validation.cpp index e2a1f31228..0f5975922b 100644 --- a/src/main.cpp +++ b/src/validation.cpp @@ -3,11 +3,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "main.h" +#include "validation.h" -#include "addrman.h" #include "arith_uint256.h" -#include "blockencodings.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" @@ -16,10 +14,6 @@ #include "consensus/validation.h" #include "hash.h" #include "init.h" -#include "merkleblock.h" -#include "net.h" -#include "netmessagemaker.h" -#include "netbase.h" #include "policy/fees.h" #include "policy/policy.h" #include "pow.h" @@ -29,6 +23,7 @@ #include "script/script.h" #include "script/sigcache.h" #include "script/standard.h" +#include "timedata.h" #include "tinyformat.h" #include "txdb.h" #include "txmempool.h" @@ -39,6 +34,7 @@ #include "utilstrencodings.h" #include "validationinterface.h" #include "versionbits.h" +#include "warnings.h" #include <atomic> #include <sstream> @@ -65,7 +61,6 @@ CCriticalSection cs_main; BlockMap mapBlockIndex; CChain chainActive; CBlockIndex *pindexBestHeader = NULL; -int64_t nTimeBestReceived = 0; // Used only to inform the wallet of when we last received a block CWaitableCriticalSection csBestBlock; CConditionVariable cvBlockChange; int nScriptCheckThreads = 0; @@ -88,25 +83,6 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CTxMemPool mempool(::minRelayTxFee); -FeeFilterRounder filterRounder(::minRelayTxFee); - -struct IteratorComparator -{ - template<typename I> - bool operator()(const I& a, const I& b) - { - return &(*a) < &(*b); - } -}; - -struct COrphanTx { - CTransaction tx; - NodeId fromPeer; - int64_t nTimeExpire; -}; -map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main); -map<COutPoint, set<map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); -void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -115,8 +91,6 @@ CScript COINBASE_FLAGS; const string strMessageMagic = "Bitcoin Signed Message:\n"; -static const uint64_t RANDOMIZER_ID_ADDRESS_RELAY = 0x3cac0035b5866b90ULL; // SHA256("main address relay")[0:8] - // Internal stuff namespace { @@ -149,8 +123,6 @@ namespace { * missing the data for the block. */ set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; - /** Number of nodes with fSyncStarted. */ - int nSyncStarted = 0; /** All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. * Pruned nodes may have entries where B is missing data. */ @@ -177,527 +149,13 @@ namespace { /** chainwork for the last block that preciousblock has been applied to. */ arith_uint256 nLastPreciousChainwork = 0; - /** - * Sources of received blocks, saved to be able to send them reject - * messages or ban them when processing happens afterwards. Protected by - * cs_main. - * Set mapBlockSource[hash].second to false if the node should not be - * punished if the block is invalid. - */ - map<uint256, std::pair<NodeId, bool>> mapBlockSource; - - /** - * Filter for transactions that were recently rejected by - * AcceptToMemoryPool. These are not rerequested until the chain tip - * changes, at which point the entire filter is reset. Protected by - * cs_main. - * - * Without this filter we'd be re-requesting txs from each of our peers, - * increasing bandwidth consumption considerably. For instance, with 100 - * peers, half of which relay a tx we don't accept, that might be a 50x - * bandwidth increase. A flooding attacker attempting to roll-over the - * filter using minimum-sized, 60byte, transactions might manage to send - * 1000/sec if we have fast peers, so we pick 120,000 to give our peers a - * two minute window to send invs to us. - * - * Decreasing the false positive rate is fairly cheap, so we pick one in a - * million to make it highly unlikely for users to have issues with this - * filter. - * - * Memory used: 1.3 MB - */ - std::unique_ptr<CRollingBloomFilter> recentRejects; - uint256 hashRecentRejectsChainTip; - - /** Blocks that are in flight, and that are in the queue to be downloaded. Protected by cs_main. */ - struct QueuedBlock { - uint256 hash; - CBlockIndex* pindex; //!< Optional. - bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. - std::unique_ptr<PartiallyDownloadedBlock> partialBlock; //!< Optional, used for CMPCTBLOCK downloads - }; - map<uint256, pair<NodeId, list<QueuedBlock>::iterator> > mapBlocksInFlight; - - /** Stack of nodes which we have set to announce using compact blocks */ - list<NodeId> lNodesAnnouncingHeaderAndIDs; - - /** Number of preferable block download peers. */ - int nPreferredDownload = 0; - /** Dirty block index entries. */ set<CBlockIndex*> setDirtyBlockIndex; /** Dirty block file entries. */ set<int> setDirtyFileInfo; - - /** Number of peers from which we're downloading blocks. */ - int nPeersWithValidatedDownloads = 0; - - /** Relay map, protected by cs_main. */ - typedef std::map<uint256, CTransactionRef> MapRelay; - MapRelay mapRelay; - /** Expiration-time ordered list of (expire time, relay map entry) pairs, protected by cs_main). */ - std::deque<std::pair<int64_t, MapRelay::iterator>> vRelayExpiration; -} // anon namespace - -////////////////////////////////////////////////////////////////////////////// -// -// Registration of network node signals. -// - -namespace { - -struct CBlockReject { - unsigned char chRejectCode; - string strRejectReason; - uint256 hashBlock; -}; - -/** - * Maintain validation-specific state about nodes, protected by cs_main, instead - * by CNode's own locks. This simplifies asynchronous operation, where - * processing of incoming data is done after the ProcessMessage call returns, - * and we're no longer holding the node's locks. - */ -struct CNodeState { - //! The peer's address - const CService address; - //! Whether we have a fully established connection. - bool fCurrentlyConnected; - //! Accumulated misbehaviour score for this peer. - int nMisbehavior; - //! Whether this peer should be disconnected and banned (unless whitelisted). - bool fShouldBan; - //! String name of this peer (debugging/logging purposes). - const std::string name; - //! List of asynchronously-determined block rejections to notify this peer about. - std::vector<CBlockReject> rejects; - //! The best known block we know this peer has announced. - CBlockIndex *pindexBestKnownBlock; - //! The hash of the last unknown block this peer has announced. - uint256 hashLastUnknownBlock; - //! The last full block we both have. - CBlockIndex *pindexLastCommonBlock; - //! The best header we have sent our peer. - CBlockIndex *pindexBestHeaderSent; - //! Length of current-streak of unconnecting headers announcements - int nUnconnectingHeaders; - //! Whether we've started headers synchronization with this peer. - bool fSyncStarted; - //! Since when we're stalling block download progress (in microseconds), or 0. - int64_t nStallingSince; - list<QueuedBlock> vBlocksInFlight; - //! When the first entry in vBlocksInFlight started downloading. Don't care when vBlocksInFlight is empty. - int64_t nDownloadingSince; - int nBlocksInFlight; - int nBlocksInFlightValidHeaders; - //! Whether we consider this a preferred download peer. - bool fPreferredDownload; - //! Whether this peer wants invs or headers (when possible) for block announcements. - bool fPreferHeaders; - //! Whether this peer wants invs or cmpctblocks (when possible) for block announcements. - bool fPreferHeaderAndIDs; - /** - * Whether this peer will send us cmpctblocks if we request them. - * This is not used to gate request logic, as we really only care about fSupportsDesiredCmpctVersion, - * but is used as a flag to "lock in" the version of compact blocks (fWantsCmpctWitness) we send. - */ - bool fProvidesHeaderAndIDs; - //! Whether this peer can give us witnesses - bool fHaveWitness; - //! Whether this peer wants witnesses in cmpctblocks/blocktxns - bool fWantsCmpctWitness; - /** - * If we've announced NODE_WITNESS to this peer: whether the peer sends witnesses in cmpctblocks/blocktxns, - * otherwise: whether this peer sends non-witnesses in cmpctblocks/blocktxns. - */ - bool fSupportsDesiredCmpctVersion; - - CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { - fCurrentlyConnected = false; - nMisbehavior = 0; - fShouldBan = false; - pindexBestKnownBlock = NULL; - hashLastUnknownBlock.SetNull(); - pindexLastCommonBlock = NULL; - pindexBestHeaderSent = NULL; - nUnconnectingHeaders = 0; - fSyncStarted = false; - nStallingSince = 0; - nDownloadingSince = 0; - nBlocksInFlight = 0; - nBlocksInFlightValidHeaders = 0; - fPreferredDownload = false; - fPreferHeaders = false; - fPreferHeaderAndIDs = false; - fProvidesHeaderAndIDs = false; - fHaveWitness = false; - fWantsCmpctWitness = false; - fSupportsDesiredCmpctVersion = false; - } -}; - -/** Map maintaining per-node state. Requires cs_main. */ -map<NodeId, CNodeState> mapNodeState; - -// Requires cs_main. -CNodeState *State(NodeId pnode) { - map<NodeId, CNodeState>::iterator it = mapNodeState.find(pnode); - if (it == mapNodeState.end()) - return NULL; - return &it->second; -} - -void UpdatePreferredDownload(CNode* node, CNodeState* state) -{ - nPreferredDownload -= state->fPreferredDownload; - - // Whether this node should be marked as a preferred download node. - state->fPreferredDownload = (!node->fInbound || node->fWhitelisted) && !node->fOneShot && !node->fClient; - - nPreferredDownload += state->fPreferredDownload; -} - -void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) -{ - ServiceFlags nLocalNodeServices = pnode->GetLocalServices(); - uint64_t nonce = pnode->GetLocalNonce(); - int nNodeStartingHeight = pnode->GetMyStartingHeight(); - NodeId nodeid = pnode->GetId(); - CAddress addr = pnode->addr; - - CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); - CAddress addrMe = CAddress(CService(), nLocalNodeServices); - - connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, - nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes)); - - if (fLogIPs) - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); - else - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); -} - -void InitializeNode(CNode *pnode, CConnman& connman) { - CAddress addr = pnode->addr; - std::string addrName = pnode->addrName; - NodeId nodeid = pnode->GetId(); - { - LOCK(cs_main); - mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); - } - if(!pnode->fInbound) - PushNodeVersion(pnode, connman, GetTime()); -} - -void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { - fUpdateConnectionTime = false; - LOCK(cs_main); - CNodeState *state = State(nodeid); - - if (state->fSyncStarted) - nSyncStarted--; - - if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { - fUpdateConnectionTime = true; - } - - BOOST_FOREACH(const QueuedBlock& entry, state->vBlocksInFlight) { - mapBlocksInFlight.erase(entry.hash); - } - EraseOrphansFor(nodeid); - nPreferredDownload -= state->fPreferredDownload; - nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); - assert(nPeersWithValidatedDownloads >= 0); - - mapNodeState.erase(nodeid); - - if (mapNodeState.empty()) { - // Do a consistency check after the last peer is removed. - assert(mapBlocksInFlight.empty()); - assert(nPreferredDownload == 0); - assert(nPeersWithValidatedDownloads == 0); - } -} - -// Requires cs_main. -// Returns a bool indicating whether we requested this block. -// Also used if a block was /not/ received and timed out or started with another peer -bool MarkBlockAsReceived(const uint256& hash) { - map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); - if (itInFlight != mapBlocksInFlight.end()) { - CNodeState *state = State(itInFlight->second.first); - state->nBlocksInFlightValidHeaders -= itInFlight->second.second->fValidatedHeaders; - if (state->nBlocksInFlightValidHeaders == 0 && itInFlight->second.second->fValidatedHeaders) { - // Last validated block on the queue was received. - nPeersWithValidatedDownloads--; - } - if (state->vBlocksInFlight.begin() == itInFlight->second.second) { - // First block on the queue was received, update the start download time for the next one - state->nDownloadingSince = std::max(state->nDownloadingSince, GetTimeMicros()); - } - state->vBlocksInFlight.erase(itInFlight->second.second); - state->nBlocksInFlight--; - state->nStallingSince = 0; - mapBlocksInFlight.erase(itInFlight); - return true; - } - return false; -} - -// Requires cs_main. -// returns false, still setting pit, if the block was already in flight from the same peer -// pit will only be valid as long as the same cs_main lock is being held -bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, CBlockIndex *pindex = NULL, list<QueuedBlock>::iterator **pit = NULL) { - CNodeState *state = State(nodeid); - assert(state != NULL); - - // Short-circuit most stuff in case its from the same node - map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); - if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { - *pit = &itInFlight->second.second; - return false; - } - - // Make sure it's not listed somewhere already. - MarkBlockAsReceived(hash); - - list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), - {hash, pindex, pindex != NULL, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); - state->nBlocksInFlight++; - state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; - if (state->nBlocksInFlight == 1) { - // We're starting a block download (batch) from this peer. - state->nDownloadingSince = GetTimeMicros(); - } - if (state->nBlocksInFlightValidHeaders == 1 && pindex != NULL) { - nPeersWithValidatedDownloads++; - } - itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; - if (pit) - *pit = &itInFlight->second.second; - return true; -} - -/** Check whether the last unknown block a peer advertised is not yet known. */ -void ProcessBlockAvailability(NodeId nodeid) { - CNodeState *state = State(nodeid); - assert(state != NULL); - - if (!state->hashLastUnknownBlock.IsNull()) { - BlockMap::iterator itOld = mapBlockIndex.find(state->hashLastUnknownBlock); - if (itOld != mapBlockIndex.end() && itOld->second->nChainWork > 0) { - if (state->pindexBestKnownBlock == NULL || itOld->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) - state->pindexBestKnownBlock = itOld->second; - state->hashLastUnknownBlock.SetNull(); - } - } -} - -/** Update tracking information about which blocks a peer is assumed to have. */ -void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { - CNodeState *state = State(nodeid); - assert(state != NULL); - - ProcessBlockAvailability(nodeid); - - BlockMap::iterator it = mapBlockIndex.find(hash); - if (it != mapBlockIndex.end() && it->second->nChainWork > 0) { - // An actually better block was announced. - if (state->pindexBestKnownBlock == NULL || it->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) - state->pindexBestKnownBlock = it->second; - } else { - // An unknown block was announced; just assume that the latest one is the best one. - state->hashLastUnknownBlock = hash; - } -} - -void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom, CConnman& connman) { - if (!nodestate->fSupportsDesiredCmpctVersion) { - // Never ask from peers who can't provide witnesses. - return; - } - if (nodestate->fProvidesHeaderAndIDs) { - for (std::list<NodeId>::iterator it = lNodesAnnouncingHeaderAndIDs.begin(); it != lNodesAnnouncingHeaderAndIDs.end(); it++) { - if (*it == pfrom->GetId()) { - lNodesAnnouncingHeaderAndIDs.erase(it); - lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); - return; - } - } - bool fAnnounceUsingCMPCTBLOCK = false; - uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; - if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { - // As per BIP152, we only get 3 of our peers to announce - // blocks using compact encodings. - connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ - connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); - return true; - }); - lNodesAnnouncingHeaderAndIDs.pop_front(); - } - fAnnounceUsingCMPCTBLOCK = true; - connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); - lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); - } -} - -// Requires cs_main -bool CanDirectFetch(const Consensus::Params &consensusParams) -{ - return chainActive.Tip()->GetBlockTime() > GetAdjustedTime() - consensusParams.nPowTargetSpacing * 20; -} - -// Requires cs_main -bool PeerHasHeader(CNodeState *state, CBlockIndex *pindex) -{ - if (state->pindexBestKnownBlock && pindex == state->pindexBestKnownBlock->GetAncestor(pindex->nHeight)) - return true; - if (state->pindexBestHeaderSent && pindex == state->pindexBestHeaderSent->GetAncestor(pindex->nHeight)) - return true; - return false; -} - -/** Find the last common ancestor two blocks have. - * Both pa and pb must be non-NULL. */ -CBlockIndex* LastCommonAncestor(CBlockIndex* pa, CBlockIndex* pb) { - if (pa->nHeight > pb->nHeight) { - pa = pa->GetAncestor(pb->nHeight); - } else if (pb->nHeight > pa->nHeight) { - pb = pb->GetAncestor(pa->nHeight); - } - - while (pa != pb && pa && pb) { - pa = pa->pprev; - pb = pb->pprev; - } - - // Eventually all chain branches meet at the genesis block. - assert(pa == pb); - return pa; -} - -/** Update pindexLastCommonBlock and add not-in-flight missing successors to vBlocks, until it has - * at most count entries. */ -void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<CBlockIndex*>& vBlocks, NodeId& nodeStaller, const Consensus::Params& consensusParams) { - if (count == 0) - return; - - vBlocks.reserve(vBlocks.size() + count); - CNodeState *state = State(nodeid); - assert(state != NULL); - - // Make sure pindexBestKnownBlock is up to date, we'll need it. - ProcessBlockAvailability(nodeid); - - if (state->pindexBestKnownBlock == NULL || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork) { - // This peer has nothing interesting. - return; - } - - if (state->pindexLastCommonBlock == NULL) { - // Bootstrap quickly by guessing a parent of our best tip is the forking point. - // Guessing wrong in either direction is not a problem. - state->pindexLastCommonBlock = chainActive[std::min(state->pindexBestKnownBlock->nHeight, chainActive.Height())]; - } - - // If the peer reorganized, our previous pindexLastCommonBlock may not be an ancestor - // of its current tip anymore. Go back enough to fix that. - state->pindexLastCommonBlock = LastCommonAncestor(state->pindexLastCommonBlock, state->pindexBestKnownBlock); - if (state->pindexLastCommonBlock == state->pindexBestKnownBlock) - return; - - std::vector<CBlockIndex*> vToFetch; - CBlockIndex *pindexWalk = state->pindexLastCommonBlock; - // Never fetch further than the best block we know the peer has, or more than BLOCK_DOWNLOAD_WINDOW + 1 beyond the last - // linked block we have in common with this peer. The +1 is so we can detect stalling, namely if we would be able to - // download that next block if the window were 1 larger. - int nWindowEnd = state->pindexLastCommonBlock->nHeight + BLOCK_DOWNLOAD_WINDOW; - int nMaxHeight = std::min<int>(state->pindexBestKnownBlock->nHeight, nWindowEnd + 1); - NodeId waitingfor = -1; - while (pindexWalk->nHeight < nMaxHeight) { - // Read up to 128 (or more, if more blocks than that are needed) successors of pindexWalk (towards - // pindexBestKnownBlock) into vToFetch. We fetch 128, because CBlockIndex::GetAncestor may be as expensive - // as iterating over ~100 CBlockIndex* entries anyway. - int nToFetch = std::min(nMaxHeight - pindexWalk->nHeight, std::max<int>(count - vBlocks.size(), 128)); - vToFetch.resize(nToFetch); - pindexWalk = state->pindexBestKnownBlock->GetAncestor(pindexWalk->nHeight + nToFetch); - vToFetch[nToFetch - 1] = pindexWalk; - for (unsigned int i = nToFetch - 1; i > 0; i--) { - vToFetch[i - 1] = vToFetch[i]->pprev; - } - - // Iterate over those blocks in vToFetch (in forward direction), adding the ones that - // are not yet downloaded and not in flight to vBlocks. In the mean time, update - // pindexLastCommonBlock as long as all ancestors are already downloaded, or if it's - // already part of our chain (and therefore don't need it even if pruned). - BOOST_FOREACH(CBlockIndex* pindex, vToFetch) { - if (!pindex->IsValid(BLOCK_VALID_TREE)) { - // We consider the chain that this peer is on invalid. - return; - } - if (!State(nodeid)->fHaveWitness && IsWitnessEnabled(pindex->pprev, consensusParams)) { - // We wouldn't download this block or its descendants from this peer. - return; - } - if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) { - if (pindex->nChainTx) - state->pindexLastCommonBlock = pindex; - } else if (mapBlocksInFlight.count(pindex->GetBlockHash()) == 0) { - // The block is not already downloaded, and not yet in flight. - if (pindex->nHeight > nWindowEnd) { - // We reached the end of the window. - if (vBlocks.size() == 0 && waitingfor != nodeid) { - // We aren't able to fetch anything, but we would be if the download window was one larger. - nodeStaller = waitingfor; - } - return; - } - vBlocks.push_back(pindex); - if (vBlocks.size() == count) { - return; - } - } else if (waitingfor == -1) { - // This is the first already-in-flight block. - waitingfor = mapBlocksInFlight[pindex->GetBlockHash()].first; - } - } - } -} - } // anon namespace -bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { - LOCK(cs_main); - CNodeState *state = State(nodeid); - if (state == NULL) - return false; - stats.nMisbehavior = state->nMisbehavior; - stats.nSyncHeight = state->pindexBestKnownBlock ? state->pindexBestKnownBlock->nHeight : -1; - stats.nCommonHeight = state->pindexLastCommonBlock ? state->pindexLastCommonBlock->nHeight : -1; - BOOST_FOREACH(const QueuedBlock& queue, state->vBlocksInFlight) { - if (queue.pindex) - stats.vHeightInFlight.push_back(queue.pindex->nHeight); - } - return true; -} - -void RegisterNodeSignals(CNodeSignals& nodeSignals) -{ - nodeSignals.ProcessMessages.connect(&ProcessMessages); - nodeSignals.SendMessages.connect(&SendMessages); - nodeSignals.InitializeNode.connect(&InitializeNode); - nodeSignals.FinalizeNode.connect(&FinalizeNode); -} - -void UnregisterNodeSignals(CNodeSignals& nodeSignals) -{ - nodeSignals.ProcessMessages.disconnect(&ProcessMessages); - nodeSignals.SendMessages.disconnect(&SendMessages); - nodeSignals.InitializeNode.disconnect(&InitializeNode); - nodeSignals.FinalizeNode.disconnect(&FinalizeNode); -} - CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) { // Find the first block the caller has in the main chain @@ -729,112 +187,6 @@ enum FlushStateMode { // See definition for documentation bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode); -////////////////////////////////////////////////////////////////////////////// -// -// mapOrphanTransactions -// - -bool AddOrphanTx(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) -{ - uint256 hash = tx.GetHash(); - if (mapOrphanTransactions.count(hash)) - return false; - - // Ignore big transactions, to avoid a - // send-big-orphans memory exhaustion attack. If a peer has a legitimate - // large transaction with a missing parent then we assume - // it will rebroadcast it later, after the parent transaction(s) - // have been mined or received. - // 100 orphans, each of which is at most 99,999 bytes big is - // at most 10 megabytes of orphans and somewhat more byprev index (in the worst case): - unsigned int sz = GetTransactionWeight(tx); - if (sz >= MAX_STANDARD_TX_WEIGHT) - { - LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); - return false; - } - - auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME}); - assert(ret.second); - BOOST_FOREACH(const CTxIn& txin, tx.vin) { - mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); - } - - LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), - mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); - return true; -} - -int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) -{ - map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.find(hash); - if (it == mapOrphanTransactions.end()) - return 0; - BOOST_FOREACH(const CTxIn& txin, it->second.tx.vin) - { - auto itPrev = mapOrphanTransactionsByPrev.find(txin.prevout); - if (itPrev == mapOrphanTransactionsByPrev.end()) - continue; - itPrev->second.erase(it); - if (itPrev->second.empty()) - mapOrphanTransactionsByPrev.erase(itPrev); - } - mapOrphanTransactions.erase(it); - return 1; -} - -void EraseOrphansFor(NodeId peer) -{ - int nErased = 0; - map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); - while (iter != mapOrphanTransactions.end()) - { - map<uint256, COrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid - if (maybeErase->second.fromPeer == peer) - { - nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); - } - } - if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer %d\n", nErased, peer); -} - - -unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) -{ - unsigned int nEvicted = 0; - static int64_t nNextSweep; - int64_t nNow = GetTime(); - if (nNextSweep <= nNow) { - // Sweep out expired orphan pool entries: - int nErased = 0; - int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL; - map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); - while (iter != mapOrphanTransactions.end()) - { - map<uint256, COrphanTx>::iterator maybeErase = iter++; - if (maybeErase->second.nTimeExpire <= nNow) { - nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); - } else { - nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime); - } - } - // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. - nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; - if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx due to expiration\n", nErased); - } - while (mapOrphanTransactions.size() > nMaxOrphans) - { - // Evict a random orphan: - uint256 randomhash = GetRandHash(); - map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.lower_bound(randomhash); - if (it == mapOrphanTransactions.end()) - it = mapOrphanTransactions.begin(); - EraseOrphanTx(it->first); - ++nEvicted; - } - return nEvicted; -} - bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) { if (tx.nLockTime == 0) @@ -1189,14 +541,6 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C if (tx.IsCoinBase()) return state.DoS(100, false, REJECT_INVALID, "coinbase"); - // Don't relay version 2 transactions until CSV is active, and we can be - // sure that such transactions will be mined (unless we're on - // -testnet/-regtest). - const CChainParams& chainparams = Params(); - if (fRequireStandard && tx.nVersion >= 2 && VersionBitsTipState(chainparams.GetConsensus(), Consensus::DEPLOYMENT_CSV) != THRESHOLD_ACTIVE) { - return state.DoS(0, false, REJECT_NONSTANDARD, "premature-version2-tx"); - } - // Reject transactions with witness before segregated witness activates (override with -prematurewitness) bool witnessEnabled = IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus()); if (!GetBoolArg("-prematurewitness",false) && !tx.wit.IsNull() && !witnessEnabled) { @@ -1633,7 +977,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa } /** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */ -bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) +bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { CBlockIndex *pindexSlow = NULL; @@ -1642,7 +986,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P CTransactionRef ptx = mempool.get(hash); if (ptx) { - txOut = *ptx; + txOut = ptx; return true; } @@ -1661,7 +1005,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } hashBlock = header.GetHash(); - if (txOut.GetHash() != hash) + if (txOut->GetHash() != hash) return error("%s: txid mismatch", __func__); return true; } @@ -1684,7 +1028,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P if (ReadBlockFromDisk(block, pindexSlow, consensusParams)) { for (const auto& tx : block.vtx) { if (tx->GetHash() == hash) { - txOut = *tx; + txOut = tx; hashBlock = pindexSlow->GetBlockHash(); return true; } @@ -1798,8 +1142,6 @@ bool IsInitialBlockDownload() return false; } -bool fLargeWorkForkFound = false; -bool fLargeWorkInvalidChainFound = false; CBlockIndex *pindexBestForkTip = NULL, *pindexBestForkBase = NULL; static void AlertNotify(const std::string& strMessage) @@ -1834,7 +1176,7 @@ void CheckForkWarningConditions() if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nChainWork > chainActive.Tip()->nChainWork + (GetBlockProof(*chainActive.Tip()) * 6))) { - if (!fLargeWorkForkFound && pindexBestForkBase) + if (!GetfLargeWorkForkFound() && pindexBestForkBase) { std::string warning = std::string("'Warning: Large-work fork detected, forking after block ") + pindexBestForkBase->phashBlock->ToString() + std::string("'"); @@ -1845,18 +1187,18 @@ void CheckForkWarningConditions() LogPrintf("%s: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", __func__, pindexBestForkBase->nHeight, pindexBestForkBase->phashBlock->ToString(), pindexBestForkTip->nHeight, pindexBestForkTip->phashBlock->ToString()); - fLargeWorkForkFound = true; + SetfLargeWorkForkFound(true); } else { LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); - fLargeWorkInvalidChainFound = true; + SetfLargeWorkInvalidChainFound(true); } } else { - fLargeWorkForkFound = false; - fLargeWorkInvalidChainFound = false; + SetfLargeWorkForkFound(false); + SetfLargeWorkInvalidChainFound(false); } } @@ -1893,26 +1235,6 @@ void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) CheckForkWarningConditions(); } -// Requires cs_main. -void Misbehaving(NodeId pnode, int howmuch) -{ - if (howmuch == 0) - return; - - CNodeState *state = State(pnode); - if (state == NULL) - return; - - state->nMisbehavior += howmuch; - int banscore = GetArg("-banscore", DEFAULT_BANSCORE_THRESHOLD); - if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) - { - LogPrintf("%s: %s peer=%d (%d -> %d) BAN THRESHOLD EXCEEDED\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior); - state->fShouldBan = true; - } else - LogPrintf("%s: %s peer=%d (%d -> %d)\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior); -} - void static InvalidChainFound(CBlockIndex* pindexNew) { if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) @@ -2152,7 +1474,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin /** Abort with a message */ bool AbortNode(const std::string& strMessage, const std::string& userMessage="") { - strMiscWarning = strMessage; + SetMiscWarning(strMessage); LogPrintf("*** %s\n", strMessage); uiInterface.ThreadSafeMessageBox( userMessage.empty() ? _("Error: A fatal internal error occurred, see debug.log for details") : userMessage, @@ -2721,9 +2043,10 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); if (state == THRESHOLD_ACTIVE || state == THRESHOLD_LOCKED_IN) { if (state == THRESHOLD_ACTIVE) { - strMiscWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); + std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); + SetMiscWarning(strWarning); if (!fWarned) { - AlertNotify(strMiscWarning); + AlertNotify(strWarning); fWarned = true; } } else { @@ -2743,10 +2066,11 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); if (nUpgraded > 100/2) { - // strMiscWarning is read by GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - strMiscWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); + std::string strWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); + // notify GetWarnings(), called by Qt and the JSON-RPC code to warn the user: + SetMiscWarning(strWarning); if (!fWarned) { - AlertNotify(strMiscWarning); + AlertNotify(strWarning); fWarned = true; } } @@ -2777,7 +2101,8 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara CCoinsViewCache view(pcoinsTip); if (!DisconnectBlock(block, state, pindexDelete, view)) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); - assert(view.Flush()); + bool flushed = view.Flush(); + assert(flushed); } LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. @@ -2822,28 +2147,43 @@ static int64_t nTimeChainState = 0; static int64_t nTimePostConnect = 0; /** + * Used to track blocks whose transactions were applied to the UTXO state as a + * part of a single ActivateBestChainStep call. + */ +struct ConnectTrace { + std::vector<std::pair<CBlockIndex*, std::shared_ptr<const CBlock> > > blocksConnected; +}; + +/** * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. + * + * The block is always added to connectTrace (either after loading from disk or by copying + * pblock) - if that is not intended, care must be taken to remove the last entry in + * blocksConnected in case of failure. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock, std::vector<CTransactionRef> &txConflicted, std::vector<std::tuple<CTransactionRef,CBlockIndex*,int>> &txChanged) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. int64_t nTime1 = GetTimeMicros(); - CBlock block; if (!pblock) { - if (!ReadBlockFromDisk(block, pindexNew, chainparams.GetConsensus())) + std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); + connectTrace.blocksConnected.emplace_back(pindexNew, pblockNew); + if (!ReadBlockFromDisk(*pblockNew, pindexNew, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); - pblock = █ + } else { + connectTrace.blocksConnected.emplace_back(pindexNew, pblock); } + const CBlock& blockConnecting = *connectTrace.blocksConnected.back().second; // Apply the block atomically to the chain state. int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; int64_t nTime3; LogPrint("bench", " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { CCoinsViewCache view(pcoinsTip); - bool rv = ConnectBlock(*pblock, state, pindexNew, view, chainparams); - GetMainSignals().BlockChecked(*pblock, state); + bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); + GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) InvalidBlockFound(pindexNew, state); @@ -2851,7 +2191,8 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; LogPrint("bench", " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); - assert(view.Flush()); + bool flushed = view.Flush(); + assert(flushed); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint("bench", " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); @@ -2861,13 +2202,10 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; - mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, &txConflicted, !IsInitialBlockDownload()); + mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight, !IsInitialBlockDownload()); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); - for (unsigned int i=0; i < pblock->vtx.size(); i++) - txChanged.emplace_back(pblock->vtx[i], pindexNew, i); - int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint("bench", " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); LogPrint("bench", "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); @@ -2948,7 +2286,7 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound, std::vector<CTransactionRef>& txConflicted, std::vector<std::tuple<CTransactionRef,CBlockIndex*,int>>& txChanged) +static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); const CBlockIndex *pindexOldTip = chainActive.Tip(); @@ -2981,7 +2319,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Connect new blocks. BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : NULL, txConflicted, txChanged)) { + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace)) { if (state.IsInvalid()) { // The block violates a consensus rule. if (!state.CorruptionPossible()) @@ -2989,6 +2327,8 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c state = CValidationState(); fInvalidFound = true; fContinue = false; + // If we didn't actually connect the block, don't notify listeners about it + connectTrace.blocksConnected.pop_back(); break; } else { // A system error occurred (disk space, database error, ...). @@ -3046,20 +2386,16 @@ static void NotifyHeaderTip() { * or an activated best chain. pblock is either NULL or a pointer to a block * that is already loaded (to avoid loading it again from disk). */ -bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, const CBlock *pblock) { +bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { CBlockIndex *pindexMostWork = NULL; CBlockIndex *pindexNewTip = NULL; - std::vector<std::tuple<CTransactionRef,CBlockIndex*,int>> txChanged; - if (pblock) - txChanged.reserve(pblock->vtx.size()); do { - txChanged.clear(); boost::this_thread::interruption_point(); if (ShutdownRequested()) break; const CBlockIndex *pindexFork; - std::vector<CTransactionRef> txConflicted; + ConnectTrace connectTrace; bool fInitialDownload; { LOCK(cs_main); @@ -3073,7 +2409,8 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, return true; bool fInvalidFound = false; - if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL, fInvalidFound, txConflicted, txChanged)) + std::shared_ptr<const CBlock> nullBlockPtr; + if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullBlockPtr, fInvalidFound, connectTrace)) return false; if (fInvalidFound) { @@ -3090,13 +2427,12 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, // throw all transactions though the signal-interface // while _not_ holding the cs_main lock - for (const auto& tx : txConflicted) - { - GetMainSignals().SyncTransaction(*tx, pindexNewTip, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + for (const auto& pair : connectTrace.blocksConnected) { + assert(pair.second); + const CBlock& block = *(pair.second); + for (unsigned int i = 0; i < block.vtx.size(); i++) + GetMainSignals().SyncTransaction(*block.vtx[i], pair.first, i); } - // ... and about transactions that got confirmed: - for (unsigned int i = 0; i < txChanged.size(); i++) - GetMainSignals().SyncTransaction(*std::get<0>(txChanged[i]), std::get<1>(txChanged[i]), std::get<2>(txChanged[i])); // Notify external listeners about the new tip. GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); @@ -3528,8 +2864,9 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc out.scriptPubKey[5] = 0xed; memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32); commitment = std::vector<unsigned char>(out.scriptPubKey.begin(), out.scriptPubKey.end()); - const_cast<std::vector<CTxOut>*>(&block.vtx[0]->vout)->push_back(out); - block.vtx[0]->UpdateHash(); + CMutableTransaction tx(*block.vtx[0]); + tx.vout.push_back(out); + block.vtx[0] = MakeTransactionRef(std::move(tx)); } } UpdateUncommittedBlockStructures(block, pindexPrev, consensusParams); @@ -3780,7 +3117,7 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha return true; } -bool ProcessNewBlock(const CChainParams& chainparams, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp, bool *fNewBlock) +bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock) { { LOCK(cs_main); @@ -3789,7 +3126,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const CBlock* pblock, bool CBlockIndex *pindex = NULL; if (fNewBlock) *fNewBlock = false; CValidationState state; - bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fForceProcessing, dbp, fNewBlock); + bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fForceProcessing, NULL, fNewBlock); CheckBlockIndex(chainparams.GetConsensus()); if (!ret) { GetMainSignals().BlockChecked(*pblock, state); @@ -4311,8 +3648,6 @@ void UnloadBlockIndex() pindexBestInvalid = NULL; pindexBestHeader = NULL; mempool.clear(); - mapOrphanTransactions.clear(); - mapOrphanTransactionsByPrev.clear(); mapBlocksUnlinked.clear(); vinfoBlockFile.clear(); nLastBlockFile = 0; @@ -4676,2360 +4011,11 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams) assert(nNodes == forward.size()); } -std::string GetWarnings(const std::string& strFor) -{ - string strStatusBar; - string strRPC; - string strGUI; - const string uiAlertSeperator = "<hr />"; - - if (!CLIENT_VERSION_IS_RELEASE) { - strStatusBar = "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"; - strGUI = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); - } - - if (GetBoolArg("-testsafemode", DEFAULT_TESTSAFEMODE)) - strStatusBar = strRPC = strGUI = "testsafemode enabled"; - - // Misc warnings like out of disk space and clock is wrong - if (strMiscWarning != "") - { - strStatusBar = strMiscWarning; - strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + strMiscWarning; - } - - if (fLargeWorkForkFound) - { - strStatusBar = strRPC = "Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."; - strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); - } - else if (fLargeWorkInvalidChainFound) - { - strStatusBar = strRPC = "Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."; - strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); - } - - if (strFor == "gui") - return strGUI; - else if (strFor == "statusbar") - return strStatusBar; - else if (strFor == "rpc") - return strRPC; - assert(!"GetWarnings(): invalid parameter"); - return "error"; -} - - - - - - - - -////////////////////////////////////////////////////////////////////////////// -// -// blockchain -> download logic notification -// - -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) { - // Initialize global variables that cannot be constructed at startup. - recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); -} - -void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) { - if (nPosInBlock == CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK) - return; - - LOCK(cs_main); - - std::vector<uint256> vOrphanErase; - // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); - if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const CTransaction& orphanTx = (*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); - vOrphanErase.push_back(orphanHash); - } - } - - // Erase orphan transactions include or precluded by this block - if (vOrphanErase.size()) { - int nErased = 0; - BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { - nErased += EraseOrphanTx(orphanHash); - } - LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); - } -} - -void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { - const int nNewHeight = pindexNew->nHeight; - connman->SetBestHeight(nNewHeight); - - if (!fInitialDownload) { - // Find the hashes of all blocks that weren't previously in the best chain. - std::vector<uint256> vHashes; - const CBlockIndex *pindexToAnnounce = pindexNew; - while (pindexToAnnounce != pindexFork) { - vHashes.push_back(pindexToAnnounce->GetBlockHash()); - pindexToAnnounce = pindexToAnnounce->pprev; - if (vHashes.size() == MAX_BLOCKS_TO_ANNOUNCE) { - // Limit announcements in case of a huge reorganization. - // Rely on the peer's synchronization mechanism in that case. - break; - } - } - // Relay inventory, but don't relay old inventory during initial block download. - connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { - if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { - BOOST_REVERSE_FOREACH(const uint256& hash, vHashes) { - pnode->PushBlockHash(hash); - } - } - }); - } - - nTimeBestReceived = GetTime(); -} - -void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationState& state) { - LOCK(cs_main); - - const uint256 hash(block.GetHash()); - std::map<uint256, std::pair<NodeId, bool>>::iterator it = mapBlockSource.find(hash); - - int nDoS = 0; - if (state.IsInvalid(nDoS)) { - if (it != mapBlockSource.end() && State(it->second.first)) { - assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes - CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; - State(it->second.first)->rejects.push_back(reject); - if (nDoS > 0 && it->second.second) - Misbehaving(it->second.first, nDoS); - } - } - if (it != mapBlockSource.end()) - mapBlockSource.erase(it); -} - -////////////////////////////////////////////////////////////////////////////// -// -// Messages -// - - -bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) -{ - switch (inv.type) - { - case MSG_TX: - case MSG_WITNESS_TX: - { - assert(recentRejects); - if (chainActive.Tip()->GetBlockHash() != hashRecentRejectsChainTip) - { - // If the chain tip has changed previously rejected transactions - // might be now valid, e.g. due to a nLockTime'd tx becoming valid, - // or a double-spend. Reset the rejects filter and give those - // txs a second chance. - hashRecentRejectsChainTip = chainActive.Tip()->GetBlockHash(); - recentRejects->reset(); - } - - // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude - // requesting or processing some txs which have already been included in a block - return recentRejects->contains(inv.hash) || - mempool.exists(inv.hash) || - mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoinsInCache(inv.hash); - } - case MSG_BLOCK: - case MSG_WITNESS_BLOCK: - return mapBlockIndex.count(inv.hash); - } - // Don't know what it is, just say we already got one - return true; -} - -static void RelayTransaction(const CTransaction& tx, CConnman& connman) -{ - CInv inv(MSG_TX, tx.GetHash()); - connman.ForEachNode([&inv](CNode* pnode) - { - pnode->PushInventory(inv); - }); -} - -static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connman) -{ - unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) - - // Relay to a limited number of other nodes - // Use deterministic randomness to send to the same nodes for 24 hours - // at a time so the addrKnowns of the chosen nodes prevent repeats - uint64_t hashAddr = addr.GetHash(); - const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); - FastRandomContext insecure_rand; - - std::array<std::pair<uint64_t, CNode*>,2> best{{{0, nullptr}, {0, nullptr}}}; - assert(nRelayNodes <= best.size()); - - auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { - if (pnode->nVersion >= CADDR_TIME_VERSION) { - uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); - for (unsigned int i = 0; i < nRelayNodes; i++) { - if (hashKey > best[i].first) { - std::copy(best.begin() + i, best.begin() + nRelayNodes - 1, best.begin() + i + 1); - best[i] = std::make_pair(hashKey, pnode); - break; - } - } - } - }; - - auto pushfunc = [&addr, &best, nRelayNodes, &insecure_rand] { - for (unsigned int i = 0; i < nRelayNodes && best[i].first != 0; i++) { - best[i].second->PushAddress(addr, insecure_rand); - } - }; - - connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); -} - -void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParams, CConnman& connman) -{ - std::deque<CInv>::iterator it = pfrom->vRecvGetData.begin(); - unsigned int nMaxSendBufferSize = connman.GetSendBufferSize(); - vector<CInv> vNotFound; - CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - LOCK(cs_main); - - while (it != pfrom->vRecvGetData.end()) { - // Don't bother if send buffer is too full to respond anyway - if (pfrom->nSendSize >= nMaxSendBufferSize) - break; - - const CInv &inv = *it; - { - boost::this_thread::interruption_point(); - it++; - - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) - { - bool send = false; - BlockMap::iterator mi = mapBlockIndex.find(inv.hash); - if (mi != mapBlockIndex.end()) - { - if (chainActive.Contains(mi->second)) { - send = true; - } else { - static const int nOneMonth = 30 * 24 * 60 * 60; - // To prevent fingerprinting attacks, only send blocks outside of the active - // chain if they are valid, and no more than a month older (both in time, and in - // best equivalent proof of work) than the best header chain we know about. - send = mi->second->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != NULL) && - (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() < nOneMonth) && - (GetBlockProofEquivalentTime(*pindexBestHeader, *mi->second, *pindexBestHeader, consensusParams) < nOneMonth); - if (!send) { - LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); - } - } - } - // disconnect node in case we have reached the outbound limit for serving historical blocks - // never disconnect whitelisted nodes - static const int nOneWeek = 7 * 24 * 60 * 60; // assume > 1 week = historical - if (send && connman.OutboundTargetReached(true) && ( ((pindexBestHeader != NULL) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > nOneWeek)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) - { - LogPrint("net", "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); - - //disconnect node - pfrom->fDisconnect = true; - send = false; - } - // Pruned nodes may have deleted the block, so check whether - // it's available before trying to send. - if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) - { - // Send block from disk - CBlock block; - if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) - assert(!"cannot load block from disk"); - if (inv.type == MSG_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block)); - else if (inv.type == MSG_WITNESS_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); - else if (inv.type == MSG_FILTERED_BLOCK) - { - bool sendMerkleBlock = false; - CMerkleBlock merkleBlock; - { - LOCK(pfrom->cs_filter); - if (pfrom->pfilter) { - sendMerkleBlock = true; - merkleBlock = CMerkleBlock(block, *pfrom->pfilter); - } - } - if (sendMerkleBlock) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); - // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see - // This avoids hurting performance by pointlessly requiring a round-trip - // Note that there is currently no way for a node to request any single transactions we didn't send here - - // they must either disconnect and retry or request the full block. - // Thus, the protocol spec specified allows for us to provide duplicate txn here, - // however we MUST always provide at least what the remote peer needs - typedef std::pair<unsigned int, uint256> PairType; - BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *block.vtx[pair.first])); - } - // else - // no response - } - else if (inv.type == MSG_CMPCT_BLOCK) - { - // If a peer is asking for old blocks, we're almost guaranteed - // they won't have a useful mempool to match against a compact block, - // and we don't feel like constructing the object for them, so - // instead we respond with the full, non-compact block. - bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; - int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; - if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { - CBlockHeaderAndShortTxIDs cmpctblock(block, fPeerWantsWitness); - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); - } else - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, block)); - } - - // Trigger the peer node to send a getblocks request for the next batch of inventory - if (inv.hash == pfrom->hashContinue) - { - // Bypass PushInventory, this must send even if redundant, - // and we want it right after the last block so they don't - // wait for other stuff first. - vector<CInv> vInv; - vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); - pfrom->hashContinue.SetNull(); - } - } - } - else if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) - { - // Send stream from relay memory - bool push = false; - auto mi = mapRelay.find(inv.hash); - int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - if (mi != mapRelay.end()) { - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *mi->second)); - push = true; - } else if (pfrom->timeLastMempoolReq) { - auto txinfo = mempool.info(inv.hash); - // To protect privacy, do not answer getdata using the mempool when - // that TX couldn't have been INVed in reply to a MEMPOOL request. - if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *txinfo.tx)); - push = true; - } - } - if (!push) { - vNotFound.push_back(inv); - } - } - - // Track requests for our stuff. - GetMainSignals().Inventory(inv.hash); - - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) - break; - } - } - - pfrom->vRecvGetData.erase(pfrom->vRecvGetData.begin(), it); - - if (!vNotFound.empty()) { - // Let the peer know that we didn't find what it asked for, so it doesn't - // have to wait around forever. Currently only SPV clients actually care - // about this message: it's needed when they are recursively walking the - // dependencies of relevant unconfirmed transactions. SPV clients want to - // do that because they want to know about (and store and rebroadcast and - // risk analyze) the dependencies of transactions relevant to them, without - // having to download the entire memory pool. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); - } -} - -uint32_t GetFetchFlags(CNode* pfrom, CBlockIndex* pprev, const Consensus::Params& chainparams) { - uint32_t nFetchFlags = 0; - if ((pfrom->GetLocalServices() & NODE_WITNESS) && State(pfrom->GetId())->fHaveWitness) { - nFetchFlags |= MSG_WITNESS_FLAG; - } - return nFetchFlags; -} - -bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman) -{ - unsigned int nMaxSendBufferSize = connman.GetSendBufferSize(); - - LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); - if (mapArgs.count("-dropmessagestest") && GetRand(atoi(mapArgs["-dropmessagestest"])) == 0) - { - LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); - return true; - } - - - if (!(pfrom->GetLocalServices() & NODE_BLOOM) && - (strCommand == NetMsgType::FILTERLOAD || - strCommand == NetMsgType::FILTERADD)) - { - if (pfrom->nVersion >= NO_BLOOM_VERSION) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 100); - return false; - } else { - pfrom->fDisconnect = true; - return false; - } - } - - - if (strCommand == NetMsgType::VERSION) - { - // Each connection can only send one version message - if (pfrom->nVersion != 0) - { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message"))); - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 1); - return false; - } - - int64_t nTime; - CAddress addrMe; - CAddress addrFrom; - uint64_t nNonce = 1; - uint64_t nServiceInt; - vRecv >> pfrom->nVersion >> nServiceInt >> nTime >> addrMe; - pfrom->nServices = ServiceFlags(nServiceInt); - if (!pfrom->fInbound) - { - connman.SetServices(pfrom->addr, pfrom->nServices); - } - if (pfrom->nServicesExpected & ~pfrom->nServices) - { - LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, pfrom->nServices, pfrom->nServicesExpected); - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, - strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); - pfrom->fDisconnect = true; - return false; - } - - if (pfrom->nVersion < MIN_PEER_PROTO_VERSION) - { - // disconnect from peers older than this proto version - LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, pfrom->nVersion); - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, - strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); - pfrom->fDisconnect = true; - return false; - } - - if (pfrom->nVersion == 10300) - pfrom->nVersion = 300; - if (!vRecv.empty()) - vRecv >> addrFrom >> nNonce; - if (!vRecv.empty()) { - vRecv >> LIMITED_STRING(pfrom->strSubVer, MAX_SUBVERSION_LENGTH); - pfrom->cleanSubVer = SanitizeString(pfrom->strSubVer); - } - if (!vRecv.empty()) { - vRecv >> pfrom->nStartingHeight; - } - { - LOCK(pfrom->cs_filter); - if (!vRecv.empty()) - vRecv >> pfrom->fRelayTxes; // set to true after we get the first filter* message - else - pfrom->fRelayTxes = true; - } - - // Disconnect if we connected to ourself - if (pfrom->fInbound && !connman.CheckIncomingNonce(nNonce)) - { - LogPrintf("connected to self at %s, disconnecting\n", pfrom->addr.ToString()); - pfrom->fDisconnect = true; - return true; - } - - pfrom->addrLocal = addrMe; - if (pfrom->fInbound && addrMe.IsRoutable()) - { - SeenLocal(addrMe); - } - - // Be shy and don't send version until we hear - if (pfrom->fInbound) - PushNodeVersion(pfrom, connman, GetAdjustedTime()); - - pfrom->fClient = !(pfrom->nServices & NODE_NETWORK); - - if((pfrom->nServices & NODE_WITNESS)) - { - LOCK(cs_main); - State(pfrom->GetId())->fHaveWitness = true; - } - - // Potentially mark this peer as a preferred download peer. - { - LOCK(cs_main); - UpdatePreferredDownload(pfrom, State(pfrom->GetId())); - } - - // Change version - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); - int nSendVersion = std::min(pfrom->nVersion, PROTOCOL_VERSION); - pfrom->SetSendVersion(nSendVersion); - - if (!pfrom->fInbound) - { - // Advertise our address - if (fListen && !IsInitialBlockDownload()) - { - CAddress addr = GetLocalAddress(&pfrom->addr, pfrom->GetLocalServices()); - FastRandomContext insecure_rand; - if (addr.IsRoutable()) - { - LogPrint("net", "ProcessMessages: advertising address %s\n", addr.ToString()); - pfrom->PushAddress(addr, insecure_rand); - } else if (IsPeerAddrLocalGood(pfrom)) { - addr.SetIP(pfrom->addrLocal); - LogPrint("net", "ProcessMessages: advertising address %s\n", addr.ToString()); - pfrom->PushAddress(addr, insecure_rand); - } - } - - // Get recent addresses - if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman.GetAddressCount() < 1000) - { - connman.PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); - pfrom->fGetAddr = true; - } - connman.MarkAddressGood(pfrom->addr); - } - - pfrom->fSuccessfullyConnected = true; - - string remoteAddr; - if (fLogIPs) - remoteAddr = ", peeraddr=" + pfrom->addr.ToString(); - - LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", - pfrom->cleanSubVer, pfrom->nVersion, - pfrom->nStartingHeight, addrMe.ToString(), pfrom->id, - remoteAddr); - - int64_t nTimeOffset = nTime - GetTime(); - pfrom->nTimeOffset = nTimeOffset; - AddTimeData(pfrom->addr, nTimeOffset); - - // Feeler connections exist only to verify if address is online. - if (pfrom->fFeeler) { - assert(pfrom->fInbound == false); - pfrom->fDisconnect = true; - } - return true; - } - - - else if (pfrom->nVersion == 0) - { - // Must have a version message before anything else - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 1); - return false; - } - - // At this point, the outgoing message serialization version can't change. - CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - - if (strCommand == NetMsgType::VERACK) - { - pfrom->SetRecvVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); - - if (!pfrom->fInbound) { - // Mark this node as currently connected, so we update its timestamp later. - LOCK(cs_main); - State(pfrom->GetId())->fCurrentlyConnected = true; - } - - if (pfrom->nVersion >= SENDHEADERS_VERSION) { - // Tell our peer we prefer to receive headers rather than inv's - // We send this to non-NODE NETWORK peers as well, because even - // non-NODE NETWORK peers can announce blocks (such as pruning - // nodes) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); - } - if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { - // Tell our peer we are willing to provide version 1 or 2 cmpctblocks - // However, we do not request new block announcements using - // cmpctblock messages. - // We send this to non-NODE NETWORK peers as well, because - // they may wish to request compact blocks from us - bool fAnnounceUsingCMPCTBLOCK = false; - uint64_t nCMPCTBLOCKVersion = 2; - if (pfrom->GetLocalServices() & NODE_WITNESS) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); - nCMPCTBLOCKVersion = 1; - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); - } - } - - - else if (strCommand == NetMsgType::ADDR) - { - vector<CAddress> vAddr; - vRecv >> vAddr; - - // Don't want addr from older versions unless seeding - if (pfrom->nVersion < CADDR_TIME_VERSION && connman.GetAddressCount() > 1000) - return true; - if (vAddr.size() > 1000) - { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20); - return error("message addr size() = %u", vAddr.size()); - } - - // Store the new addresses - vector<CAddress> vAddrOk; - int64_t nNow = GetAdjustedTime(); - int64_t nSince = nNow - 10 * 60; - BOOST_FOREACH(CAddress& addr, vAddr) - { - boost::this_thread::interruption_point(); - - if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) - continue; - - if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) - addr.nTime = nNow - 5 * 24 * 60 * 60; - pfrom->AddAddressKnown(addr); - bool fReachable = IsReachable(addr); - if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) - { - // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, connman); - } - // Do not store addresses outside our network - if (fReachable) - vAddrOk.push_back(addr); - } - connman.AddNewAddresses(vAddrOk, pfrom->addr, 2 * 60 * 60); - if (vAddr.size() < 1000) - pfrom->fGetAddr = false; - if (pfrom->fOneShot) - pfrom->fDisconnect = true; - } - - else if (strCommand == NetMsgType::SENDHEADERS) - { - LOCK(cs_main); - State(pfrom->GetId())->fPreferHeaders = true; - } - - else if (strCommand == NetMsgType::SENDCMPCT) - { - bool fAnnounceUsingCMPCTBLOCK = false; - uint64_t nCMPCTBLOCKVersion = 0; - vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; - if (nCMPCTBLOCKVersion == 1 || ((pfrom->GetLocalServices() & NODE_WITNESS) && nCMPCTBLOCKVersion == 2)) { - LOCK(cs_main); - // fProvidesHeaderAndIDs is used to "lock in" version of compact blocks we send (fWantsCmpctWitness) - if (!State(pfrom->GetId())->fProvidesHeaderAndIDs) { - State(pfrom->GetId())->fProvidesHeaderAndIDs = true; - State(pfrom->GetId())->fWantsCmpctWitness = nCMPCTBLOCKVersion == 2; - } - if (State(pfrom->GetId())->fWantsCmpctWitness == (nCMPCTBLOCKVersion == 2)) // ignore later version announces - State(pfrom->GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; - if (!State(pfrom->GetId())->fSupportsDesiredCmpctVersion) { - if (pfrom->GetLocalServices() & NODE_WITNESS) - State(pfrom->GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 2); - else - State(pfrom->GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 1); - } - } - } - - - else if (strCommand == NetMsgType::INV) - { - vector<CInv> vInv; - vRecv >> vInv; - if (vInv.size() > MAX_INV_SZ) - { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20); - return error("message inv size() = %u", vInv.size()); - } - - bool fBlocksOnly = !fRelayTxes; - - // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true - if (pfrom->fWhitelisted && GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY)) - fBlocksOnly = false; - - LOCK(cs_main); - - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); - - std::vector<CInv> vToFetch; - - for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) - { - CInv &inv = vInv[nInv]; - - boost::this_thread::interruption_point(); - - bool fAlreadyHave = AlreadyHave(inv); - LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); - - if (inv.type == MSG_TX) { - inv.type |= nFetchFlags; - } - - if (inv.type == MSG_BLOCK) { - UpdateBlockAvailability(pfrom->GetId(), inv.hash); - if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { - // We used to request the full block here, but since headers-announcements are now the - // primary method of announcement on the network, and since, in the case that a node - // fell back to inv we probably have a reorg which we should get the headers for first, - // we now only provide a getheaders response here. When we receive the headers, we will - // then ask for the blocks we need. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); - LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); - } - } - else - { - pfrom->AddInventoryKnown(inv); - if (fBlocksOnly) - LogPrint("net", "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); - else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) - pfrom->AskFor(inv); - } - - // Track requests for our stuff - GetMainSignals().Inventory(inv.hash); - - if (pfrom->nSendSize > (nMaxSendBufferSize * 2)) { - Misbehaving(pfrom->GetId(), 50); - return error("send buffer size() = %u", pfrom->nSendSize); - } - } - - if (!vToFetch.empty()) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vToFetch)); - } - - - else if (strCommand == NetMsgType::GETDATA) - { - vector<CInv> vInv; - vRecv >> vInv; - if (vInv.size() > MAX_INV_SZ) - { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20); - return error("message getdata size() = %u", vInv.size()); - } - - if (fDebug || (vInv.size() != 1)) - LogPrint("net", "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); - - if ((fDebug && vInv.size() > 0) || (vInv.size() == 1)) - LogPrint("net", "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); - - pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); - ProcessGetData(pfrom, chainparams.GetConsensus(), connman); - } - - - else if (strCommand == NetMsgType::GETBLOCKS) - { - CBlockLocator locator; - uint256 hashStop; - vRecv >> locator >> hashStop; - - LOCK(cs_main); - - // Find the last block the caller has in the main chain - CBlockIndex* pindex = FindForkInGlobalIndex(chainActive, locator); - - // Send the rest of the chain - if (pindex) - pindex = chainActive.Next(pindex); - int nLimit = 500; - LogPrint("net", "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); - for (; pindex; pindex = chainActive.Next(pindex)) - { - if (pindex->GetBlockHash() == hashStop) - { - LogPrint("net", " getblocks stopping at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); - break; - } - // If pruning, don't inv blocks unless we have on disk and are likely to still have - // for some reasonable time window (1 hour) that block relay might require. - const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / chainparams.GetConsensus().nPowTargetSpacing; - if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= chainActive.Tip()->nHeight - nPrunedBlocksLikelyToHave)) - { - LogPrint("net", " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); - break; - } - pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); - if (--nLimit <= 0) - { - // When this block is requested, we'll send an inv that'll - // trigger the peer to getblocks the next batch of inventory. - LogPrint("net", " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); - pfrom->hashContinue = pindex->GetBlockHash(); - break; - } - } - } - - - else if (strCommand == NetMsgType::GETBLOCKTXN) - { - BlockTransactionsRequest req; - vRecv >> req; - - LOCK(cs_main); - - BlockMap::iterator it = mapBlockIndex.find(req.blockhash); - if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); - return true; - } - - if (it->second->nHeight < chainActive.Height() - MAX_BLOCKTXN_DEPTH) { - // If an older block is requested (should never happen in practice, - // but can happen in tests) send a block response instead of a - // blocktxn response. Sending a full block response instead of a - // small blocktxn response is preferable in the case where a peer - // might maliciously send lots of getblocktxn requests to trigger - // expensive disk reads, because it will require the peer to - // actually receive all the data read from disk over the network. - LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); - CInv inv; - inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; - inv.hash = req.blockhash; - pfrom->vRecvGetData.push_back(inv); - ProcessGetData(pfrom, chainparams.GetConsensus(), connman); - return true; - } - - CBlock block; - assert(ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())); - - BlockTransactions resp(req); - for (size_t i = 0; i < req.indexes.size(); i++) { - if (req.indexes[i] >= block.vtx.size()) { - Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); - return true; - } - resp.txn[i] = block.vtx[req.indexes[i]]; - } - int nSendFlags = State(pfrom->GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); - } - - - else if (strCommand == NetMsgType::GETHEADERS) - { - CBlockLocator locator; - uint256 hashStop; - vRecv >> locator >> hashStop; - - LOCK(cs_main); - if (IsInitialBlockDownload() && !pfrom->fWhitelisted) { - LogPrint("net", "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); - return true; - } - - CNodeState *nodestate = State(pfrom->GetId()); - CBlockIndex* pindex = NULL; - if (locator.IsNull()) - { - // If locator is null, return the hashStop block - BlockMap::iterator mi = mapBlockIndex.find(hashStop); - if (mi == mapBlockIndex.end()) - return true; - pindex = (*mi).second; - } - else - { - // Find the last block the caller has in the main chain - pindex = FindForkInGlobalIndex(chainActive, locator); - if (pindex) - pindex = chainActive.Next(pindex); - } - - // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end - vector<CBlock> vHeaders; - int nLimit = MAX_HEADERS_RESULTS; - LogPrint("net", "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->id); - for (; pindex; pindex = chainActive.Next(pindex)) - { - vHeaders.push_back(pindex->GetBlockHeader()); - if (--nLimit <= 0 || pindex->GetBlockHash() == hashStop) - break; - } - // pindex can be NULL either if we sent chainActive.Tip() OR - // if our peer has chainActive.Tip() (and thus we are sending an empty - // headers message). In both cases it's safe to update - // pindexBestHeaderSent to be our tip. - nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); - } - - - else if (strCommand == NetMsgType::TX) - { - // Stop processing the transaction early if - // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off - if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) - { - LogPrint("net", "transaction sent in violation of protocol peer=%d\n", pfrom->id); - return true; - } - - deque<COutPoint> vWorkQueue; - vector<uint256> vEraseQueue; - CTransaction tx; - vRecv >> tx; - - CInv inv(MSG_TX, tx.GetHash()); - pfrom->AddInventoryKnown(inv); - - LOCK(cs_main); - - bool fMissingInputs = false; - CValidationState state; - - pfrom->setAskFor.erase(inv.hash); - mapAlreadyAskedFor.erase(inv.hash); - - if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) { - mempool.check(pcoinsTip); - RelayTransaction(tx, connman); - for (unsigned int i = 0; i < tx.vout.size(); i++) { - vWorkQueue.emplace_back(inv.hash, i); - } - - pfrom->nLastTXTime = GetTime(); - - LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", - pfrom->id, - tx.GetHash().ToString(), - mempool.size(), mempool.DynamicMemoryUsage() / 1000); - - // Recursively process any orphan transactions that depended on this one - set<NodeId> setMisbehaving; - while (!vWorkQueue.empty()) { - auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); - vWorkQueue.pop_front(); - if (itByPrev == mapOrphanTransactionsByPrev.end()) - continue; - for (auto mi = itByPrev->second.begin(); - mi != itByPrev->second.end(); - ++mi) - { - const CTransaction& orphanTx = (*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); - NodeId fromPeer = (*mi)->second.fromPeer; - bool fMissingInputs2 = false; - // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan - // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get - // anyone relaying LegitTxX banned) - CValidationState stateDummy; - - - if (setMisbehaving.count(fromPeer)) - continue; - if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { - LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanTx, connman); - for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { - vWorkQueue.emplace_back(orphanHash, i); - } - vEraseQueue.push_back(orphanHash); - } - else if (!fMissingInputs2) - { - int nDos = 0; - if (stateDummy.IsInvalid(nDos) && nDos > 0) - { - // Punish peer that gave us an invalid orphan tx - Misbehaving(fromPeer, nDos); - setMisbehaving.insert(fromPeer); - LogPrint("mempool", " invalid orphan tx %s\n", orphanHash.ToString()); - } - // Has inputs but not accepted to mempool - // Probably non-standard or insufficient fee/priority - LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); - vEraseQueue.push_back(orphanHash); - if (orphanTx.wit.IsNull() && !stateDummy.CorruptionPossible()) { - // Do not use rejection cache for witness transactions or - // witness-stripped transactions, as they can have been malleated. - // See https://github.com/bitcoin/bitcoin/issues/8279 for details. - assert(recentRejects); - recentRejects->insert(orphanHash); - } - } - mempool.check(pcoinsTip); - } - } - - BOOST_FOREACH(uint256 hash, vEraseQueue) - EraseOrphanTx(hash); - } - else if (fMissingInputs) - { - bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected - BOOST_FOREACH(const CTxIn& txin, tx.vin) { - if (recentRejects->contains(txin.prevout.hash)) { - fRejectedParents = true; - break; - } - } - if (!fRejectedParents) { - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); - BOOST_FOREACH(const CTxIn& txin, tx.vin) { - CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); - pfrom->AddInventoryKnown(_inv); - if (!AlreadyHave(_inv)) pfrom->AskFor(_inv); - } - AddOrphanTx(tx, pfrom->GetId()); - - // DoS prevention: do not allow mapOrphanTransactions to grow unbounded - unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); - unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); - if (nEvicted > 0) - LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); - } else { - LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); - } - } else { - if (tx.wit.IsNull() && !state.CorruptionPossible()) { - // Do not use rejection cache for witness transactions or - // witness-stripped transactions, as they can have been malleated. - // See https://github.com/bitcoin/bitcoin/issues/8279 for details. - assert(recentRejects); - recentRejects->insert(tx.GetHash()); - } - - if (pfrom->fWhitelisted && GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { - // Always relay transactions received from whitelisted peers, even - // if they were already in the mempool or rejected from it due - // to policy, allowing the node to function as a gateway for - // nodes hidden behind it. - // - // Never relay transactions that we would assign a non-zero DoS - // score for, as we expect peers to do the same with us in that - // case. - int nDoS = 0; - if (!state.IsInvalid(nDoS) || nDoS == 0) { - LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); - RelayTransaction(tx, connman); - } else { - LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); - } - } - } - int nDoS = 0; - if (state.IsInvalid(nDoS)) - { - LogPrint("mempoolrej", "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), - pfrom->id, - FormatStateMessage(state)); - if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); - if (nDoS > 0) { - Misbehaving(pfrom->GetId(), nDoS); - } - } - } - - - else if (strCommand == NetMsgType::CMPCTBLOCK && !fImporting && !fReindex) // Ignore blocks received while importing - { - CBlockHeaderAndShortTxIDs cmpctblock; - vRecv >> cmpctblock; - - { - LOCK(cs_main); - - if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { - // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers - if (!IsInitialBlockDownload()) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); - return true; - } - } - - CBlockIndex *pindex = NULL; - CValidationState state; - if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { - int nDoS; - if (state.IsInvalid(nDoS)) { - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } - LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); - return true; - } - } - - LOCK(cs_main); - // If AcceptBlockHeader returned true, it set pindex - assert(pindex); - UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); - - std::map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); - bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); - - if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here - return true; - - if (pindex->nChainWork <= chainActive.Tip()->nChainWork || // We know something better - pindex->nTx != 0) { // We had this block at some point, but pruned it - if (fAlreadyInFlight) { - // We requested this block for some reason, but our mempool will probably be useless - // so we just grab the block via normal getdata - std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); - } - return true; - } - - // If we're not close to tip yet, give up and let parallel block fetch work its magic - if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) - return true; - - CNodeState *nodestate = State(pfrom->GetId()); - - if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus()) && !nodestate->fSupportsDesiredCmpctVersion) { - // Don't bother trying to process compact blocks from v1 peers - // after segwit activates. - return true; - } - - // We want to be a bit conservative just to be extra careful about DoS - // possibilities in compact block processing... - if (pindex->nHeight <= chainActive.Height() + 2) { - if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || - (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { - list<QueuedBlock>::iterator *queuedBlockIt = NULL; - if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { - if (!(*queuedBlockIt)->partialBlock) - (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); - else { - // The block was already in flight using compact blocks from the same peer - LogPrint("net", "Peer sent us compact block we were already syncing!\n"); - return true; - } - } - - PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; - ReadStatus status = partialBlock.InitData(cmpctblock); - if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist - Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); - return true; - } else if (status == READ_STATUS_FAILED) { - // Duplicate txindexes, the block is now in-flight, so just request it - std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); - return true; - } - - if (!fAlreadyInFlight && mapBlocksInFlight.size() == 1 && pindex->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // We seem to be rather well-synced, so it appears pfrom was the first to provide us - // with this block! Let's get them to announce using compact blocks in the future. - MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman); - } - - BlockTransactionsRequest req; - for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { - if (!partialBlock.IsTxAvailable(i)) - req.indexes.push_back(i); - } - if (req.indexes.empty()) { - // Dirty hack to jump to BLOCKTXN code (TODO: move message handling into their own functions) - BlockTransactions txn; - txn.blockhash = cmpctblock.header.GetHash(); - CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); - blockTxnMsg << txn; - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman); - } else { - req.blockhash = pindex->GetBlockHash(); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); - } - } - } else { - if (fAlreadyInFlight) { - // We requested this block, but its far into the future, so our - // mempool will probably be useless - request the block normally - std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); - return true; - } else { - // If this was an announce-cmpctblock, we want the same treatment as a header message - // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) - std::vector<CBlock> headers; - headers.push_back(cmpctblock.header); - CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); - vHeadersMsg << headers; - return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman); - } - } - } - - else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing - { - BlockTransactions resp; - vRecv >> resp; - - CBlock block; - bool fBlockRead = false; - { - LOCK(cs_main); - - map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); - if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || - it->second.first != pfrom->GetId()) { - LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); - return true; - } - - PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; - ReadStatus status = partialBlock.FillBlock(block, resp.txn); - if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist - Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); - return true; - } else if (status == READ_STATUS_FAILED) { - // Might have collided, fall back to getdata now :( - std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); - } else { - // Block is either okay, or possibly we received - // READ_STATUS_CHECKBLOCK_FAILED. - // Note that CheckBlock can only fail for one of a few reasons: - // 1. bad-proof-of-work (impossible here, because we've already - // accepted the header) - // 2. merkleroot doesn't match the transactions given (already - // caught in FillBlock with READ_STATUS_FAILED, so - // impossible here) - // 3. the block is otherwise invalid (eg invalid coinbase, - // block is too big, too many legacy sigops, etc). - // So if CheckBlock failed, #3 is the only possibility. - // Under BIP 152, we don't DoS-ban unless proof of work is - // invalid (we don't require all the stateless checks to have - // been run). This is handled below, so just treat this as - // though the block was successfully read, and rely on the - // handling in ProcessNewBlock to ensure the block index is - // updated, reject messages go out, etc. - MarkBlockAsReceived(resp.blockhash); // it is now an empty pointer - fBlockRead = true; - // mapBlockSource is only used for sending reject messages and DoS scores, - // so the race between here and cs_main in ProcessNewBlock is fine. - // BIP 152 permits peers to relay compact blocks after validating - // the header only; we should not punish peers if the block turns - // out to be invalid. - mapBlockSource.emplace(resp.blockhash, std::make_pair(pfrom->GetId(), false)); - } - } // Don't hold cs_main when we call into ProcessNewBlock - if (fBlockRead) { - bool fNewBlock = false; - // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, - // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) - ProcessNewBlock(chainparams, &block, true, NULL, &fNewBlock); - if (fNewBlock) - pfrom->nLastBlockTime = GetTime(); - } - } - - - else if (strCommand == NetMsgType::HEADERS && !fImporting && !fReindex) // Ignore headers received while importing - { - std::vector<CBlockHeader> headers; - - // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. - unsigned int nCount = ReadCompactSize(vRecv); - if (nCount > MAX_HEADERS_RESULTS) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20); - return error("headers message size = %u", nCount); - } - headers.resize(nCount); - for (unsigned int n = 0; n < nCount; n++) { - vRecv >> headers[n]; - ReadCompactSize(vRecv); // ignore tx count; assume it is 0. - } - - if (nCount == 0) { - // Nothing interesting. Stop asking this peers for more headers. - return true; - } - - CBlockIndex *pindexLast = NULL; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - - // If this looks like it could be a block announcement (nCount < - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); - LogPrint("net", "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - pindexBestHeader->nHeight, - pfrom->id, nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(pfrom->GetId(), 20); - } - return true; - } - - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(pfrom->GetId(), 20); - return error("non-continuous headers sequence"); - } - hashLastBlock = header.GetHash(); - } - } - - CValidationState state; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { - int nDoS; - if (state.IsInvalid(nDoS)) { - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } - return error("invalid header received"); - } - } - - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint("net", "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue - // from there instead. - LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); - } - - bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { - vector<CBlockIndex *> vToFetch; - CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && - (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!chainActive.Contains(pindexWalk)) { - LogPrint("net", "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - vector<CInv> vGetData; - // Download as much as possible, from earliest to latest. - BOOST_REVERSE_FOREACH(CBlockIndex *pindex, vToFetch) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); - vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); - LogPrint("net", "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->id); - } - if (vGetData.size() > 1) { - LogPrint("net", "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // We seem to be rather well-synced, so it appears pfrom was the first to provide us - // with this block! Let's get them to announce using compact blocks in the future. - MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman); - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } - } - } - - else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing - { - CBlock block; - vRecv >> block; - - LogPrint("net", "received block %s peer=%d\n", block.GetHash().ToString(), pfrom->id); - - // Process all blocks from whitelisted peers, even if not requested, - // unless we're still syncing with the network. - // Such an unrequested block may still be processed, subject to the - // conditions in AcceptBlock(). - bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); - const uint256 hash(block.GetHash()); - { - LOCK(cs_main); - // Also always process if we requested the block explicitly, as we may - // need it even though it is not a candidate for a new best tip. - forceProcessing |= MarkBlockAsReceived(hash); - // mapBlockSource is only used for sending reject messages and DoS scores, - // so the race between here and cs_main in ProcessNewBlock is fine. - mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); - } - bool fNewBlock = false; - ProcessNewBlock(chainparams, &block, forceProcessing, NULL, &fNewBlock); - if (fNewBlock) - pfrom->nLastBlockTime = GetTime(); - } - - - else if (strCommand == NetMsgType::GETADDR) - { - // This asymmetric behavior for inbound and outbound connections was introduced - // to prevent a fingerprinting attack: an attacker can send specific fake addresses - // to users' AddrMan and later request them by sending getaddr messages. - // Making nodes which are behind NAT and can only make outgoing connections ignore - // the getaddr message mitigates the attack. - if (!pfrom->fInbound) { - LogPrint("net", "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); - return true; - } - - // Only send one GetAddr response per connection to reduce resource waste - // and discourage addr stamping of INV announcements. - if (pfrom->fSentAddr) { - LogPrint("net", "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); - return true; - } - pfrom->fSentAddr = true; - - pfrom->vAddrToSend.clear(); - vector<CAddress> vAddr = connman.GetAddresses(); - FastRandomContext insecure_rand; - BOOST_FOREACH(const CAddress &addr, vAddr) - pfrom->PushAddress(addr, insecure_rand); - } - - - else if (strCommand == NetMsgType::MEMPOOL) - { - if (!(pfrom->GetLocalServices() & NODE_BLOOM) && !pfrom->fWhitelisted) - { - LogPrint("net", "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); - pfrom->fDisconnect = true; - return true; - } - - if (connman.OutboundTargetReached(false) && !pfrom->fWhitelisted) - { - LogPrint("net", "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); - pfrom->fDisconnect = true; - return true; - } - - LOCK(pfrom->cs_inventory); - pfrom->fSendMempool = true; - } - - - else if (strCommand == NetMsgType::PING) - { - if (pfrom->nVersion > BIP0031_VERSION) - { - uint64_t nonce = 0; - vRecv >> nonce; - // Echo the message back with the nonce. This allows for two useful features: - // - // 1) A remote node can quickly check if the connection is operational - // 2) Remote nodes can measure the latency of the network thread. If this node - // is overloaded it won't respond to pings quickly and the remote node can - // avoid sending us more work, like chain download requests. - // - // The nonce stops the remote getting confused between different pings: without - // it, if the remote node sends a ping once per second and this node takes 5 - // seconds to respond to each, the 5th ping the remote sends would appear to - // return very quickly. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); - } - } - - - else if (strCommand == NetMsgType::PONG) - { - int64_t pingUsecEnd = nTimeReceived; - uint64_t nonce = 0; - size_t nAvail = vRecv.in_avail(); - bool bPingFinished = false; - std::string sProblem; - - if (nAvail >= sizeof(nonce)) { - vRecv >> nonce; - - // Only process pong message if there is an outstanding ping (old ping without nonce should never pong) - if (pfrom->nPingNonceSent != 0) { - if (nonce == pfrom->nPingNonceSent) { - // Matching pong received, this ping is no longer outstanding - bPingFinished = true; - int64_t pingUsecTime = pingUsecEnd - pfrom->nPingUsecStart; - if (pingUsecTime > 0) { - // Successful ping time measurement, replace previous - pfrom->nPingUsecTime = pingUsecTime; - pfrom->nMinPingUsecTime = std::min(pfrom->nMinPingUsecTime, pingUsecTime); - } else { - // This should never happen - sProblem = "Timing mishap"; - } - } else { - // Nonce mismatches are normal when pings are overlapping - sProblem = "Nonce mismatch"; - if (nonce == 0) { - // This is most likely a bug in another implementation somewhere; cancel this ping - bPingFinished = true; - sProblem = "Nonce zero"; - } - } - } else { - sProblem = "Unsolicited pong without ping"; - } - } else { - // This is most likely a bug in another implementation somewhere; cancel this ping - bPingFinished = true; - sProblem = "Short payload"; - } - - if (!(sProblem.empty())) { - LogPrint("net", "pong peer=%d: %s, %x expected, %x received, %u bytes\n", - pfrom->id, - sProblem, - pfrom->nPingNonceSent, - nonce, - nAvail); - } - if (bPingFinished) { - pfrom->nPingNonceSent = 0; - } - } - - - else if (strCommand == NetMsgType::FILTERLOAD) - { - CBloomFilter filter; - vRecv >> filter; - - if (!filter.IsWithinSizeConstraints()) - { - // There is no excuse for sending a too-large filter - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 100); - } - else - { - LOCK(pfrom->cs_filter); - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(filter); - pfrom->pfilter->UpdateEmptyFull(); - pfrom->fRelayTxes = true; - } - } - - - else if (strCommand == NetMsgType::FILTERADD) - { - vector<unsigned char> vData; - vRecv >> vData; - - // Nodes must NEVER send a data item > 520 bytes (the max size for a script data object, - // and thus, the maximum size any matched object can have) in a filteradd message - bool bad = false; - if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { - bad = true; - } else { - LOCK(pfrom->cs_filter); - if (pfrom->pfilter) { - pfrom->pfilter->insert(vData); - } else { - bad = true; - } - } - if (bad) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), 100); - } - } - - - else if (strCommand == NetMsgType::FILTERCLEAR) - { - LOCK(pfrom->cs_filter); - if (pfrom->GetLocalServices() & NODE_BLOOM) { - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(); - } - pfrom->fRelayTxes = true; - } - - - else if (strCommand == NetMsgType::REJECT) - { - if (fDebug) { - try { - string strMsg; unsigned char ccode; string strReason; - vRecv >> LIMITED_STRING(strMsg, CMessageHeader::COMMAND_SIZE) >> ccode >> LIMITED_STRING(strReason, MAX_REJECT_MESSAGE_LENGTH); - - ostringstream ss; - ss << strMsg << " code " << itostr(ccode) << ": " << strReason; - - if (strMsg == NetMsgType::BLOCK || strMsg == NetMsgType::TX) - { - uint256 hash; - vRecv >> hash; - ss << ": hash " << hash.ToString(); - } - LogPrint("net", "Reject %s\n", SanitizeString(ss.str())); - } catch (const std::ios_base::failure&) { - // Avoid feedback loops by preventing reject messages from triggering a new reject message. - LogPrint("net", "Unparseable reject message received\n"); - } - } - } - - else if (strCommand == NetMsgType::FEEFILTER) { - CAmount newFeeFilter = 0; - vRecv >> newFeeFilter; - if (MoneyRange(newFeeFilter)) { - { - LOCK(pfrom->cs_feeFilter); - pfrom->minFeeFilter = newFeeFilter; - } - LogPrint("net", "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); - } - } - - else if (strCommand == NetMsgType::NOTFOUND) { - // We do not care about the NOTFOUND message, but logging an Unknown Command - // message would be undesirable as we transmit it ourselves. - } - - else { - // Ignore unknown commands for extensibility - LogPrint("net", "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); - } - - - - return true; -} - -// requires LOCK(cs_vRecvMsg) -bool ProcessMessages(CNode* pfrom, CConnman& connman) +std::string CBlockFileInfo::ToString() const { - const CChainParams& chainparams = Params(); - unsigned int nMaxSendBufferSize = connman.GetSendBufferSize(); - //if (fDebug) - // LogPrintf("%s(%u messages)\n", __func__, pfrom->vRecvMsg.size()); - - // - // Message format - // (4) message start - // (12) command - // (4) size - // (4) checksum - // (x) data - // - bool fOk = true; - - if (!pfrom->vRecvGetData.empty()) - ProcessGetData(pfrom, chainparams.GetConsensus(), connman); - - // this maintains the order of responses - if (!pfrom->vRecvGetData.empty()) return fOk; - - std::deque<CNetMessage>::iterator it = pfrom->vRecvMsg.begin(); - while (!pfrom->fDisconnect && it != pfrom->vRecvMsg.end()) { - // Don't bother if send buffer is too full to respond anyway - if (pfrom->nSendSize >= nMaxSendBufferSize) - break; - - // get next message - CNetMessage& msg = *it; - - //if (fDebug) - // LogPrintf("%s(message %u msgsz, %u bytes, complete:%s)\n", __func__, - // msg.hdr.nMessageSize, msg.vRecv.size(), - // msg.complete() ? "Y" : "N"); - - // end, if an incomplete message is found - if (!msg.complete()) - break; - - // at this point, any failure means we can delete the current message - it++; - - // Scan for message start - if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); - fOk = false; - break; - } - - // Read header - CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid(chainparams.MessageStart())) - { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); - continue; - } - string strCommand = hdr.GetCommand(); - - // Message size - unsigned int nMessageSize = hdr.nMessageSize; - - // Checksum - CDataStream& vRecv = msg.vRecv; - const uint256& hash = msg.GetMessageHash(); - if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) - { - LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, - SanitizeString(strCommand), nMessageSize, - HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), - HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); - continue; - } - - // Process message - bool fRet = false; - try - { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman); - boost::this_thread::interruption_point(); - } - catch (const std::ios_base::failure& e) - { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, string("error parsing message"))); - if (strstr(e.what(), "end of data")) - { - // Allow exceptions from under-length message on vRecv - LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "size too large")) - { - // Allow exceptions from over-long size - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "non-canonical ReadCompactSize()")) - { - // Allow exceptions from non-canonical encoding - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else - { - PrintExceptionContinue(&e, "ProcessMessages()"); - } - } - catch (const boost::thread_interrupted&) { - throw; - } - catch (const std::exception& e) { - PrintExceptionContinue(&e, "ProcessMessages()"); - } catch (...) { - PrintExceptionContinue(NULL, "ProcessMessages()"); - } - - if (!fRet) - LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); - - break; - } - - // In case the connection got shut down, its receive buffer was wiped - if (!pfrom->fDisconnect) - pfrom->vRecvMsg.erase(pfrom->vRecvMsg.begin(), it); - - return fOk; + return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst), DateTimeStrFormat("%Y-%m-%d", nTimeLast)); } -class CompareInvMempoolOrder -{ - CTxMemPool *mp; -public: - CompareInvMempoolOrder(CTxMemPool *_mempool) - { - mp = _mempool; - } - - bool operator()(std::set<uint256>::iterator a, std::set<uint256>::iterator b) - { - /* As std::make_heap produces a max-heap, we want the entries with the - * fewest ancestors/highest fee to sort later. */ - return mp->CompareDepthAndScore(*b, *a); - } -}; - -bool SendMessages(CNode* pto, CConnman& connman) -{ - const Consensus::Params& consensusParams = Params().GetConsensus(); - { - // Don't send anything until we get its version message - if (pto->nVersion == 0 || pto->fDisconnect) - return true; - - // If we get here, the outgoing message serialization version is set and can't change. - CNetMsgMaker msgMaker(pto->GetSendVersion()); - - // - // Message: ping - // - bool pingSend = false; - if (pto->fPingQueued) { - // RPC ping request by user - pingSend = true; - } - if (pto->nPingNonceSent == 0 && pto->nPingUsecStart + PING_INTERVAL * 1000000 < GetTimeMicros()) { - // Ping automatically sent as a latency probe & keepalive. - pingSend = true; - } - if (pingSend) { - uint64_t nonce = 0; - while (nonce == 0) { - GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); - } - pto->fPingQueued = false; - pto->nPingUsecStart = GetTimeMicros(); - if (pto->nVersion > BIP0031_VERSION) { - pto->nPingNonceSent = nonce; - connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); - } else { - // Peer is too old to support ping command with nonce, pong will never arrive. - pto->nPingNonceSent = 0; - connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); - } - } - - TRY_LOCK(cs_main, lockMain); // Acquire cs_main for IsInitialBlockDownload() and CNodeState() - if (!lockMain) - return true; - - CNodeState &state = *State(pto->GetId()); - - BOOST_FOREACH(const CBlockReject& reject, state.rejects) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::REJECT, (string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock)); - state.rejects.clear(); - - if (state.fShouldBan) { - state.fShouldBan = false; - if (pto->fWhitelisted) - LogPrintf("Warning: not punishing whitelisted peer %s!\n", pto->addr.ToString()); - else { - pto->fDisconnect = true; - if (pto->addr.IsLocal()) - LogPrintf("Warning: not banning local peer %s!\n", pto->addr.ToString()); - else - { - connman.Ban(pto->addr, BanReasonNodeMisbehaving); - } - return true; - } - } - - // Address refresh broadcast - int64_t nNow = GetTimeMicros(); - if (!IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { - AdvertiseLocal(pto); - pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); - } - - // - // Message: addr - // - if (pto->nNextAddrSend < nNow) { - pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); - vector<CAddress> vAddr; - vAddr.reserve(pto->vAddrToSend.size()); - BOOST_FOREACH(const CAddress& addr, pto->vAddrToSend) - { - if (!pto->addrKnown.contains(addr.GetKey())) - { - pto->addrKnown.insert(addr.GetKey()); - vAddr.push_back(addr); - // receiver rejects addr messages larger than 1000 - if (vAddr.size() >= 1000) - { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); - vAddr.clear(); - } - } - } - pto->vAddrToSend.clear(); - if (!vAddr.empty()) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); - // we only send the big addr message once - if (pto->vAddrToSend.capacity() > 40) - pto->vAddrToSend.shrink_to_fit(); - } - - // Start block sync - if (pindexBestHeader == NULL) - pindexBestHeader = chainActive.Tip(); - bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); // Download if this is a nice peer, or we have no nice peers and this one might do. - if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { - // Only actively request headers from a single peer, unless we're close to today. - if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { - state.fSyncStarted = true; - nSyncStarted++; - const CBlockIndex *pindexStart = pindexBestHeader; - /* If possible, start at the block preceding the currently - best known header. This ensures that we always get a - non-empty list of headers back as long as the peer - is up-to-date. With a non-empty response, we can initialise - the peer's known best block. This wouldn't be possible - if we requested starting at pindexBestHeader and - got back an empty response. */ - if (pindexStart->pprev) - pindexStart = pindexStart->pprev; - LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); - } - } - - // Resend wallet transactions that haven't gotten in a block yet - // Except during reindex, importing and IBD, when old wallet - // transactions become unconfirmed and spams other nodes. - if (!fReindex && !fImporting && !IsInitialBlockDownload()) - { - GetMainSignals().Broadcast(nTimeBestReceived, &connman); - } - - // - // Try sending block announcements via headers - // - { - // If we have less than MAX_BLOCKS_TO_ANNOUNCE in our - // list of block hashes we're relaying, and our peer wants - // headers announcements, then find the first header - // not yet known to our peer but would connect, and send. - // If no header would connect, or if we have too many - // blocks, or if the peer doesn't want headers, just - // add all to the inv queue. - LOCK(pto->cs_inventory); - vector<CBlock> vHeaders; - bool fRevertToInv = ((!state.fPreferHeaders && - (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || - pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); - CBlockIndex *pBestIndex = NULL; // last header queued for delivery - ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date - - if (!fRevertToInv) { - bool fFoundStartingHeader = false; - // Try to find first header that our peer doesn't have, and - // then send all headers past that one. If we come across any - // headers that aren't on chainActive, give up. - BOOST_FOREACH(const uint256 &hash, pto->vBlockHashesToAnnounce) { - BlockMap::iterator mi = mapBlockIndex.find(hash); - assert(mi != mapBlockIndex.end()); - CBlockIndex *pindex = mi->second; - if (chainActive[pindex->nHeight] != pindex) { - // Bail out if we reorged away from this block - fRevertToInv = true; - break; - } - if (pBestIndex != NULL && pindex->pprev != pBestIndex) { - // This means that the list of blocks to announce don't - // connect to each other. - // This shouldn't really be possible to hit during - // regular operation (because reorgs should take us to - // a chain that has some block not on the prior chain, - // which should be caught by the prior check), but one - // way this could happen is by using invalidateblock / - // reconsiderblock repeatedly on the tip, causing it to - // be added multiple times to vBlockHashesToAnnounce. - // Robustly deal with this rare situation by reverting - // to an inv. - fRevertToInv = true; - break; - } - pBestIndex = pindex; - if (fFoundStartingHeader) { - // add this to the headers message - vHeaders.push_back(pindex->GetBlockHeader()); - } else if (PeerHasHeader(&state, pindex)) { - continue; // keep looking for the first new block - } else if (pindex->pprev == NULL || PeerHasHeader(&state, pindex->pprev)) { - // Peer doesn't have this header but they do have the prior one. - // Start sending headers. - fFoundStartingHeader = true; - vHeaders.push_back(pindex->GetBlockHeader()); - } else { - // Peer doesn't have this header or the prior one -- nothing will - // connect, so bail out. - fRevertToInv = true; - break; - } - } - } - if (!fRevertToInv && !vHeaders.empty()) { - if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { - // We only send up to 1 block as header-and-ids, as otherwise - // probably means we're doing an initial-ish-sync or they're slow - LogPrint("net", "%s sending header-and-ids %s to peer %d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); - //TODO: Shouldn't need to reload block from disk, but requires refactor - CBlock block; - assert(ReadBlockFromDisk(block, pBestIndex, consensusParams)); - CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); - int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; - connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); - state.pindexBestHeaderSent = pBestIndex; - } else if (state.fPreferHeaders) { - if (vHeaders.size() > 1) { - LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, - vHeaders.size(), - vHeaders.front().GetHash().ToString(), - vHeaders.back().GetHash().ToString(), pto->id); - } else { - LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); - } - connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); - state.pindexBestHeaderSent = pBestIndex; - } else - fRevertToInv = true; - } - if (fRevertToInv) { - // If falling back to using an inv, just try to inv the tip. - // The last entry in vBlockHashesToAnnounce was our tip at some point - // in the past. - if (!pto->vBlockHashesToAnnounce.empty()) { - const uint256 &hashToAnnounce = pto->vBlockHashesToAnnounce.back(); - BlockMap::iterator mi = mapBlockIndex.find(hashToAnnounce); - assert(mi != mapBlockIndex.end()); - CBlockIndex *pindex = mi->second; - - // Warn if we're announcing a block that is not on the main chain. - // This should be very rare and could be optimized out. - // Just log for now. - if (chainActive[pindex->nHeight] != pindex) { - LogPrint("net", "Announcing block %s not on main chain (tip=%s)\n", - hashToAnnounce.ToString(), chainActive.Tip()->GetBlockHash().ToString()); - } - - // If the peer's chain has this block, don't inv it back. - if (!PeerHasHeader(&state, pindex)) { - pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); - LogPrint("net", "%s: sending inv peer=%d hash=%s\n", __func__, - pto->id, hashToAnnounce.ToString()); - } - } - } - pto->vBlockHashesToAnnounce.clear(); - } - - // - // Message: inventory - // - vector<CInv> vInv; - { - LOCK(pto->cs_inventory); - vInv.reserve(std::max<size_t>(pto->vInventoryBlockToSend.size(), INVENTORY_BROADCAST_MAX)); - - // Add blocks - BOOST_FOREACH(const uint256& hash, pto->vInventoryBlockToSend) { - vInv.push_back(CInv(MSG_BLOCK, hash)); - if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); - vInv.clear(); - } - } - pto->vInventoryBlockToSend.clear(); - - // Check whether periodic sends should happen - bool fSendTrickle = pto->fWhitelisted; - if (pto->nNextInvSend < nNow) { - fSendTrickle = true; - // Use half the delay for outbound peers, as there is less privacy concern for them. - pto->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> !pto->fInbound); - } - - // Time to send but the peer has requested we not relay transactions. - if (fSendTrickle) { - LOCK(pto->cs_filter); - if (!pto->fRelayTxes) pto->setInventoryTxToSend.clear(); - } - - // Respond to BIP35 mempool requests - if (fSendTrickle && pto->fSendMempool) { - auto vtxinfo = mempool.infoAll(); - pto->fSendMempool = false; - CAmount filterrate = 0; - { - LOCK(pto->cs_feeFilter); - filterrate = pto->minFeeFilter; - } - - LOCK(pto->cs_filter); - - for (const auto& txinfo : vtxinfo) { - const uint256& hash = txinfo.tx->GetHash(); - CInv inv(MSG_TX, hash); - pto->setInventoryTxToSend.erase(hash); - if (filterrate) { - if (txinfo.feeRate.GetFeePerK() < filterrate) - continue; - } - if (pto->pfilter) { - if (!pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; - } - pto->filterInventoryKnown.insert(hash); - vInv.push_back(inv); - if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); - vInv.clear(); - } - } - pto->timeLastMempoolReq = GetTime(); - } - - // Determine transactions to relay - if (fSendTrickle) { - // Produce a vector with all candidates for sending - vector<std::set<uint256>::iterator> vInvTx; - vInvTx.reserve(pto->setInventoryTxToSend.size()); - for (std::set<uint256>::iterator it = pto->setInventoryTxToSend.begin(); it != pto->setInventoryTxToSend.end(); it++) { - vInvTx.push_back(it); - } - CAmount filterrate = 0; - { - LOCK(pto->cs_feeFilter); - filterrate = pto->minFeeFilter; - } - // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. - // A heap is used so that not all items need sorting if only a few are being sent. - CompareInvMempoolOrder compareInvMempoolOrder(&mempool); - std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); - // No reason to drain out at many times the network's capacity, - // especially since we have many peers and some will draw much shorter delays. - unsigned int nRelayedTransactions = 0; - LOCK(pto->cs_filter); - while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { - // Fetch the top element from the heap - std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); - std::set<uint256>::iterator it = vInvTx.back(); - vInvTx.pop_back(); - uint256 hash = *it; - // Remove it from the to-be-sent set - pto->setInventoryTxToSend.erase(it); - // Check if not in the filter already - if (pto->filterInventoryKnown.contains(hash)) { - continue; - } - // Not in the mempool anymore? don't bother sending it. - auto txinfo = mempool.info(hash); - if (!txinfo.tx) { - continue; - } - if (filterrate && txinfo.feeRate.GetFeePerK() < filterrate) { - continue; - } - if (pto->pfilter && !pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; - // Send - vInv.push_back(CInv(MSG_TX, hash)); - nRelayedTransactions++; - { - // Expire old relay messages - while (!vRelayExpiration.empty() && vRelayExpiration.front().first < nNow) - { - mapRelay.erase(vRelayExpiration.front().second); - vRelayExpiration.pop_front(); - } - - auto ret = mapRelay.insert(std::make_pair(hash, std::move(txinfo.tx))); - if (ret.second) { - vRelayExpiration.push_back(std::make_pair(nNow + 15 * 60 * 1000000, ret.first)); - } - } - if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); - vInv.clear(); - } - pto->filterInventoryKnown.insert(hash); - } - } - } - if (!vInv.empty()) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); - - // Detect whether we're stalling - nNow = GetTimeMicros(); - if (state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) { - // Stalling only triggers when the block download window cannot move. During normal steady state, - // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection - // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); - pto->fDisconnect = true; - return true; - } - // In case there is a block that has been in flight from this peer for 2 + 0.5 * N times the block interval - // (with N the number of peers from which we're downloading validated blocks), disconnect due to timeout. - // We compensate for other peers to prevent killing off peers due to our own downstream link - // being saturated. We only count validated in-flight blocks so peers can't advertise non-existing block hashes - // to unreasonably increase our timeout. - if (state.vBlocksInFlight.size() > 0) { - QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); - int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); - if (nNow > state.nDownloadingSince + consensusParams.nPowTargetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); - pto->fDisconnect = true; - return true; - } - } - - // - // Message: getdata (blocks) - // - vector<CInv> vGetData; - if (!pto->fClient && (fFetch || !IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - vector<CBlockIndex*> vToDownload; - NodeId staller = -1; - FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); - BOOST_FOREACH(CBlockIndex *pindex, vToDownload) { - uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); - vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); - LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), - pindex->nHeight, pto->id); - } - if (state.nBlocksInFlight == 0 && staller != -1) { - if (State(staller)->nStallingSince == 0) { - State(staller)->nStallingSince = nNow; - LogPrint("net", "Stall started peer=%d\n", staller); - } - } - } - - // - // Message: getdata (non-blocks) - // - while (!pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) - { - const CInv& inv = (*pto->mapAskFor.begin()).second; - if (!AlreadyHave(inv)) - { - if (fDebug) - LogPrint("net", "Requesting %s peer=%d\n", inv.ToString(), pto->id); - vGetData.push_back(inv); - if (vGetData.size() >= 1000) - { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - vGetData.clear(); - } - } else { - //If we're not going to ask, don't expect a response. - pto->setAskFor.erase(inv.hash); - } - pto->mapAskFor.erase(pto->mapAskFor.begin()); - } - if (!vGetData.empty()) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - - // - // Message: feefilter - // - // We don't want white listed peers to filter txs to us if we have -whitelistforcerelay - if (pto->nVersion >= FEEFILTER_VERSION && GetBoolArg("-feefilter", DEFAULT_FEEFILTER) && - !(pto->fWhitelisted && GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY))) { - CAmount currentFilter = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); - int64_t timeNow = GetTimeMicros(); - if (timeNow > pto->nextSendTimeFeeFilter) { - CAmount filterToSend = filterRounder.round(currentFilter); - if (filterToSend != pto->lastSentFeeFilter) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); - pto->lastSentFeeFilter = filterToSend; - } - pto->nextSendTimeFeeFilter = PoissonNextSend(timeNow, AVG_FEEFILTER_BROADCAST_INTERVAL); - } - // If the fee filter has changed substantially and it's still more than MAX_FEEFILTER_CHANGE_DELAY - // until scheduled broadcast, then move the broadcast to within MAX_FEEFILTER_CHANGE_DELAY. - else if (timeNow + MAX_FEEFILTER_CHANGE_DELAY * 1000000 < pto->nextSendTimeFeeFilter && - (currentFilter < 3 * pto->lastSentFeeFilter / 4 || currentFilter > 4 * pto->lastSentFeeFilter / 3)) { - pto->nextSendTimeFeeFilter = timeNow + GetRandInt(MAX_FEEFILTER_CHANGE_DELAY) * 1000000; - } - } - } - return true; -} - - std::string CBlockFileInfo::ToString() const { - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst), DateTimeStrFormat("%Y-%m-%d", nTimeLast)); - } - ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos) { LOCK(cs_main); @@ -7069,10 +4055,9 @@ bool LoadMempool(void) file >> num; double prioritydummy = 0; while (num--) { - CTransaction tx; int64_t nTime; int64_t nFeeDelta; - file >> tx; + CTransaction tx(deserialize, file); file >> nTime; file >> nFeeDelta; @@ -7165,9 +4150,5 @@ public: for (; it1 != mapBlockIndex.end(); it1++) delete (*it1).second; mapBlockIndex.clear(); - - // orphan transactions - mapOrphanTransactions.clear(); - mapOrphanTransactionsByPrev.clear(); } } instance_of_cmaincleanup; diff --git a/src/main.h b/src/validation.h index 6b2668e568..f0f54786c7 100644 --- a/src/main.h +++ b/src/validation.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_MAIN_H -#define BITCOIN_MAIN_H +#ifndef BITCOIN_VALIDATION_H +#define BITCOIN_VALIDATION_H #if defined(HAVE_CONFIG_H) #include "config/bitcoin-config.h" @@ -13,10 +13,9 @@ #include "amount.h" #include "chain.h" #include "coins.h" -#include "net.h" +#include "protocol.h" // For CMessageHeader::MessageStartChars #include "script/script_error.h" #include "sync.h" -#include "validationinterface.h" #include "versionbits.h" #include <algorithm> @@ -28,7 +27,10 @@ #include <utility> #include <vector> +#include <atomic> + #include <boost/unordered_map.hpp> +#include <boost/filesystem/path.hpp> class CBlockIndex; class CBlockTreeDB; @@ -135,7 +137,6 @@ static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; -static const bool DEFAULT_TESTSAFEMODE = false; /** Default for -mempoolreplacement */ static const bool DEFAULT_ENABLE_REPLACEMENT = true; /** Default for using fee filter */ @@ -227,11 +228,10 @@ static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; * * @param[in] pblock The block we want to process. * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. - * @param[out] dbp The already known disk position of pblock, or NULL if not yet stored. * @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call * @return True if state.IsValid() */ -bool ProcessNewBlock(const CChainParams& chainparams, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp, bool* fNewBlock); +bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock); /** * Process incoming block headers. @@ -274,9 +274,9 @@ bool IsInitialBlockDownload(); */ std::string GetWarnings(const std::string& strFor); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ -bool GetTransaction(const uint256 &hash, CTransaction &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); +bool GetTransaction(const uint256 &hash, CTransactionRef &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); /** Find the best known block, and make it the tip of the block chain */ -bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, const CBlock* pblock = NULL); +bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** @@ -560,46 +560,4 @@ void DumpMempool(); /** Load the mempool from disk. */ bool LoadMempool(); -// The following things handle network-processing logic -// (and should be moved to a separate file) - -/** Register with a network node to receive its signals */ -void RegisterNodeSignals(CNodeSignals& nodeSignals); -/** Unregister a network node */ -void UnregisterNodeSignals(CNodeSignals& nodeSignals); - -class PeerLogicValidation : public CValidationInterface { -private: - CConnman* connman; - -public: - PeerLogicValidation(CConnman* connmanIn); - - virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock); - virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); - virtual void BlockChecked(const CBlock& block, const CValidationState& state); -}; - -struct CNodeStateStats { - int nMisbehavior; - int nSyncHeight; - int nCommonHeight; - std::vector<int> vHeightInFlight; -}; - -/** Get statistics from node state */ -bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); -/** Increase a node's misbehavior score. */ -void Misbehaving(NodeId nodeid, int howmuch); - -/** Process protocol messages received from a given node */ -bool ProcessMessages(CNode* pfrom, CConnman& connman); -/** - * Send queued protocol messages to be sent to a give node. - * - * @param[in] pto The node which we are sending messages to. - * @param[in] connman The connection manager for that node. - */ -bool SendMessages(CNode* pto, CConnman& connman); - -#endif // BITCOIN_MAIN_H +#endif // BITCOIN_VALIDATION_H diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index bb5337c4ad..008a4ece19 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -6,7 +6,7 @@ #include "chain.h" #include "rpc/server.h" #include "init.h" -#include "main.h" +#include "validation.h" #include "script/script.h" #include "script/standard.h" #include "sync.h" @@ -267,11 +267,11 @@ UniValue importprunedfunds(const JSONRPCRequest& request) "2. \"txoutproof\" (string, required) The hex output from gettxoutproof that contains the transaction\n" ); - CTransaction tx; + CMutableTransaction tx; if (!DecodeHexTx(tx, request.params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); uint256 hashTx = tx.GetHash(); - CWalletTx wtx(pwalletMain,tx); + CWalletTx wtx(pwalletMain, MakeTransactionRef(std::move(tx))); CDataStream ssMB(ParseHexV(request.params[1], "proof"), SER_NETWORK, PROTOCOL_VERSION); CMerkleBlock merkleBlock; @@ -304,7 +304,7 @@ UniValue importprunedfunds(const JSONRPCRequest& request) LOCK2(cs_main, pwalletMain->cs_wallet); - if (pwalletMain->IsMine(tx)) { + if (pwalletMain->IsMine(wtx)) { pwalletMain->AddToWallet(wtx, false); return NullUniValue; } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 5a22e0278d..5a4fcc743c 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -9,7 +9,7 @@ #include "consensus/validation.h" #include "core_io.h" #include "init.h" -#include "main.h" +#include "validation.h" #include "net.h" #include "policy/rbf.h" #include "rpc/server.h" @@ -362,7 +362,7 @@ static void SendMoney(const CTxDestination &address, CAmount nValue, bool fSubtr CRecipient recipient = {scriptPubKey, nValue, fSubtractFeeFromAmount}; vecSend.push_back(recipient); if (!pwalletMain->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError)) { - if (!fSubtractFeeFromAmount && nValue + nFeeRequired > pwalletMain->GetBalance()) + if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); throw JSONRPCError(RPC_WALLET_ERROR, strError); } @@ -583,10 +583,10 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) { const CWalletTx& wtx = (*it).second; - if (wtx.IsCoinBase() || !CheckFinalTx(wtx)) + if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) if (txout.scriptPubKey == scriptPubKey) if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; @@ -637,10 +637,10 @@ UniValue getreceivedbyaccount(const JSONRPCRequest& request) for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) { const CWalletTx& wtx = (*it).second; - if (wtx.IsCoinBase() || !CheckFinalTx(wtx)) + if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { CTxDestination address; if (ExtractDestination(txout.scriptPubKey, address) && IsMine(*pwalletMain, address) && setAddress.count(address)) @@ -1149,14 +1149,14 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) { const CWalletTx& wtx = (*it).second; - if (wtx.IsCoinBase() || !CheckFinalTx(wtx)) + if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; int nDepth = wtx.GetDepthInMainChain(); if (nDepth < nMinDepth) continue; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { CTxDestination address; if (!ExtractDestination(txout.scriptPubKey, address)) @@ -1780,7 +1780,7 @@ UniValue gettransaction(const JSONRPCRequest& request) CAmount nCredit = wtx.GetCredit(filter); CAmount nDebit = wtx.GetDebit(filter); CAmount nNet = nCredit - nDebit; - CAmount nFee = (wtx.IsFromMe(filter) ? wtx.GetValueOut() - nDebit : 0); + CAmount nFee = (wtx.IsFromMe(filter) ? wtx.tx->GetValueOut() - nDebit : 0); entry.push_back(Pair("amount", ValueFromAmount(nNet - nFee))); if (wtx.IsFromMe(filter)) @@ -1792,7 +1792,7 @@ UniValue gettransaction(const JSONRPCRequest& request) ListTransactions(wtx, "*", 0, false, details, filter); entry.push_back(Pair("details", details)); - string strHex = EncodeHexTx(static_cast<CTransaction>(wtx)); + string strHex = EncodeHexTx(static_cast<CTransaction>(wtx), RPCSerializationFlags()); entry.push_back(Pair("hex", strHex)); return entry; @@ -2420,7 +2420,7 @@ UniValue listunspent(const JSONRPCRequest& request) continue; CTxDestination address; - const CScript& scriptPubKey = out.tx->vout[out.i].scriptPubKey; + const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); if (setAddress.size() && (!fValidAddress || !setAddress.count(address))) @@ -2445,7 +2445,7 @@ UniValue listunspent(const JSONRPCRequest& request) } entry.push_back(Pair("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end()))); - entry.push_back(Pair("amount", ValueFromAmount(out.tx->vout[out.i].nValue))); + entry.push_back(Pair("amount", ValueFromAmount(out.tx->tx->vout[out.i].nValue))); entry.push_back(Pair("confirmations", out.nDepth)); entry.push_back(Pair("spendable", out.fSpendable)); entry.push_back(Pair("solvable", out.fSolvable)); @@ -2557,17 +2557,16 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) } // parse hex string from parameter - CTransaction origTx; - if (!DecodeHexTx(origTx, request.params[0].get_str(), true)) + CMutableTransaction tx; + if (!DecodeHexTx(tx, request.params[0].get_str(), true)) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); - if (origTx.vout.size() == 0) + if (tx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); - if (changePosition != -1 && (changePosition < 0 || (unsigned int)changePosition > origTx.vout.size())) + if (changePosition != -1 && (changePosition < 0 || (unsigned int)changePosition > tx.vout.size())) throw JSONRPCError(RPC_INVALID_PARAMETER, "changePosition out of bounds"); - CMutableTransaction tx(origTx); CAmount nFeeOut; string strFailReason; diff --git a/src/wallet/test/accounting_tests.cpp b/src/wallet/test/accounting_tests.cpp index a833be13d0..eb14d176bd 100644 --- a/src/wallet/test/accounting_tests.cpp +++ b/src/wallet/test/accounting_tests.cpp @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) { CMutableTransaction tx(wtx); --tx.nLockTime; // Just to change the hash :) - *static_cast<CTransaction*>(&wtx) = CTransaction(tx); + wtx.SetTx(MakeTransactionRef(std::move(tx))); } pwalletMain->AddToWallet(wtx); vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]); @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) { CMutableTransaction tx(wtx); --tx.nLockTime; // Just to change the hash :) - *static_cast<CTransaction*>(&wtx) = CTransaction(tx); + wtx.SetTx(MakeTransactionRef(std::move(tx))); } pwalletMain->AddToWallet(wtx); vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]); diff --git a/src/wallet/test/crypto_tests.cpp b/src/wallet/test/crypto_tests.cpp index ce35c53c48..c64c76244e 100644 --- a/src/wallet/test/crypto_tests.cpp +++ b/src/wallet/test/crypto_tests.cpp @@ -42,15 +42,19 @@ bool OldEncrypt(const CKeyingMaterial& vchPlaintext, std::vector<unsigned char> int nCLen = nLen + AES_BLOCK_SIZE, nFLen = 0; vchCiphertext = std::vector<unsigned char> (nCLen); - EVP_CIPHER_CTX ctx; + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + + if (!ctx) return false; bool fOk = true; - EVP_CIPHER_CTX_init(&ctx); - if (fOk) fOk = EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; - if (fOk) fOk = EVP_EncryptUpdate(&ctx, &vchCiphertext[0], &nCLen, &vchPlaintext[0], nLen) != 0; - if (fOk) fOk = EVP_EncryptFinal_ex(&ctx, (&vchCiphertext[0]) + nCLen, &nFLen) != 0; - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_init(ctx); + if (fOk) fOk = EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; + if (fOk) fOk = EVP_EncryptUpdate(ctx, &vchCiphertext[0], &nCLen, &vchPlaintext[0], nLen) != 0; + if (fOk) fOk = EVP_EncryptFinal_ex(ctx, (&vchCiphertext[0]) + nCLen, &nFLen) != 0; + EVP_CIPHER_CTX_cleanup(ctx); + + EVP_CIPHER_CTX_free(ctx); if (!fOk) return false; @@ -66,15 +70,19 @@ bool OldDecrypt(const std::vector<unsigned char>& vchCiphertext, CKeyingMaterial vchPlaintext = CKeyingMaterial(nPLen); - EVP_CIPHER_CTX ctx; + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + + if (!ctx) return false; bool fOk = true; - EVP_CIPHER_CTX_init(&ctx); - if (fOk) fOk = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; - if (fOk) fOk = EVP_DecryptUpdate(&ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen) != 0; - if (fOk) fOk = EVP_DecryptFinal_ex(&ctx, (&vchPlaintext[0]) + nPLen, &nFLen) != 0; - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_init(ctx); + if (fOk) fOk = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; + if (fOk) fOk = EVP_DecryptUpdate(ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen) != 0; + if (fOk) fOk = EVP_DecryptFinal_ex(ctx, (&vchPlaintext[0]) + nPLen, &nFLen) != 0; + EVP_CIPHER_CTX_cleanup(ctx); + + EVP_CIPHER_CTX_free(ctx); if (!fOk) return false; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index ce42fad5de..9465d88067 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -42,7 +42,7 @@ static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = fa // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() tx.vin.resize(1); } - CWalletTx* wtx = new CWalletTx(&wallet, tx); + CWalletTx* wtx = new CWalletTx(&wallet, MakeTransactionRef(std::move(tx))); if (fIsFromMe) { wtx->fDebitCached = true; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f1d8020e75..c4051d2ac6 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -13,7 +13,7 @@ #include "consensus/validation.h" #include "key.h" #include "keystore.h" -#include "main.h" +#include "validation.h" #include "net.h" #include "policy/policy.h" #include "primitives/block.h" @@ -75,7 +75,7 @@ struct CompareValueOnly std::string COutput::ToString() const { - return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->vout[i].nValue)); + return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const @@ -400,7 +400,7 @@ set<uint256> CWallet::GetConflicts(const uint256& txid) const std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapTxSpends.count(txin.prevout) <= 1) continue; // No conflict if zero or one spends @@ -552,7 +552,7 @@ void CWallet::AddToSpends(const uint256& wtxid) if (thisTx.IsCoinBase()) // Coinbases don't spend anything! return; - BOOST_FOREACH(const CTxIn& txin, thisTx.vin) + BOOST_FOREACH(const CTxIn& txin, thisTx.tx->vin) AddToSpends(txin.prevout, wtxid); } @@ -795,7 +795,7 @@ bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bFo for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end() && account.vchPubKey.IsValid(); ++it) - BOOST_FOREACH(const CTxOut& txout, (*it).second.vout) + BOOST_FOREACH(const CTxOut& txout, (*it).second.tx->vout) if (txout.scriptPubKey == scriptPubKey) { bForceNew = true; break; @@ -954,7 +954,7 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) wtx.BindWallet(this); wtxOrdered.insert(make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); AddToSpends(hash); - BOOST_FOREACH(const CTxIn& txin, wtx.vin) { + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapWallet.count(txin.prevout.hash)) { CWalletTx& prevtx = mapWallet[txin.prevout.hash]; if (prevtx.nIndex == -1 && !prevtx.hashUnset()) { @@ -993,7 +993,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex if (fExisted && !fUpdate) return false; if (fExisted || IsMine(tx) || IsFromMe(tx)) { - CWalletTx wtx(this,tx); + CWalletTx wtx(this, MakeTransactionRef(tx)); // Get merkle branch if transaction was found in a block if (posInBlock != -1) @@ -1009,8 +1009,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK2(cs_main, cs_wallet); - // Do not flush the wallet here for performance reasons - CWalletDB walletdb(strWalletFile, "r+", false); + CWalletDB walletdb(strWalletFile, "r+"); std::set<uint256> todo; std::set<uint256> done; @@ -1052,7 +1051,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) } // If a transaction changes 'conflicted' state, that changes the balance // available of the outputs it spends. So force those to be recomputed - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapWallet.count(txin.prevout.hash)) mapWallet[txin.prevout.hash].MarkDirty(); @@ -1113,7 +1112,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } // If a transaction changes 'conflicted' state, that changes the balance // available of the outputs it spends. So force those to be recomputed - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapWallet.count(txin.prevout.hash)) mapWallet[txin.prevout.hash].MarkDirty(); @@ -1148,8 +1147,8 @@ isminetype CWallet::IsMine(const CTxIn &txin) const if (mi != mapWallet.end()) { const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.vout.size()) - return IsMine(prev.vout[txin.prevout.n]); + if (txin.prevout.n < prev.tx->vout.size()) + return IsMine(prev.tx->vout[txin.prevout.n]); } } return ISMINE_NO; @@ -1163,9 +1162,9 @@ CAmount CWallet::GetDebit(const CTxIn &txin, const isminefilter& filter) const if (mi != mapWallet.end()) { const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.vout.size()) - if (IsMine(prev.vout[txin.prevout.n]) & filter) - return prev.vout[txin.prevout.n].nValue; + if (txin.prevout.n < prev.tx->vout.size()) + if (IsMine(prev.tx->vout[txin.prevout.n]) & filter) + return prev.tx->vout[txin.prevout.n].nValue; } } return 0; @@ -1380,14 +1379,14 @@ void CWalletTx::GetAmounts(list<COutputEntry>& listReceived, CAmount nDebit = GetDebit(filter); if (nDebit > 0) // debit>0 means we signed/sent this transaction { - CAmount nValueOut = GetValueOut(); + CAmount nValueOut = tx->GetValueOut(); nFee = nDebit - nValueOut; } // Sent/received. - for (unsigned int i = 0; i < vout.size(); ++i) + for (unsigned int i = 0; i < tx->vout.size(); ++i) { - const CTxOut& txout = vout[i]; + const CTxOut& txout = tx->vout[i]; isminetype fIsMine = pwallet->IsMine(txout); // Only need to handle txouts if AT LEAST one of these is true: // 1) they debit from us (sent) @@ -1542,9 +1541,11 @@ void CWallet::ReacceptWalletTransactions() bool CWalletTx::RelayWalletTransaction(CConnman* connman) { assert(pwallet->GetBroadcastTransactions()); - if (!IsCoinBase()) + if (!IsCoinBase() && !isAbandoned() && GetDepthInMainChain() == 0) { - if (GetDepthInMainChain() == 0 && !isAbandoned() && InMempool()) { + CValidationState state; + /* GetDepthInMainChain already catches known conflicts. */ + if (InMempool() || AcceptToMemoryPool(maxTxFee, state)) { LogPrintf("Relaying wtx %s\n", GetHash().ToString()); if (connman) { CInv inv(MSG_TX, GetHash()); @@ -1573,7 +1574,7 @@ set<uint256> CWalletTx::GetConflicts() const CAmount CWalletTx::GetDebit(const isminefilter& filter) const { - if (vin.empty()) + if (tx->vin.empty()) return 0; CAmount debit = 0; @@ -1663,11 +1664,11 @@ CAmount CWalletTx::GetAvailableCredit(bool fUseCache) const CAmount nCredit = 0; uint256 hashTx = GetHash(); - for (unsigned int i = 0; i < vout.size(); i++) + for (unsigned int i = 0; i < tx->vout.size(); i++) { if (!pwallet->IsSpent(hashTx, i)) { - const CTxOut &txout = vout[i]; + const CTxOut &txout = tx->vout[i]; nCredit += pwallet->GetCredit(txout, ISMINE_SPENDABLE); if (!MoneyRange(nCredit)) throw std::runtime_error("CWalletTx::GetAvailableCredit() : value out of range"); @@ -1706,11 +1707,11 @@ CAmount CWalletTx::GetAvailableWatchOnlyCredit(const bool& fUseCache) const return nAvailableWatchCreditCached; CAmount nCredit = 0; - for (unsigned int i = 0; i < vout.size(); i++) + for (unsigned int i = 0; i < tx->vout.size(); i++) { if (!pwallet->IsSpent(GetHash(), i)) { - const CTxOut &txout = vout[i]; + const CTxOut &txout = tx->vout[i]; nCredit += pwallet->GetCredit(txout, ISMINE_WATCH_ONLY); if (!MoneyRange(nCredit)) throw std::runtime_error("CWalletTx::GetAvailableCredit() : value out of range"); @@ -1758,23 +1759,23 @@ bool CWalletTx::IsTrusted() const return false; // Trusted if all inputs are from us and are in the mempool: - BOOST_FOREACH(const CTxIn& txin, vin) + BOOST_FOREACH(const CTxIn& txin, tx->vin) { // Transactions not sent by us: not trusted const CWalletTx* parent = pwallet->GetWalletTx(txin.prevout.hash); if (parent == NULL) return false; - const CTxOut& parentOut = parent->vout[txin.prevout.n]; + const CTxOut& parentOut = parent->tx->vout[txin.prevout.n]; if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) return false; } return true; } -bool CWalletTx::IsEquivalentTo(const CWalletTx& tx) const +bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { - CMutableTransaction tx1 = *this; - CMutableTransaction tx2 = tx; + CMutableTransaction tx1 = *this->tx; + CMutableTransaction tx2 = *_tx.tx; for (unsigned int i = 0; i < tx1.vin.size(); i++) tx1.vin[i].scriptSig = CScript(); for (unsigned int i = 0; i < tx2.vin.size(); i++) tx2.vin[i].scriptSig = CScript(); return CTransaction(tx1) == CTransaction(tx2); @@ -1957,10 +1958,10 @@ void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const if (nDepth == 0 && !pcoin->InMempool()) continue; - for (unsigned int i = 0; i < pcoin->vout.size(); i++) { - isminetype mine = IsMine(pcoin->vout[i]); + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { + isminetype mine = IsMine(pcoin->tx->vout[i]); if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && - !IsLockedCoin((*it).first, i) && (pcoin->vout[i].nValue > 0 || fIncludeZeroValue) && + !IsLockedCoin((*it).first, i) && (pcoin->tx->vout[i].nValue > 0 || fIncludeZeroValue) && (!coinControl || !coinControl->HasSelected() || coinControl->fAllowOtherInputs || coinControl->IsSelected(COutPoint((*it).first, i)))) vCoins.push_back(COutput(pcoin, i, nDepth, ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || @@ -2046,7 +2047,7 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin continue; int i = output.i; - CAmount n = pcoin->vout[i].nValue; + CAmount n = pcoin->tx->vout[i].nValue; pair<CAmount,pair<const CWalletTx*,unsigned int> > coin = make_pair(n,make_pair(pcoin, i)); @@ -2133,7 +2134,7 @@ bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& { if (!out.fSpendable) continue; - nValueRet += out.tx->vout[out.i].nValue; + nValueRet += out.tx->tx->vout[out.i].nValue; setCoinsRet.insert(make_pair(out.tx, out.i)); } return (nValueRet >= nTargetValue); @@ -2153,9 +2154,9 @@ bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& { const CWalletTx* pcoin = &it->second; // Clearly invalid input, fail - if (pcoin->vout.size() <= outpoint.n) + if (pcoin->tx->vout.size() <= outpoint.n) return false; - nValueFromPresetInputs += pcoin->vout[outpoint.n].nValue; + nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; setPresetCoins.insert(make_pair(pcoin, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs @@ -2218,10 +2219,10 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool ov return false; if (nChangePosInOut != -1) - tx.vout.insert(tx.vout.begin() + nChangePosInOut, wtx.vout[nChangePosInOut]); + tx.vout.insert(tx.vout.begin() + nChangePosInOut, wtx.tx->vout[nChangePosInOut]); // Add new txins (keeping original txin scriptSig/order) - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (!coinControl.IsSelected(txin.prevout)) { @@ -2361,7 +2362,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt } BOOST_FOREACH(PAIRTYPE(const CWalletTx*, unsigned int) pcoin, setCoins) { - CAmount nCredit = pcoin.first->vout[pcoin.second].nValue; + CAmount nCredit = pcoin.first->tx->vout[pcoin.second].nValue; //The coin age after the next block (depth+1) is used instead of the current, //reflecting an assumption the user would accept a bit more delay for //a chance at a free transaction. @@ -2399,7 +2400,11 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt CPubKey vchPubKey; bool ret; ret = reservekey.GetReservedKey(vchPubKey); - assert(ret); // should never fail, as we just unlocked + if (!ret) + { + strFailReason = _("Keypool ran out, please call keypoolrefill first"); + return false; + } scriptChange = GetScriptForDestination(vchPubKey.GetID()); } @@ -2476,10 +2481,10 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt BOOST_FOREACH(const PAIRTYPE(const CWalletTx*,unsigned int)& coin, setCoins) { bool signSuccess; - const CScript& scriptPubKey = coin.first->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; SignatureData sigdata; if (sign) - signSuccess = ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.first->vout[coin.second].nValue, SIGHASH_ALL), scriptPubKey, sigdata); + signSuccess = ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.first->tx->vout[coin.second].nValue, SIGHASH_ALL), scriptPubKey, sigdata); else signSuccess = ProduceSignature(DummySignatureCreator(this), scriptPubKey, sigdata); @@ -2504,16 +2509,16 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt } // Embed the constructed transaction data in wtxNew. - *static_cast<CTransaction*>(&wtxNew) = CTransaction(txNew); + wtxNew.SetTx(MakeTransactionRef(std::move(txNew))); // Limit size - if (GetTransactionWeight(txNew) >= MAX_STANDARD_TX_WEIGHT) + if (GetTransactionWeight(wtxNew) >= MAX_STANDARD_TX_WEIGHT) { strFailReason = _("Transaction too large"); return false; } - dPriority = wtxNew.ComputePriority(dPriority, nBytes); + dPriority = wtxNew.tx->ComputePriority(dPriority, nBytes); // Allow to override the default confirmation target over the CoinControl instance int currentConfirmationTarget = nTxConfirmTarget; @@ -2580,7 +2585,7 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon { { LOCK2(cs_main, cs_wallet); - LogPrintf("CommitTransaction:\n%s", wtxNew.ToString()); + LogPrintf("CommitTransaction:\n%s", wtxNew.tx->ToString()); { // Take key pair from key pool so it won't be used again reservekey.KeepKey(); @@ -2590,7 +2595,7 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon AddToWallet(wtxNew); // Notify that old coins are spent - BOOST_FOREACH(const CTxIn& txin, wtxNew.vin) + BOOST_FOREACH(const CTxIn& txin, wtxNew.tx->vin) { CWalletTx &coin = mapWallet[txin.prevout.hash]; coin.BindWallet(this); @@ -2605,11 +2610,11 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon { // Broadcast if (!wtxNew.AcceptToMemoryPool(maxTxFee, state)) { - // This must not fail. The transaction has already been signed and recorded. - LogPrintf("CommitTransaction(): Error: Transaction not valid, %s\n", state.GetRejectReason()); - return false; + LogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", state.GetRejectReason()); + // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. + } else { + wtxNew.RelayWalletTransaction(connman); } - wtxNew.RelayWalletTransaction(connman); } } return true; @@ -2964,15 +2969,15 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() if (nDepth < (pcoin->IsFromMe(ISMINE_ALL) ? 0 : 1)) continue; - for (unsigned int i = 0; i < pcoin->vout.size(); i++) + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { CTxDestination addr; - if (!IsMine(pcoin->vout[i])) + if (!IsMine(pcoin->tx->vout[i])) continue; - if(!ExtractDestination(pcoin->vout[i].scriptPubKey, addr)) + if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(walletEntry.first, i) ? 0 : pcoin->vout[i].nValue; + CAmount n = IsSpent(walletEntry.first, i) ? 0 : pcoin->tx->vout[i].nValue; if (!balances.count(addr)) balances[addr] = 0; @@ -2994,16 +2999,16 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() { CWalletTx *pcoin = &walletEntry.second; - if (pcoin->vin.size() > 0) + if (pcoin->tx->vin.size() > 0) { bool any_mine = false; // group all input addresses with each other - BOOST_FOREACH(CTxIn txin, pcoin->vin) + BOOST_FOREACH(CTxIn txin, pcoin->tx->vin) { CTxDestination address; if(!IsMine(txin)) /* If this input isn't mine, ignore it */ continue; - if(!ExtractDestination(mapWallet[txin.prevout.hash].vout[txin.prevout.n].scriptPubKey, address)) + if(!ExtractDestination(mapWallet[txin.prevout.hash].tx->vout[txin.prevout.n].scriptPubKey, address)) continue; grouping.insert(address); any_mine = true; @@ -3012,7 +3017,7 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() // group change with input addresses if (any_mine) { - BOOST_FOREACH(CTxOut txout, pcoin->vout) + BOOST_FOREACH(CTxOut txout, pcoin->tx->vout) if (IsChange(txout)) { CTxDestination txoutAddr; @@ -3029,11 +3034,11 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (unsigned int i = 0; i < pcoin->vout.size(); i++) - if (IsMine(pcoin->vout[i])) + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) + if (IsMine(pcoin->tx->vout[i])) { CTxDestination address; - if(!ExtractDestination(pcoin->vout[i].scriptPubKey, address)) + if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, address)) continue; grouping.insert(address); groupings.insert(grouping); @@ -3300,7 +3305,7 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t> &mapKeyBirth) const { if (blit != mapBlockIndex.end() && chainActive.Contains(blit->second)) { // ... which are already in a block int nHeight = blit->second->nHeight; - BOOST_FOREACH(const CTxOut &txout, wtx.vout) { + BOOST_FOREACH(const CTxOut &txout, wtx.tx->vout) { // iterate over all their outputs CAffectedKeysVisitor(*this, vAffected).Process(txout.scriptPubKey); BOOST_FOREACH(const CKeyID &keyid, vAffected) { @@ -3668,6 +3673,9 @@ bool CWallet::ParameterInteraction() fSendFreeTransactions = GetBoolArg("-sendfreetransactions", DEFAULT_SEND_FREE_TRANSACTIONS); fWalletRbf = GetBoolArg("-walletrbf", DEFAULT_WALLET_RBF); + if (fSendFreeTransactions && GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) <= 0) + return InitError("Creation of free transactions with their relay disabled is not supported."); + return true; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 47441c7a5d..3742552f5b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -163,13 +163,14 @@ struct COutputEntry }; /** A transaction with a merkle branch linking it to the block chain. */ -class CMerkleTx : public CTransaction +class CMerkleTx { private: /** Constant used in hashBlock to indicate tx has been abandoned */ static const uint256 ABANDON_HASH; public: + CTransactionRef tx; uint256 hashBlock; /* An nIndex == -1 means that hashBlock (in nonzero) refers to the earliest @@ -181,26 +182,37 @@ public: CMerkleTx() { + SetTx(MakeTransactionRef()); Init(); } - CMerkleTx(const CTransaction& txIn) : CTransaction(txIn) + CMerkleTx(CTransactionRef arg) { + SetTx(std::move(arg)); Init(); } + /** Helper conversion operator to allow passing CMerkleTx where CTransaction is expected. + * TODO: adapt callers and remove this operator. */ + operator const CTransaction&() const { return *tx; } + void Init() { hashBlock = uint256(); nIndex = -1; } + void SetTx(CTransactionRef arg) + { + tx = std::move(arg); + } + ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { std::vector<uint256> vMerkleBranch; // For compatibility with older versions. - READWRITE(*(CTransaction*)this); + READWRITE(tx); READWRITE(hashBlock); READWRITE(vMerkleBranch); READWRITE(nIndex); @@ -223,6 +235,9 @@ public: bool hashUnset() const { return (hashBlock.IsNull() || hashBlock == ABANDON_HASH); } bool isAbandoned() const { return (hashBlock == ABANDON_HASH); } void setAbandoned() { hashBlock = ABANDON_HASH; } + + const uint256& GetHash() const { return tx->GetHash(); } + bool IsCoinBase() const { return tx->IsCoinBase(); } }; /** @@ -269,17 +284,7 @@ public: Init(NULL); } - CWalletTx(const CWallet* pwalletIn) - { - Init(pwalletIn); - } - - CWalletTx(const CWallet* pwalletIn, const CMerkleTx& txIn) : CMerkleTx(txIn) - { - Init(pwalletIn); - } - - CWalletTx(const CWallet* pwalletIn, const CTransaction& txIn) : CMerkleTx(txIn) + CWalletTx(const CWallet* pwalletIn, CTransactionRef arg) : CMerkleTx(std::move(arg)) { Init(pwalletIn); } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 43fd6a20ad..0395e4a072 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -7,7 +7,7 @@ #include "base58.h" #include "consensus/validation.h" -#include "main.h" // For CheckTransaction +#include "validation.h" // For CheckTransaction #include "protocol.h" #include "serialize.h" #include "sync.h" diff --git a/src/warnings.cpp b/src/warnings.cpp new file mode 100644 index 0000000000..2c1b1b0e12 --- /dev/null +++ b/src/warnings.cpp @@ -0,0 +1,89 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "sync.h" +#include "clientversion.h" +#include "util.h" +#include "warnings.h" + +CCriticalSection cs_warnings; +std::string strMiscWarning; +bool fLargeWorkForkFound = false; +bool fLargeWorkInvalidChainFound = false; + +void SetMiscWarning(const std::string& strWarning) +{ + LOCK(cs_warnings); + strMiscWarning = strWarning; +} + +void SetfLargeWorkForkFound(bool flag) +{ + LOCK(cs_warnings); + fLargeWorkForkFound = flag; +} + +bool GetfLargeWorkForkFound() +{ + LOCK(cs_warnings); + return fLargeWorkForkFound; +} + +void SetfLargeWorkInvalidChainFound(bool flag) +{ + LOCK(cs_warnings); + fLargeWorkInvalidChainFound = flag; +} + +bool GetfLargeWorkInvalidChainFound() +{ + LOCK(cs_warnings); + return fLargeWorkInvalidChainFound; +} + +std::string GetWarnings(const std::string& strFor) +{ + std::string strStatusBar; + std::string strRPC; + std::string strGUI; + const std::string uiAlertSeperator = "<hr />"; + + LOCK(cs_warnings); + + if (!CLIENT_VERSION_IS_RELEASE) { + strStatusBar = "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"; + strGUI = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); + } + + if (GetBoolArg("-testsafemode", DEFAULT_TESTSAFEMODE)) + strStatusBar = strRPC = strGUI = "testsafemode enabled"; + + // Misc warnings like out of disk space and clock is wrong + if (strMiscWarning != "") + { + strStatusBar = strMiscWarning; + strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + strMiscWarning; + } + + if (fLargeWorkForkFound) + { + strStatusBar = strRPC = "Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."; + strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); + } + else if (fLargeWorkInvalidChainFound) + { + strStatusBar = strRPC = "Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."; + strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); + } + + if (strFor == "gui") + return strGUI; + else if (strFor == "statusbar") + return strStatusBar; + else if (strFor == "rpc") + return strRPC; + assert(!"GetWarnings(): invalid parameter"); + return "error"; +} diff --git a/src/warnings.h b/src/warnings.h new file mode 100644 index 0000000000..a7aa657426 --- /dev/null +++ b/src/warnings.h @@ -0,0 +1,21 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WARNINGS_H +#define BITCOIN_WARNINGS_H + +#include <stdlib.h> +#include <string> + +void SetMiscWarning(const std::string& strWarning); +void SetfLargeWorkForkFound(bool flag); +bool GetfLargeWorkForkFound(); +void SetfLargeWorkInvalidChainFound(bool flag); +bool GetfLargeWorkInvalidChainFound(); +std::string GetWarnings(const std::string& strFor); + +static const bool DEFAULT_TESTSAFEMODE = false; + +#endif // BITCOIN_WARNINGS_H diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index a0196fe184..a7e20a8356 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -6,7 +6,7 @@ #include "zmqpublishnotifier.h" #include "version.h" -#include "main.h" +#include "validation.h" #include "streams.h" #include "util.h" diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index b6c907980f..a11256dfd5 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -3,9 +3,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "chainparams.h" +#include "streams.h" #include "zmqpublishnotifier.h" -#include "main.h" +#include "validation.h" #include "util.h" +#include "rpc/server.h" static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; @@ -165,7 +167,7 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) LogPrint("zmq", "zmq: Publish rawblock %s\n", pindex->GetBlockHash().GetHex()); const Consensus::Params& consensusParams = Params().GetConsensus(); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); { LOCK(cs_main); CBlock block; @@ -185,7 +187,7 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr { uint256 hash = transaction.GetHash(); LogPrint("zmq", "zmq: Publish rawtx %s\n", hash.GetHex()); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ss << transaction; return SendMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } |