aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/Makefile.am2
-rw-r--r--src/Makefile.bench.include1
-rw-r--r--src/Makefile.test.include2
-rw-r--r--src/bench/gcs_filter.cpp43
-rw-r--r--src/blockfilter.cpp260
-rw-r--r--src/blockfilter.h145
-rw-r--r--src/streams.h167
-rw-r--r--src/test/blockfilter_tests.cpp142
-rw-r--r--src/test/data/blockfilters.json9
-rw-r--r--src/test/streams_tests.cpp80
-rw-r--r--src/undo.h1
11 files changed, 852 insertions, 0 deletions
diff --git a/src/Makefile.am b/src/Makefile.am
index d1693fa85c..3701ee8f3c 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -96,6 +96,7 @@ BITCOIN_CORE_H = \
bech32.h \
bloom.h \
blockencodings.h \
+ blockfilter.h \
chain.h \
chainparams.h \
chainparamsbase.h \
@@ -219,6 +220,7 @@ libbitcoin_server_a_SOURCES = \
addrman.cpp \
bloom.cpp \
blockencodings.cpp \
+ blockfilter.cpp \
chain.cpp \
checkpoints.cpp \
consensus/tx_verify.cpp \
diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include
index f5293585a0..76177630ab 100644
--- a/src/Makefile.bench.include
+++ b/src/Makefile.bench.include
@@ -22,6 +22,7 @@ bench_bench_bitcoin_SOURCES = \
bench/rollingbloom.cpp \
bench/crypto_hash.cpp \
bench/ccoins_caching.cpp \
+ bench/gcs_filter.cpp \
bench/merkle_root.cpp \
bench/mempool_eviction.cpp \
bench/verify_script.cpp \
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index 6f401636f5..9e7d0dc745 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -9,6 +9,7 @@ TEST_BINARY=test/test_bitcoin$(EXEEXT)
JSON_TEST_FILES = \
test/data/base58_encode_decode.json \
+ test/data/blockfilters.json \
test/data/key_io_valid.json \
test/data/key_io_invalid.json \
test/data/script_tests.json \
@@ -39,6 +40,7 @@ BITCOIN_TESTS =\
test/bip32_tests.cpp \
test/blockchain_tests.cpp \
test/blockencodings_tests.cpp \
+ test/blockfilter_tests.cpp \
test/bloom_tests.cpp \
test/bswap_tests.cpp \
test/checkqueue_tests.cpp \
diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp
new file mode 100644
index 0000000000..6f4e384e3b
--- /dev/null
+++ b/src/bench/gcs_filter.cpp
@@ -0,0 +1,43 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+#include <blockfilter.h>
+
+static void ConstructGCSFilter(benchmark::State& state)
+{
+ GCSFilter::ElementSet elements;
+ for (int i = 0; i < 10000; ++i) {
+ GCSFilter::Element element(32);
+ element[0] = static_cast<unsigned char>(i);
+ element[1] = static_cast<unsigned char>(i >> 8);
+ elements.insert(std::move(element));
+ }
+
+ uint64_t siphash_k0 = 0;
+ while (state.KeepRunning()) {
+ GCSFilter filter(siphash_k0, 0, 20, 1 << 20, elements);
+
+ siphash_k0++;
+ }
+}
+
+static void MatchGCSFilter(benchmark::State& state)
+{
+ GCSFilter::ElementSet elements;
+ for (int i = 0; i < 10000; ++i) {
+ GCSFilter::Element element(32);
+ element[0] = static_cast<unsigned char>(i);
+ element[1] = static_cast<unsigned char>(i >> 8);
+ elements.insert(std::move(element));
+ }
+ GCSFilter filter(0, 0, 20, 1 << 20, elements);
+
+ while (state.KeepRunning()) {
+ filter.Match(GCSFilter::Element());
+ }
+}
+
+BENCHMARK(ConstructGCSFilter, 1000);
+BENCHMARK(MatchGCSFilter, 50 * 1000);
diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp
new file mode 100644
index 0000000000..38fcfacf7f
--- /dev/null
+++ b/src/blockfilter.cpp
@@ -0,0 +1,260 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <blockfilter.h>
+#include <hash.h>
+#include <primitives/transaction.h>
+#include <script/script.h>
+#include <streams.h>
+
+/// SerType used to serialize parameters in GCS filter encoding.
+static constexpr int GCS_SER_TYPE = SER_NETWORK;
+
+/// Protocol version used to serialize parameters in GCS filter encoding.
+static constexpr int GCS_SER_VERSION = 0;
+
+template <typename OStream>
+static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x)
+{
+ // Write quotient as unary-encoded: q 1's followed by one 0.
+ uint64_t q = x >> P;
+ while (q > 0) {
+ int nbits = q <= 64 ? static_cast<int>(q) : 64;
+ bitwriter.Write(~0ULL, nbits);
+ q -= nbits;
+ }
+ bitwriter.Write(0, 1);
+
+ // Write the remainder in P bits. Since the remainder is just the bottom
+ // P bits of x, there is no need to mask first.
+ bitwriter.Write(x, P);
+}
+
+template <typename IStream>
+static uint64_t GolombRiceDecode(BitStreamReader<IStream>& bitreader, uint8_t P)
+{
+ // Read unary-encoded quotient: q 1's followed by one 0.
+ uint64_t q = 0;
+ while (bitreader.Read(1) == 1) {
+ ++q;
+ }
+
+ uint64_t r = bitreader.Read(P);
+
+ return (q << P) + r;
+}
+
+// Map a value x that is uniformly distributed in the range [0, 2^64) to a
+// value uniformly distributed in [0, n) by returning the upper 64 bits of
+// x * n.
+//
+// See: https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
+static uint64_t MapIntoRange(uint64_t x, uint64_t n)
+{
+#ifdef __SIZEOF_INT128__
+ return (static_cast<unsigned __int128>(x) * static_cast<unsigned __int128>(n)) >> 64;
+#else
+ // To perform the calculation on 64-bit numbers without losing the
+ // result to overflow, split the numbers into the most significant and
+ // least significant 32 bits and perform multiplication piece-wise.
+ //
+ // See: https://stackoverflow.com/a/26855440
+ uint64_t x_hi = x >> 32;
+ uint64_t x_lo = x & 0xFFFFFFFF;
+ uint64_t n_hi = n >> 32;
+ uint64_t n_lo = n & 0xFFFFFFFF;
+
+ uint64_t ac = x_hi * n_hi;
+ uint64_t ad = x_hi * n_lo;
+ uint64_t bc = x_lo * n_hi;
+ uint64_t bd = x_lo * n_lo;
+
+ uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF);
+ uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32);
+ return upper64;
+#endif
+}
+
+uint64_t GCSFilter::HashToRange(const Element& element) const
+{
+ uint64_t hash = CSipHasher(m_siphash_k0, m_siphash_k1)
+ .Write(element.data(), element.size())
+ .Finalize();
+ return MapIntoRange(hash, m_F);
+}
+
+std::vector<uint64_t> GCSFilter::BuildHashedSet(const ElementSet& elements) const
+{
+ std::vector<uint64_t> hashed_elements;
+ hashed_elements.reserve(elements.size());
+ for (const Element& element : elements) {
+ hashed_elements.push_back(HashToRange(element));
+ }
+ std::sort(hashed_elements.begin(), hashed_elements.end());
+ return hashed_elements;
+}
+
+GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M)
+ : m_siphash_k0(siphash_k0), m_siphash_k1(siphash_k1), m_P(P), m_M(M), m_N(0), m_F(0)
+{}
+
+GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M,
+ std::vector<unsigned char> encoded_filter)
+ : GCSFilter(siphash_k0, siphash_k1, P, M)
+{
+ m_encoded = std::move(encoded_filter);
+
+ VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0);
+
+ uint64_t N = ReadCompactSize(stream);
+ m_N = static_cast<uint32_t>(N);
+ if (m_N != N) {
+ throw std::ios_base::failure("N must be <2^32");
+ }
+ m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_M);
+
+ // Verify that the encoded filter contains exactly N elements. If it has too much or too little
+ // data, a std::ios_base::failure exception will be raised.
+ BitStreamReader<VectorReader> bitreader(stream);
+ for (uint64_t i = 0; i < m_N; ++i) {
+ GolombRiceDecode(bitreader, m_P);
+ }
+ if (!stream.empty()) {
+ throw std::ios_base::failure("encoded_filter contains excess data");
+ }
+}
+
+GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M,
+ const ElementSet& elements)
+ : GCSFilter(siphash_k0, siphash_k1, P, M)
+{
+ size_t N = elements.size();
+ m_N = static_cast<uint32_t>(N);
+ if (m_N != N) {
+ throw std::invalid_argument("N must be <2^32");
+ }
+ m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_M);
+
+ CVectorWriter stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0);
+
+ WriteCompactSize(stream, m_N);
+
+ if (elements.empty()) {
+ return;
+ }
+
+ BitStreamWriter<CVectorWriter> bitwriter(stream);
+
+ uint64_t last_value = 0;
+ for (uint64_t value : BuildHashedSet(elements)) {
+ uint64_t delta = value - last_value;
+ GolombRiceEncode(bitwriter, m_P, delta);
+ last_value = value;
+ }
+
+ bitwriter.Flush();
+}
+
+bool GCSFilter::MatchInternal(const uint64_t* element_hashes, size_t size) const
+{
+ VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0);
+
+ // Seek forward by size of N
+ uint64_t N = ReadCompactSize(stream);
+ assert(N == m_N);
+
+ BitStreamReader<VectorReader> bitreader(stream);
+
+ uint64_t value = 0;
+ size_t hashes_index = 0;
+ for (uint32_t i = 0; i < m_N; ++i) {
+ uint64_t delta = GolombRiceDecode(bitreader, m_P);
+ value += delta;
+
+ while (true) {
+ if (hashes_index == size) {
+ return false;
+ } else if (element_hashes[hashes_index] == value) {
+ return true;
+ } else if (element_hashes[hashes_index] > value) {
+ break;
+ }
+
+ hashes_index++;
+ }
+ }
+
+ return false;
+}
+
+bool GCSFilter::Match(const Element& element) const
+{
+ uint64_t query = HashToRange(element);
+ return MatchInternal(&query, 1);
+}
+
+bool GCSFilter::MatchAny(const ElementSet& elements) const
+{
+ const std::vector<uint64_t> queries = BuildHashedSet(elements);
+ return MatchInternal(queries.data(), queries.size());
+}
+
+static GCSFilter::ElementSet BasicFilterElements(const CBlock& block,
+ const CBlockUndo& block_undo)
+{
+ GCSFilter::ElementSet elements;
+
+ for (const CTransactionRef& tx : block.vtx) {
+ for (const CTxOut& txout : tx->vout) {
+ const CScript& script = txout.scriptPubKey;
+ if (script[0] == OP_RETURN) continue;
+ elements.emplace(script.begin(), script.end());
+ }
+ }
+
+ for (const CTxUndo& tx_undo : block_undo.vtxundo) {
+ for (const Coin& prevout : tx_undo.vprevout) {
+ const CScript& script = prevout.out.scriptPubKey;
+ elements.emplace(script.begin(), script.end());
+ }
+ }
+
+ return elements;
+}
+
+BlockFilter::BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo)
+ : m_filter_type(filter_type), m_block_hash(block.GetHash())
+{
+ switch (m_filter_type) {
+ case BlockFilterType::BASIC:
+ m_filter = GCSFilter(m_block_hash.GetUint64(0), m_block_hash.GetUint64(1),
+ BASIC_FILTER_P, BASIC_FILTER_M,
+ BasicFilterElements(block, block_undo));
+ break;
+
+ default:
+ throw std::invalid_argument("unknown filter_type");
+ }
+}
+
+uint256 BlockFilter::GetHash() const
+{
+ const std::vector<unsigned char>& data = GetEncodedFilter();
+
+ uint256 result;
+ CHash256().Write(data.data(), data.size()).Finalize(result.begin());
+ return result;
+}
+
+uint256 BlockFilter::ComputeHeader(const uint256& prev_header) const
+{
+ const uint256& filter_hash = GetHash();
+
+ uint256 result;
+ CHash256()
+ .Write(filter_hash.begin(), filter_hash.size())
+ .Write(prev_header.begin(), prev_header.size())
+ .Finalize(result.begin());
+ return result;
+}
diff --git a/src/blockfilter.h b/src/blockfilter.h
new file mode 100644
index 0000000000..46833ac0be
--- /dev/null
+++ b/src/blockfilter.h
@@ -0,0 +1,145 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_BLOCKFILTER_H
+#define BITCOIN_BLOCKFILTER_H
+
+#include <set>
+#include <stdint.h>
+#include <vector>
+
+#include <primitives/block.h>
+#include <serialize.h>
+#include <uint256.h>
+#include <undo.h>
+
+/**
+ * This implements a Golomb-coded set as defined in BIP 158. It is a
+ * compact, probabilistic data structure for testing set membership.
+ */
+class GCSFilter
+{
+public:
+ typedef std::vector<unsigned char> Element;
+ typedef std::set<Element> ElementSet;
+
+private:
+ uint64_t m_siphash_k0;
+ uint64_t m_siphash_k1;
+ uint8_t m_P; //!< Golomb-Rice coding parameter
+ uint32_t m_M; //!< Inverse false positive rate
+ uint32_t m_N; //!< Number of elements in the filter
+ uint64_t m_F; //!< Range of element hashes, F = N * M
+ std::vector<unsigned char> m_encoded;
+
+ /** Hash a data element to an integer in the range [0, N * M). */
+ uint64_t HashToRange(const Element& element) const;
+
+ std::vector<uint64_t> BuildHashedSet(const ElementSet& elements) const;
+
+ /** Helper method used to implement Match and MatchAny */
+ bool MatchInternal(const uint64_t* sorted_element_hashes, size_t size) const;
+
+public:
+
+ /** Constructs an empty filter. */
+ GCSFilter(uint64_t siphash_k0 = 0, uint64_t siphash_k1 = 0, uint8_t P = 0, uint32_t M = 0);
+
+ /** Reconstructs an already-created filter from an encoding. */
+ GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M,
+ std::vector<unsigned char> encoded_filter);
+
+ /** Builds a new filter from the params and set of elements. */
+ GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M,
+ const ElementSet& elements);
+
+ uint8_t GetP() const { return m_P; }
+ uint32_t GetN() const { return m_N; }
+ uint32_t GetM() const { return m_M; }
+ const std::vector<unsigned char>& GetEncoded() const { return m_encoded; }
+
+ /**
+ * Checks if the element may be in the set. False positives are possible
+ * with probability 1/M.
+ */
+ bool Match(const Element& element) const;
+
+ /**
+ * Checks if any of the given elements may be in the set. False positives
+ * are possible with probability 1/M per element checked. This is more
+ * efficient that checking Match on multiple elements separately.
+ */
+ bool MatchAny(const ElementSet& elements) const;
+};
+
+constexpr uint8_t BASIC_FILTER_P = 19;
+constexpr uint32_t BASIC_FILTER_M = 784931;
+
+enum BlockFilterType : uint8_t
+{
+ BASIC = 0,
+};
+
+/**
+ * Complete block filter struct as defined in BIP 157. Serialization matches
+ * payload of "cfilter" messages.
+ */
+class BlockFilter
+{
+private:
+ BlockFilterType m_filter_type;
+ uint256 m_block_hash;
+ GCSFilter m_filter;
+
+public:
+
+ // Construct a new BlockFilter of the specified type from a block.
+ BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo);
+
+ BlockFilterType GetFilterType() const { return m_filter_type; }
+
+ const GCSFilter& GetFilter() const { return m_filter; }
+
+ const std::vector<unsigned char>& GetEncodedFilter() const
+ {
+ return m_filter.GetEncoded();
+ }
+
+ // Compute the filter hash.
+ uint256 GetHash() const;
+
+ // Compute the filter header given the previous one.
+ uint256 ComputeHeader(const uint256& prev_header) const;
+
+ template <typename Stream>
+ void Serialize(Stream& s) const {
+ s << m_block_hash
+ << static_cast<uint8_t>(m_filter_type)
+ << m_filter.GetEncoded();
+ }
+
+ template <typename Stream>
+ void Unserialize(Stream& s) {
+ std::vector<unsigned char> encoded_filter;
+ uint8_t filter_type;
+
+ s >> m_block_hash
+ >> filter_type
+ >> encoded_filter;
+
+ m_filter_type = static_cast<BlockFilterType>(filter_type);
+
+ switch (m_filter_type) {
+ case BlockFilterType::BASIC:
+ m_filter = GCSFilter(m_block_hash.GetUint64(0), m_block_hash.GetUint64(1),
+ BASIC_FILTER_P, BASIC_FILTER_M, std::move(encoded_filter));
+ break;
+
+ default:
+ throw std::ios_base::failure("unknown filter_type");
+ }
+ }
+};
+
+#endif // BITCOIN_BLOCKFILTER_H
diff --git a/src/streams.h b/src/streams.h
index 096ebfc9c2..d4a309be3c 100644
--- a/src/streams.h
+++ b/src/streams.h
@@ -139,6 +139,80 @@ private:
size_t nPos;
};
+/** Minimal stream for reading from an existing vector by reference
+ */
+class VectorReader
+{
+private:
+ const int m_type;
+ const int m_version;
+ const std::vector<unsigned char>& m_data;
+ size_t m_pos = 0;
+
+public:
+
+ /*
+ * @param[in] type Serialization Type
+ * @param[in] version Serialization Version (including any flags)
+ * @param[in] data Referenced byte vector to overwrite/append
+ * @param[in] pos Starting position. Vector index where reads should start.
+ */
+ VectorReader(int type, int version, const std::vector<unsigned char>& data, size_t pos)
+ : m_type(type), m_version(version), m_data(data)
+ {
+ seek(pos);
+ }
+
+ /*
+ * (other params same as above)
+ * @param[in] args A list of items to deserialize starting at pos.
+ */
+ template <typename... Args>
+ VectorReader(int type, int version, const std::vector<unsigned char>& data, size_t pos,
+ Args&&... args)
+ : VectorReader(type, version, data, pos)
+ {
+ ::UnserializeMany(*this, std::forward<Args>(args)...);
+ }
+
+ template<typename T>
+ VectorReader& operator>>(T& obj)
+ {
+ // Unserialize from this stream
+ ::Unserialize(*this, obj);
+ return (*this);
+ }
+
+ int GetVersion() const { return m_version; }
+ int GetType() const { return m_type; }
+
+ size_t size() const { return m_data.size() - m_pos; }
+ bool empty() const { return m_data.size() == m_pos; }
+
+ void read(char* dst, size_t n)
+ {
+ if (n == 0) {
+ return;
+ }
+
+ // Read from the beginning of the buffer
+ size_t pos_next = m_pos + n;
+ if (pos_next > m_data.size()) {
+ throw std::ios_base::failure("VectorReader::read(): end of data");
+ }
+ memcpy(dst, m_data.data() + m_pos, n);
+ m_pos = pos_next;
+ }
+
+ void seek(size_t n)
+ {
+ m_pos += n;
+ if (m_pos > m_data.size()) {
+ throw std::ios_base::failure("VectorReader::seek(): end of data");
+ }
+ }
+};
+
/** Double ended buffer combining vector and stream-like interfaces.
*
* >> and << read and write unformatted data using the above serialization templates.
@@ -436,12 +510,105 @@ public:
}
};
+template <typename IStream>
+class BitStreamReader
+{
+private:
+ IStream& m_istream;
+
+ /// Buffered byte read in from the input stream. A new byte is read into the
+ /// buffer when m_offset reaches 8.
+ uint8_t m_buffer{0};
+
+ /// Number of high order bits in m_buffer already returned by previous
+ /// Read() calls. The next bit to be returned is at this offset from the
+ /// most significant bit position.
+ int m_offset{8};
+
+public:
+ explicit BitStreamReader(IStream& istream) : m_istream(istream) {}
+
+ /** Read the specified number of bits from the stream. The data is returned
+ * in the nbits least signficant bits of a 64-bit uint.
+ */
+ uint64_t Read(int nbits) {
+ if (nbits < 0 || nbits > 64) {
+ throw std::out_of_range("nbits must be between 0 and 64");
+ }
+
+ uint64_t data = 0;
+ while (nbits > 0) {
+ if (m_offset == 8) {
+ m_istream >> m_buffer;
+ m_offset = 0;
+ }
+ int bits = std::min(8 - m_offset, nbits);
+ data <<= bits;
+ data |= static_cast<uint8_t>(m_buffer << m_offset) >> (8 - bits);
+ m_offset += bits;
+ nbits -= bits;
+ }
+ return data;
+ }
+};
+template <typename OStream>
+class BitStreamWriter
+{
+private:
+ OStream& m_ostream;
+ /// Buffered byte waiting to be written to the output stream. The byte is
+ /// written buffer when m_offset reaches 8 or Flush() is called.
+ uint8_t m_buffer{0};
+ /// Number of high order bits in m_buffer already written by previous
+ /// Write() calls and not yet flushed to the stream. The next bit to be
+ /// written to is at this offset from the most significant bit position.
+ int m_offset{0};
+public:
+ explicit BitStreamWriter(OStream& ostream) : m_ostream(ostream) {}
+ ~BitStreamWriter()
+ {
+ Flush();
+ }
+
+ /** Write the nbits least significant bits of a 64-bit int to the output
+ * stream. Data is buffered until it completes an octet.
+ */
+ void Write(uint64_t data, int nbits) {
+ if (nbits < 0 || nbits > 64) {
+ throw std::out_of_range("nbits must be between 0 and 64");
+ }
+
+ while (nbits > 0) {
+ int bits = std::min(8 - m_offset, nbits);
+ m_buffer |= (data << (64 - nbits)) >> (64 - 8 + m_offset);
+ m_offset += bits;
+ nbits -= bits;
+
+ if (m_offset == 8) {
+ Flush();
+ }
+ }
+ }
+
+ /** Flush any unwritten bits to the output stream, padding with 0's to the
+ * next byte boundary.
+ */
+ void Flush() {
+ if (m_offset == 0) {
+ return;
+ }
+
+ m_ostream << m_buffer;
+ m_buffer = 0;
+ m_offset = 0;
+ }
+};
diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp
new file mode 100644
index 0000000000..e3cb05f09e
--- /dev/null
+++ b/src/test/blockfilter_tests.cpp
@@ -0,0 +1,142 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <test/data/blockfilters.json.h>
+#include <test/test_bitcoin.h>
+
+#include <blockfilter.h>
+#include <core_io.h>
+#include <serialize.h>
+#include <streams.h>
+#include <univalue.h>
+#include <utilstrencodings.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(blockfilter_tests)
+
+BOOST_AUTO_TEST_CASE(gcsfilter_test)
+{
+ GCSFilter::ElementSet included_elements, excluded_elements;
+ for (int i = 0; i < 100; ++i) {
+ GCSFilter::Element element1(32);
+ element1[0] = i;
+ included_elements.insert(std::move(element1));
+
+ GCSFilter::Element element2(32);
+ element2[1] = i;
+ excluded_elements.insert(std::move(element2));
+ }
+
+ GCSFilter filter(0, 0, 10, 1 << 10, included_elements);
+ for (const auto& element : included_elements) {
+ BOOST_CHECK(filter.Match(element));
+
+ auto insertion = excluded_elements.insert(element);
+ BOOST_CHECK(filter.MatchAny(excluded_elements));
+ excluded_elements.erase(insertion.first);
+ }
+}
+
+BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
+{
+ CScript included_scripts[5], excluded_scripts[2];
+
+ // First two are outputs on a single transaction.
+ included_scripts[0] << std::vector<unsigned char>(0, 65) << OP_CHECKSIG;
+ included_scripts[1] << OP_DUP << OP_HASH160 << std::vector<unsigned char>(1, 20) << OP_EQUALVERIFY << OP_CHECKSIG;
+
+ // Third is an output on in a second transaction.
+ included_scripts[2] << OP_1 << std::vector<unsigned char>(2, 33) << OP_1 << OP_CHECKMULTISIG;
+
+ // Last two are spent by a single transaction.
+ included_scripts[3] << OP_0 << std::vector<unsigned char>(3, 32);
+ included_scripts[4] << OP_4 << OP_ADD << OP_8 << OP_EQUAL;
+
+ // OP_RETURN output is an output on the second transaction.
+ excluded_scripts[0] << OP_RETURN << std::vector<unsigned char>(4, 40);
+
+ // This script is not related to the block at all.
+ excluded_scripts[1] << std::vector<unsigned char>(5, 33) << OP_CHECKSIG;
+
+ CMutableTransaction tx_1;
+ tx_1.vout.emplace_back(100, included_scripts[0]);
+ tx_1.vout.emplace_back(200, included_scripts[1]);
+
+ CMutableTransaction tx_2;
+ tx_2.vout.emplace_back(300, included_scripts[2]);
+ tx_2.vout.emplace_back(0, excluded_scripts[0]);
+
+ CBlock block;
+ block.vtx.push_back(MakeTransactionRef(tx_1));
+ block.vtx.push_back(MakeTransactionRef(tx_2));
+
+ CBlockUndo block_undo;
+ block_undo.vtxundo.emplace_back();
+ block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(400, included_scripts[3]), 1000, true);
+ block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(500, included_scripts[4]), 10000, false);
+
+ BlockFilter block_filter(BlockFilterType::BASIC, block, block_undo);
+ const GCSFilter& filter = block_filter.GetFilter();
+
+ for (const CScript& script : included_scripts) {
+ BOOST_CHECK(filter.Match(GCSFilter::Element(script.begin(), script.end())));
+ }
+ for (const CScript& script : excluded_scripts) {
+ BOOST_CHECK(!filter.Match(GCSFilter::Element(script.begin(), script.end())));
+ }
+}
+
+BOOST_AUTO_TEST_CASE(blockfilters_json_test)
+{
+ UniValue json;
+ std::string json_data(json_tests::blockfilters,
+ json_tests::blockfilters + sizeof(json_tests::blockfilters));
+ if (!json.read(json_data) || !json.isArray()) {
+ BOOST_ERROR("Parse error.");
+ return;
+ }
+
+ const UniValue& tests = json.get_array();
+ for (unsigned int i = 0; i < tests.size(); i++) {
+ UniValue test = tests[i];
+ std::string strTest = test.write();
+
+ if (test.size() == 1) {
+ continue;
+ } else if (test.size() < 7) {
+ BOOST_ERROR("Bad test: " << strTest);
+ continue;
+ }
+
+ unsigned int pos = 0;
+ /*int block_height =*/ test[pos++].get_int();
+ /*uint256 block_hash =*/ ParseHashStr(test[pos++].get_str(), "block_hash");
+
+ CBlock block;
+ BOOST_REQUIRE(DecodeHexBlk(block, test[pos++].get_str()));
+
+ CBlockUndo block_undo;
+ block_undo.vtxundo.emplace_back();
+ CTxUndo& tx_undo = block_undo.vtxundo.back();
+ const UniValue& prev_scripts = test[pos++].get_array();
+ for (unsigned int ii = 0; ii < prev_scripts.size(); ii++) {
+ std::vector<unsigned char> raw_script = ParseHex(prev_scripts[ii].get_str());
+ CTxOut txout(0, CScript(raw_script.begin(), raw_script.end()));
+ tx_undo.vprevout.emplace_back(txout, 0, false);
+ }
+
+ uint256 prev_filter_header_basic = ParseHashStr(test[pos++].get_str(), "prev_filter_header_basic");
+ std::vector<unsigned char> filter_basic = ParseHex(test[pos++].get_str());
+ uint256 filter_header_basic = ParseHashStr(test[pos++].get_str(), "filter_header_basic");
+
+ BlockFilter computed_filter_basic(BlockFilterType::BASIC, block, block_undo);
+ BOOST_CHECK(computed_filter_basic.GetFilter().GetEncoded() == filter_basic);
+
+ uint256 computed_header_basic = computed_filter_basic.ComputeHeader(prev_filter_header_basic);
+ BOOST_CHECK(computed_header_basic == filter_header_basic);
+ }
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/test/data/blockfilters.json b/src/test/data/blockfilters.json
new file mode 100644
index 0000000000..2f00728ff6
--- /dev/null
+++ b/src/test/data/blockfilters.json
@@ -0,0 +1,9 @@
+[
+["Block Height,Block Hash,Block,[Prev Output Scripts for Block],Previous Basic Header,Basic Filter,Basic Header,Notes"],
+[0,"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff001d1aa4ae180101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000",[],"0000000000000000000000000000000000000000000000000000000000000000","019dfca8","21584579b7eb08997773e5aeff3a7f932700042d0ed2a6129012b7d7ae81b750","Genesis block"],
+[2,"000000006c02c8ea6e4ff69651f7fcde348fb9d557a06e6957b65552002a7820","0100000006128e87be8b1b4dea47a7247d5528d2702c96826c7a648497e773b800000000e241352e3bec0a95a6217e10c3abb54adfa05abb12c126695595580fb92e222032e7494dffff001d00d235340101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0432e7494d010e062f503253482fffffffff0100f2052a010000002321038a7f6ef1c8ca0c588aa53fa860128077c9e6c11e6830f4d7ee4e763a56b7718fac00000000",[],"d7bdac13a59d745b1add0d2ce852f1a0442e8945fc1bf3848d3cbffd88c24fe1","0174a170","186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0",""],
+[3,"000000008b896e272758da5297bcd98fdc6d97c9b765ecec401e286dc1fdbe10","0100000020782a005255b657696ea057d5b98f34defcf75196f64f6eeac8026c0000000041ba5afc532aae03151b8aa87b65e1594f97504a768e010c98c0add79216247186e7494dffff001d058dc2b60101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0486e7494d0151062f503253482fffffffff0100f2052a01000000232103f6d9ff4c12959445ca5549c811683bf9c88e637b222dd2e0311154c4c85cf423ac00000000",[],"186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0","016cf7a0","8d63aadf5ab7257cb6d2316a57b16f517bff1c6388f124ec4c04af1212729d2a",""],
+[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"da49977ba1ee0d620a2c4f8f646b03cd0d230f5c6c994722e3ba884889f0be1a","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","4cd9dd007a325199102f1fc0b7d77ca25ee3c84d46018c4353ecfcb56c0d3e7a","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"],
+[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"e9d729b72d533c29abe5276d5cf6c152f3723f10efe000b1e0c9ca5265a8beb6","010c0b40","e6137ae5a8424c40da1e5023c16975cc97b09300b4c050e6b1c713add3836c40","Coinbase tx has unparseable output script"],
+[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"a4a4d6c6034da8aa06f01fe71f1fffbd79e032006b07f6c7a2c60a66aa310c01","0385acb4f0fe889ef0","3588f34fbbc11640f9ed40b2a66a4e096215d50389691309c1dac74d4268aa81","Includes witness data"]
+]
diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp
index 6a0aa864d9..26cf74830d 100644
--- a/src/test/streams_tests.cpp
+++ b/src/test/streams_tests.cpp
@@ -68,6 +68,86 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer)
vch.clear();
}
+BOOST_AUTO_TEST_CASE(streams_vector_reader)
+{
+ std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6};
+
+ VectorReader reader(SER_NETWORK, INIT_PROTO_VERSION, vch, 0);
+ BOOST_CHECK_EQUAL(reader.size(), 6);
+ BOOST_CHECK(!reader.empty());
+
+ // Read a single byte as an unsigned char.
+ unsigned char a;
+ reader >> a;
+ BOOST_CHECK_EQUAL(a, 1);
+ BOOST_CHECK_EQUAL(reader.size(), 5);
+ BOOST_CHECK(!reader.empty());
+
+ // Read a single byte as a signed char.
+ signed char b;
+ reader >> b;
+ BOOST_CHECK_EQUAL(b, -1);
+ BOOST_CHECK_EQUAL(reader.size(), 4);
+ BOOST_CHECK(!reader.empty());
+
+ // Read a 4 bytes as an unsigned int.
+ unsigned int c;
+ reader >> c;
+ BOOST_CHECK_EQUAL(c, 100992003); // 3,4,5,6 in little-endian base-256
+ BOOST_CHECK_EQUAL(reader.size(), 0);
+ BOOST_CHECK(reader.empty());
+
+ // Reading after end of byte vector throws an error.
+ signed int d;
+ BOOST_CHECK_THROW(reader >> d, std::ios_base::failure);
+
+ // Read a 4 bytes as a signed int from the beginning of the buffer.
+ reader.seek(-6);
+ reader >> d;
+ BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256
+ BOOST_CHECK_EQUAL(reader.size(), 2);
+ BOOST_CHECK(!reader.empty());
+
+ // Reading after end of byte vector throws an error even if the reader is
+ // not totally empty.
+ BOOST_CHECK_THROW(reader >> d, std::ios_base::failure);
+}
+
+BOOST_AUTO_TEST_CASE(bitstream_reader_writer)
+{
+ CDataStream data(SER_NETWORK, INIT_PROTO_VERSION);
+
+ BitStreamWriter<CDataStream> bit_writer(data);
+ bit_writer.Write(0, 1);
+ bit_writer.Write(2, 2);
+ bit_writer.Write(6, 3);
+ bit_writer.Write(11, 4);
+ bit_writer.Write(1, 5);
+ bit_writer.Write(32, 6);
+ bit_writer.Write(7, 7);
+ bit_writer.Write(30497, 16);
+ bit_writer.Flush();
+
+ CDataStream data_copy(data);
+ uint32_t serialized_int1;
+ data >> serialized_int1;
+ BOOST_CHECK_EQUAL(serialized_int1, (uint32_t)0x7700C35A); // NOTE: Serialized as LE
+ uint16_t serialized_int2;
+ data >> serialized_int2;
+ BOOST_CHECK_EQUAL(serialized_int2, (uint16_t)0x1072); // NOTE: Serialized as LE
+
+ BitStreamReader<CDataStream> bit_reader(data_copy);
+ BOOST_CHECK_EQUAL(bit_reader.Read(1), 0);
+ BOOST_CHECK_EQUAL(bit_reader.Read(2), 2);
+ BOOST_CHECK_EQUAL(bit_reader.Read(3), 6);
+ BOOST_CHECK_EQUAL(bit_reader.Read(4), 11);
+ BOOST_CHECK_EQUAL(bit_reader.Read(5), 1);
+ BOOST_CHECK_EQUAL(bit_reader.Read(6), 32);
+ BOOST_CHECK_EQUAL(bit_reader.Read(7), 7);
+ BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497);
+ BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure);
+}
+
BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
{
std::vector<char> in;
diff --git a/src/undo.h b/src/undo.h
index a1398b7624..4a78238944 100644
--- a/src/undo.h
+++ b/src/undo.h
@@ -6,6 +6,7 @@
#ifndef BITCOIN_UNDO_H
#define BITCOIN_UNDO_H
+#include <coins.h>
#include <compressor.h>
#include <consensus/consensus.h>
#include <primitives/transaction.h>
generated by cgit v1.2.3 (git 2.26.2) at 2024-10-08 19:33:56 +0000