aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarcoFalke <falke.marco@gmail.com>2019-04-18 09:48:14 -0400
committerMarcoFalke <falke.marco@gmail.com>2019-04-18 09:48:25 -0400
commite4beef611a423e9f836fa210a51634e94f14d830 (patch)
treeeafe6e014cc523ef97fb403ed273360596a12672
parentdae72998e8576439254ee497e82e2b9ce4917977 (diff)
parentc7efb652f3543b001b4dd22186a354605b14f47e (diff)
downloadbitcoin-e4beef611a423e9f836fa210a51634e94f14d830.tar.xz
Merge #14121: Index for BIP 157 block filters
c7efb652f3 blockfilter: Update BIP 158 test vectors. (Jim Posen) 19308c9e21 rpc: Add getblockfilter RPC method. (Jim Posen) ff35105096 init: Add CLI option to enable block filter index. (Jim Posen) accc8b8b18 index: Access functions for global block filter indexes. (Jim Posen) 2bc90e4e7b test: Unit test for block filter index reorg handling. (Jim Posen) 6bcf0998c0 test: Unit tests for block index filter. (Jim Posen) b5e8200db7 index: Implement lookup methods on block filter index. (Jim Posen) 75a76e3619 index: Implement block filter index with write operations. (Jim Posen) 2ad2338ef9 serialize: Serialization support for big-endian 32-bit ints. (Jim Posen) ba6ff9a6f7 blockfilter: Functions to translate filter types to/from names. (Jim Posen) 62b7a4f094 index: Ensure block locator is not stale after chain reorg. (Jim Posen) 4368384f1d index: Allow atomic commits of index state to be extended. (Jim Posen) Pull request description: This introduces a new BlockFilterIndex class, which is required for BIP 157 support. The index is uses the asynchronous BaseIndex infrastructure driven by the ValidationInterface callbacks. Filters are stored sequentially in flat files and the disk location of each filter is indexed in LevelDB along with the filter hash and header. The index is designed to ensure persistence of filters reorganized out of the main chain to simplify the BIP 157 net implementation. Stats (block height = 565500): - Syncing the index from scratch takes 45m - Total index size is 3.8 GiB ACKs for commit c7efb6: MarcoFalke: utACK c7efb652f3543b001b4dd22186a354605b14f47e ryanofsky: Slightly tested ACK c7efb652f3543b001b4dd22186a354605b14f47e (I just rebuilt the index with the updated PR and tested the RPC). Changes since last review: rebase, fixed compile errors in internal commits, new comments, updated error messages, tweaked cache size logic, renamed commit method, renamed constants and globals, fixed whitespace, extra BlockFilterIndex::Init error check. Tree-SHA512: f8ed7a9b6f76df45933aa5eba92b27b3af83f6df2ccb3728a5c89eec80f654344dc14f055f6f63eb9b3a7649dd8af6553fe14969889e7e2fd2f8461574d18f28
-rw-r--r--src/Makefile.am2
-rw-r--r--src/Makefile.test.include1
-rw-r--r--src/blockfilter.cpp58
-rw-r--r--src/blockfilter.h13
-rw-r--r--src/index/base.cpp63
-rw-r--r--src/index/base.h21
-rw-r--r--src/index/blockfilterindex.cpp467
-rw-r--r--src/index/blockfilterindex.h94
-rw-r--r--src/init.cpp45
-rw-r--r--src/rpc/blockchain.cpp82
-rw-r--r--src/serialize.h11
-rw-r--r--src/test/blockfilter_index_tests.cpp307
-rw-r--r--src/test/blockfilter_tests.cpp25
-rw-r--r--src/test/data/blockfilters.json10
-rw-r--r--src/txdb.h2
-rw-r--r--src/validation.h1
-rwxr-xr-xtest/functional/rpc_getblockfilter.py59
-rwxr-xr-xtest/functional/test_runner.py1
18 files changed, 1240 insertions, 22 deletions
diff --git a/src/Makefile.am b/src/Makefile.am
index 859c17afbd..059ed18134 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -134,6 +134,7 @@ BITCOIN_CORE_H = \
httprpc.h \
httpserver.h \
index/base.h \
+ index/blockfilterindex.h \
index/txindex.h \
indirectmap.h \
init.h \
@@ -263,6 +264,7 @@ libbitcoin_server_a_SOURCES = \
httprpc.cpp \
httpserver.cpp \
index/base.cpp \
+ index/blockfilterindex.cpp \
index/txindex.cpp \
interfaces/chain.cpp \
interfaces/node.cpp \
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index e248ef970f..692f8d97b3 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -92,6 +92,7 @@ BITCOIN_TESTS =\
test/blockchain_tests.cpp \
test/blockencodings_tests.cpp \
test/blockfilter_tests.cpp \
+ test/blockfilter_index_tests.cpp \
test/bloom_tests.cpp \
test/bswap_tests.cpp \
test/checkqueue_tests.cpp \
diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp
index e15213c552..787390be31 100644
--- a/src/blockfilter.cpp
+++ b/src/blockfilter.cpp
@@ -2,6 +2,9 @@
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#include <mutex>
+#include <sstream>
+
#include <blockfilter.h>
#include <crypto/siphash.h>
#include <hash.h>
@@ -15,6 +18,10 @@ static constexpr int GCS_SER_TYPE = SER_NETWORK;
/// Protocol version used to serialize parameters in GCS filter encoding.
static constexpr int GCS_SER_VERSION = 0;
+static const std::map<BlockFilterType, std::string> g_filter_types = {
+ {BlockFilterType::BASIC, "basic"},
+};
+
template <typename OStream>
static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x)
{
@@ -197,6 +204,57 @@ bool GCSFilter::MatchAny(const ElementSet& elements) const
return MatchInternal(queries.data(), queries.size());
}
+const std::string& BlockFilterTypeName(BlockFilterType filter_type)
+{
+ static std::string unknown_retval = "";
+ auto it = g_filter_types.find(filter_type);
+ return it != g_filter_types.end() ? it->second : unknown_retval;
+}
+
+bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type) {
+ for (const auto& entry : g_filter_types) {
+ if (entry.second == name) {
+ filter_type = entry.first;
+ return true;
+ }
+ }
+ return false;
+}
+
+const std::vector<BlockFilterType>& AllBlockFilterTypes()
+{
+ static std::vector<BlockFilterType> types;
+
+ static std::once_flag flag;
+ std::call_once(flag, []() {
+ types.reserve(g_filter_types.size());
+ for (auto entry : g_filter_types) {
+ types.push_back(entry.first);
+ }
+ });
+
+ return types;
+}
+
+const std::string& ListBlockFilterTypes()
+{
+ static std::string type_list;
+
+ static std::once_flag flag;
+ std::call_once(flag, []() {
+ std::stringstream ret;
+ bool first = true;
+ for (auto entry : g_filter_types) {
+ if (!first) ret << ", ";
+ ret << entry.second;
+ first = false;
+ }
+ type_list = ret.str();
+ });
+
+ return type_list;
+}
+
static GCSFilter::ElementSet BasicFilterElements(const CBlock& block,
const CBlockUndo& block_undo)
{
diff --git a/src/blockfilter.h b/src/blockfilter.h
index e5e087ed5a..914b94fec1 100644
--- a/src/blockfilter.h
+++ b/src/blockfilter.h
@@ -6,6 +6,7 @@
#define BITCOIN_BLOCKFILTER_H
#include <stdint.h>
+#include <string>
#include <unordered_set>
#include <vector>
@@ -89,6 +90,18 @@ enum class BlockFilterType : uint8_t
INVALID = 255,
};
+/** Get the human-readable name for a filter type. Returns empty string for unknown types. */
+const std::string& BlockFilterTypeName(BlockFilterType filter_type);
+
+/** Find a filter type by its human-readable name. */
+bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type);
+
+/** Get a list of known filter types. */
+const std::vector<BlockFilterType>& AllBlockFilterTypes();
+
+/** Get a comma-separated list of known filter type names. */
+const std::string& ListBlockFilterTypes();
+
/**
* Complete block filter struct as defined in BIP 157. Serialization matches
* payload of "cfilter" messages.
diff --git a/src/index/base.cpp b/src/index/base.cpp
index f6f59572ce..9e48f0bd27 100644
--- a/src/index/base.cpp
+++ b/src/index/base.cpp
@@ -41,9 +41,9 @@ bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
return success;
}
-bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator)
+void BaseIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator)
{
- return Write(DB_BEST_BLOCK, locator);
+ batch.Write(DB_BEST_BLOCK, locator);
}
BaseIndex::~BaseIndex()
@@ -95,7 +95,11 @@ void BaseIndex::ThreadSync()
int64_t last_locator_write_time = 0;
while (true) {
if (m_interrupt) {
- WriteBestBlock(pindex);
+ m_best_block_index = pindex;
+ // No need to handle errors in Commit. If it fails, the error will be already be
+ // logged. The best way to recover is to continue, as index cannot be corrupted by
+ // a missed commit to disk for an advanced index state.
+ Commit();
return;
}
@@ -103,11 +107,17 @@ void BaseIndex::ThreadSync()
LOCK(cs_main);
const CBlockIndex* pindex_next = NextSyncBlock(pindex);
if (!pindex_next) {
- WriteBestBlock(pindex);
m_best_block_index = pindex;
m_synced = true;
+ // No need to handle errors in Commit. See rationale above.
+ Commit();
break;
}
+ if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) {
+ FatalError("%s: Failed to rewind index %s to a previous chain tip",
+ __func__, GetName());
+ return;
+ }
pindex = pindex_next;
}
@@ -119,8 +129,10 @@ void BaseIndex::ThreadSync()
}
if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
- WriteBestBlock(pindex);
+ m_best_block_index = pindex;
last_locator_write_time = current_time;
+ // No need to handle errors in Commit. See rationale above.
+ Commit();
}
CBlock block;
@@ -144,12 +156,35 @@ void BaseIndex::ThreadSync()
}
}
-bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index)
+bool BaseIndex::Commit()
+{
+ CDBBatch batch(GetDB());
+ if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) {
+ return error("%s: Failed to commit latest %s state", __func__, GetName());
+ }
+ return true;
+}
+
+bool BaseIndex::CommitInternal(CDBBatch& batch)
{
LOCK(cs_main);
- if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) {
- return error("%s: Failed to write locator to disk", __func__);
+ GetDB().WriteBestBlock(batch, chainActive.GetLocator(m_best_block_index));
+ return true;
+}
+
+bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
+{
+ assert(current_tip == m_best_block_index);
+ assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
+
+ // In the case of a reorg, ensure persisted block locator is not stale.
+ m_best_block_index = new_tip;
+ if (!Commit()) {
+ // If commit fails, revert the best block index to avoid corruption.
+ m_best_block_index = current_tip;
+ return false;
}
+
return true;
}
@@ -180,6 +215,11 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const
best_block_index->GetBlockHash().ToString());
return;
}
+ if (best_block_index != pindex->pprev && !Rewind(best_block_index, pindex->pprev)) {
+ FatalError("%s: Failed to rewind index %s to a previous chain tip",
+ __func__, GetName());
+ return;
+ }
}
if (WriteBlock(*block, pindex)) {
@@ -224,9 +264,10 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
return;
}
- if (!GetDB().WriteBestBlock(locator)) {
- error("%s: Failed to write locator to disk", __func__);
- }
+ // No need to handle errors in Commit. If it fails, the error will be already be logged. The
+ // best way to recover is to continue, as index cannot be corrupted by a missed commit to disk
+ // for an advanced index state.
+ Commit();
}
bool BaseIndex::BlockUntilSyncedToCurrentChain()
diff --git a/src/index/base.h b/src/index/base.h
index 04ee6e6cc2..31acbed0c1 100644
--- a/src/index/base.h
+++ b/src/index/base.h
@@ -32,7 +32,7 @@ protected:
bool ReadBestBlock(CBlockLocator& locator) const;
/// Write block locator of the chain that the txindex is in sync with.
- bool WriteBestBlock(const CBlockLocator& locator);
+ void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator);
};
private:
@@ -54,8 +54,15 @@ private:
/// over and the sync thread exits.
void ThreadSync();
- /// Write the current chain block locator to the DB.
- bool WriteBestBlock(const CBlockIndex* block_index);
+ /// Write the current index state (eg. chain block locator and subclass-specific items) to disk.
+ ///
+ /// Recommendations for error handling:
+ /// If called on a successor of the previous committed best block in the index, the index can
+ /// continue processing without risk of corruption, though the index state will need to catch up
+ /// from further behind on reboot. If the new state is not a successor of the previous state (due
+ /// to a chain reorganization), the index must halt until Commit succeeds or else it could end up
+ /// getting corrupted.
+ bool Commit();
protected:
void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
@@ -69,6 +76,14 @@ protected:
/// Write update index entries for a newly connected block.
virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
+ /// Virtual method called internally by Commit that can be overridden to atomically
+ /// commit more index state.
+ virtual bool CommitInternal(CDBBatch& batch);
+
+ /// Rewind index to an earlier chain tip during a chain reorg. The tip must
+ /// be an ancestor of the current best block.
+ virtual bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip);
+
virtual DB& GetDB() const = 0;
/// Get the name of the index for display in logs.
diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp
new file mode 100644
index 0000000000..20f33baf2c
--- /dev/null
+++ b/src/index/blockfilterindex.cpp
@@ -0,0 +1,467 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <map>
+
+#include <dbwrapper.h>
+#include <index/blockfilterindex.h>
+#include <util/system.h>
+#include <validation.h>
+
+/* The index database stores three items for each block: the disk location of the encoded filter,
+ * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by
+ * height, and those belonging to blocks that have been reorganized out of the active chain are
+ * indexed by block hash. This ensures that filter data for any block that becomes part of the
+ * active chain can always be retrieved, alleviating timing concerns.
+ *
+ * The filters themselves are stored in flat files and referenced by the LevelDB entries. This
+ * minimizes the amount of data written to LevelDB and keeps the database values constant size. The
+ * disk location of the next block filter to be written (represented as a FlatFilePos) is stored
+ * under the DB_FILTER_POS key.
+ *
+ * Keys for the height index have the type [DB_BLOCK_HEIGHT, uint32 (BE)]. The height is represented
+ * as big-endian so that sequential reads of filters by height are fast.
+ * Keys for the hash index have the type [DB_BLOCK_HASH, uint256].
+ */
+constexpr char DB_BLOCK_HASH = 's';
+constexpr char DB_BLOCK_HEIGHT = 't';
+constexpr char DB_FILTER_POS = 'P';
+
+constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB
+/** The pre-allocation chunk size for fltr?????.dat files */
+constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB
+
+namespace {
+
+struct DBVal {
+ uint256 hash;
+ uint256 header;
+ FlatFilePos pos;
+
+ ADD_SERIALIZE_METHODS;
+
+ template <typename Stream, typename Operation>
+ inline void SerializationOp(Stream& s, Operation ser_action) {
+ READWRITE(hash);
+ READWRITE(header);
+ READWRITE(pos);
+ }
+};
+
+struct DBHeightKey {
+ int height;
+
+ DBHeightKey() : height(0) {}
+ DBHeightKey(int height_in) : height(height_in) {}
+
+ template<typename Stream>
+ void Serialize(Stream& s) const
+ {
+ ser_writedata8(s, DB_BLOCK_HEIGHT);
+ ser_writedata32be(s, height);
+ }
+
+ template<typename Stream>
+ void Unserialize(Stream& s)
+ {
+ char prefix = ser_readdata8(s);
+ if (prefix != DB_BLOCK_HEIGHT) {
+ throw std::ios_base::failure("Invalid format for block filter index DB height key");
+ }
+ height = ser_readdata32be(s);
+ }
+};
+
+struct DBHashKey {
+ uint256 hash;
+
+ DBHashKey(const uint256& hash_in) : hash(hash_in) {}
+
+ ADD_SERIALIZE_METHODS;
+
+ template <typename Stream, typename Operation>
+ inline void SerializationOp(Stream& s, Operation ser_action) {
+ char prefix = DB_BLOCK_HASH;
+ READWRITE(prefix);
+ if (prefix != DB_BLOCK_HASH) {
+ throw std::ios_base::failure("Invalid format for block filter index DB hash key");
+ }
+
+ READWRITE(hash);
+ }
+};
+
+}; // namespace
+
+static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes;
+
+BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type,
+ size_t n_cache_size, bool f_memory, bool f_wipe)
+ : m_filter_type(filter_type)
+{
+ const std::string& filter_name = BlockFilterTypeName(filter_type);
+ if (filter_name.empty()) throw std::invalid_argument("unknown filter_type");
+
+ fs::path path = GetDataDir() / "indexes" / "blockfilter" / filter_name;
+ fs::create_directories(path);
+
+ m_name = filter_name + " block filter index";
+ m_db = MakeUnique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
+ m_filter_fileseq = MakeUnique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE);
+}
+
+bool BlockFilterIndex::Init()
+{
+ if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) {
+ // Check that the cause of the read failure is that the key does not exist. Any other errors
+ // indicate database corruption or a disk failure, and starting the index would cause
+ // further corruption.
+ if (m_db->Exists(DB_FILTER_POS)) {
+ return error("%s: Cannot read current %s state; index may be corrupted",
+ __func__, GetName());
+ }
+
+ // If the DB_FILTER_POS is not set, then initialize to the first location.
+ m_next_filter_pos.nFile = 0;
+ m_next_filter_pos.nPos = 0;
+ }
+ return BaseIndex::Init();
+}
+
+bool BlockFilterIndex::CommitInternal(CDBBatch& batch)
+{
+ const FlatFilePos& pos = m_next_filter_pos;
+
+ // Flush current filter file to disk.
+ CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
+ if (file.IsNull()) {
+ return error("%s: Failed to open filter file %d", __func__, pos.nFile);
+ }
+ if (!FileCommit(file.Get())) {
+ return error("%s: Failed to commit filter file %d", __func__, pos.nFile);
+ }
+
+ batch.Write(DB_FILTER_POS, pos);
+ return BaseIndex::CommitInternal(batch);
+}
+
+bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const
+{
+ CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION);
+ if (filein.IsNull()) {
+ return false;
+ }
+
+ uint256 block_hash;
+ std::vector<unsigned char> encoded_filter;
+ try {
+ filein >> block_hash >> encoded_filter;
+ filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));
+ }
+ catch (const std::exception& e) {
+ return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what());
+ }
+
+ return true;
+}
+
+size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter)
+{
+ assert(filter.GetFilterType() == GetFilterType());
+
+ size_t data_size =
+ GetSerializeSize(filter.GetBlockHash(), CLIENT_VERSION) +
+ GetSerializeSize(filter.GetEncodedFilter(), CLIENT_VERSION);
+
+ // If writing the filter would overflow the file, flush and move to the next one.
+ if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) {
+ CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
+ if (last_file.IsNull()) {
+ LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
+ return 0;
+ }
+ if (!TruncateFile(last_file.Get(), pos.nPos)) {
+ LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile);
+ return 0;
+ }
+ if (!FileCommit(last_file.Get())) {
+ LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile);
+ return 0;
+ }
+
+ pos.nFile++;
+ pos.nPos = 0;
+ }
+
+ // Pre-allocate sufficient space for filter data.
+ bool out_of_space;
+ m_filter_fileseq->Allocate(pos, data_size, out_of_space);
+ if (out_of_space) {
+ LogPrintf("%s: out of disk space\n", __func__);
+ return 0;
+ }
+
+ CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
+ if (fileout.IsNull()) {
+ LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
+ return 0;
+ }
+
+ fileout << filter.GetBlockHash() << filter.GetEncodedFilter();
+ return data_size;
+}
+
+bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
+{
+ CBlockUndo block_undo;
+ uint256 prev_header;
+
+ if (pindex->nHeight > 0) {
+ if (!UndoReadFromDisk(block_undo, pindex)) {
+ return false;
+ }
+
+ std::pair<uint256, DBVal> read_out;
+ if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
+ return false;
+ }
+
+ uint256 expected_block_hash = pindex->pprev->GetBlockHash();
+ if (read_out.first != expected_block_hash) {
+ return error("%s: previous block header belongs to unexpected block %s; expected %s",
+ __func__, read_out.first.ToString(), expected_block_hash.ToString());
+ }
+
+ prev_header = read_out.second.header;
+ }
+
+ BlockFilter filter(m_filter_type, block, block_undo);
+
+ size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter);
+ if (bytes_written == 0) return false;
+
+ std::pair<uint256, DBVal> value;
+ value.first = pindex->GetBlockHash();
+ value.second.hash = filter.GetHash();
+ value.second.header = filter.ComputeHeader(prev_header);
+ value.second.pos = m_next_filter_pos;
+
+ if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) {
+ return false;
+ }
+
+ m_next_filter_pos.nPos += bytes_written;
+ return true;
+}
+
+static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch,
+ const std::string& index_name,
+ int start_height, int stop_height)
+{
+ DBHeightKey key(start_height);
+ db_it.Seek(key);
+
+ for (int height = start_height; height <= stop_height; ++height) {
+ if (!db_it.GetKey(key) || key.height != height) {
+ return error("%s: unexpected key in %s: expected (%c, %d)",
+ __func__, index_name, DB_BLOCK_HEIGHT, height);
+ }
+
+ std::pair<uint256, DBVal> value;
+ if (!db_it.GetValue(value)) {
+ return error("%s: unable to read value in %s at key (%c, %d)",
+ __func__, index_name, DB_BLOCK_HEIGHT, height);
+ }
+
+ batch.Write(DBHashKey(value.first), std::move(value.second));
+
+ db_it.Next();
+ }
+ return true;
+}
+
+bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
+{
+ assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
+
+ CDBBatch batch(*m_db);
+ std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
+
+ // During a reorg, we need to copy all filters for blocks that are getting disconnected from the
+ // height index to the hash index so we can still find them when the height index entries are
+ // overwritten.
+ if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) {
+ return false;
+ }
+
+ // The latest filter position gets written in Commit by the call to the BaseIndex::Rewind.
+ // But since this creates new references to the filter, the position should get updated here
+ // atomically as well in case Commit fails.
+ batch.Write(DB_FILTER_POS, m_next_filter_pos);
+ if (!m_db->WriteBatch(batch)) return false;
+
+ return BaseIndex::Rewind(current_tip, new_tip);
+}
+
+static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result)
+{
+ // First check if the result is stored under the height index and the value there matches the
+ // block hash. This should be the case if the block is on the active chain.
+ std::pair<uint256, DBVal> read_out;
+ if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) {
+ return false;
+ }
+ if (read_out.first == block_index->GetBlockHash()) {
+ result = std::move(read_out.second);
+ return true;
+ }
+
+ // If value at the height index corresponds to an different block, the result will be stored in
+ // the hash index.
+ return db.Read(DBHashKey(block_index->GetBlockHash()), result);
+}
+
+static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start_height,
+ const CBlockIndex* stop_index, std::vector<DBVal>& results)
+{
+ if (start_height < 0) {
+ return error("%s: start height (%d) is negative", __func__, start_height);
+ }
+ if (start_height > stop_index->nHeight) {
+ return error("%s: start height (%d) is greater than stop height (%d)",
+ __func__, start_height, stop_index->nHeight);
+ }
+
+ size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1);
+ std::vector<std::pair<uint256, DBVal>> values(results_size);
+
+ DBHeightKey key(start_height);
+ std::unique_ptr<CDBIterator> db_it(db.NewIterator());
+ db_it->Seek(DBHeightKey(start_height));
+ for (int height = start_height; height <= stop_index->nHeight; ++height) {
+ if (!db_it->Valid() || !db_it->GetKey(key) || key.height != height) {
+ return false;
+ }
+
+ size_t i = static_cast<size_t>(height - start_height);
+ if (!db_it->GetValue(values[i])) {
+ return error("%s: unable to read value in %s at key (%c, %d)",
+ __func__, index_name, DB_BLOCK_HEIGHT, height);
+ }
+
+ db_it->Next();
+ }
+
+ results.resize(results_size);
+
+ // Iterate backwards through block indexes collecting results in order to access the block hash
+ // of each entry in case we need to look it up in the hash index.
+ for (const CBlockIndex* block_index = stop_index;
+ block_index && block_index->nHeight >= start_height;
+ block_index = block_index->pprev) {
+ uint256 block_hash = block_index->GetBlockHash();
+
+ size_t i = static_cast<size_t>(block_index->nHeight - start_height);
+ if (block_hash == values[i].first) {
+ results[i] = std::move(values[i].second);
+ continue;
+ }
+
+ if (!db.Read(DBHashKey(block_hash), results[i])) {
+ return error("%s: unable to read value in %s at key (%c, %s)",
+ __func__, index_name, DB_BLOCK_HASH, block_hash.ToString());
+ }
+ }
+
+ return true;
+}
+
+bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const
+{
+ DBVal entry;
+ if (!LookupOne(*m_db, block_index, entry)) {
+ return false;
+ }
+
+ return ReadFilterFromDisk(entry.pos, filter_out);
+}
+
+bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const
+{
+ DBVal entry;
+ if (!LookupOne(*m_db, block_index, entry)) {
+ return false;
+ }
+
+ header_out = entry.header;
+ return true;
+}
+
+bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* stop_index,
+ std::vector<BlockFilter>& filters_out) const
+{
+ std::vector<DBVal> entries;
+ if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
+ return false;
+ }
+
+ filters_out.resize(entries.size());
+ auto filter_pos_it = filters_out.begin();
+ for (const auto& entry : entries) {
+ if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) {
+ return false;
+ }
+ ++filter_pos_it;
+ }
+
+ return true;
+}
+
+bool BlockFilterIndex::LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
+ std::vector<uint256>& hashes_out) const
+
+{
+ std::vector<DBVal> entries;
+ if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
+ return false;
+ }
+
+ hashes_out.clear();
+ hashes_out.reserve(entries.size());
+ for (const auto& entry : entries) {
+ hashes_out.push_back(entry.hash);
+ }
+ return true;
+}
+
+BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type)
+{
+ auto it = g_filter_indexes.find(filter_type);
+ return it != g_filter_indexes.end() ? &it->second : nullptr;
+}
+
+void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn)
+{
+ for (auto& entry : g_filter_indexes) fn(entry.second);
+}
+
+bool InitBlockFilterIndex(BlockFilterType filter_type,
+ size_t n_cache_size, bool f_memory, bool f_wipe)
+{
+ auto result = g_filter_indexes.emplace(std::piecewise_construct,
+ std::forward_as_tuple(filter_type),
+ std::forward_as_tuple(filter_type,
+ n_cache_size, f_memory, f_wipe));
+ return result.second;
+}
+
+bool DestroyBlockFilterIndex(BlockFilterType filter_type)
+{
+ return g_filter_indexes.erase(filter_type);
+}
+
+void DestroyAllBlockFilterIndexes()
+{
+ g_filter_indexes.clear();
+}
diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h
new file mode 100644
index 0000000000..436d52515f
--- /dev/null
+++ b/src/index/blockfilterindex.h
@@ -0,0 +1,94 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_INDEX_BLOCKFILTERINDEX_H
+#define BITCOIN_INDEX_BLOCKFILTERINDEX_H
+
+#include <blockfilter.h>
+#include <chain.h>
+#include <flatfile.h>
+#include <index/base.h>
+
+/**
+ * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of
+ * blocks by height. An index is constructed for each supported filter type with its own database
+ * (ie. filter data for different types are stored in separate databases).
+ *
+ * This index is used to serve BIP 157 net requests.
+ */
+class BlockFilterIndex final : public BaseIndex
+{
+private:
+ BlockFilterType m_filter_type;
+ std::string m_name;
+ std::unique_ptr<BaseIndex::DB> m_db;
+
+ FlatFilePos m_next_filter_pos;
+ std::unique_ptr<FlatFileSeq> m_filter_fileseq;
+
+ bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const;
+ size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter);
+
+protected:
+ bool Init() override;
+
+ bool CommitInternal(CDBBatch& batch) override;
+
+ bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
+
+ bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override;
+
+ BaseIndex::DB& GetDB() const override { return *m_db; }
+
+ const char* GetName() const override { return m_name.c_str(); }
+
+public:
+ /** Constructs the index, which becomes available to be queried. */
+ explicit BlockFilterIndex(BlockFilterType filter_type,
+ size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
+
+ BlockFilterType GetFilterType() const { return m_filter_type; }
+
+ /** Get a single filter by block. */
+ bool LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const;
+
+ /** Get a single filter header by block. */
+ bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const;
+
+ /** Get a range of filters between two heights on a chain. */
+ bool LookupFilterRange(int start_height, const CBlockIndex* stop_index,
+ std::vector<BlockFilter>& filters_out) const;
+
+ /** Get a range of filter hashes between two heights on a chain. */
+ bool LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
+ std::vector<uint256>& hashes_out) const;
+};
+
+/**
+ * Get a block filter index by type. Returns nullptr if index has not been initialized or was
+ * already destroyed.
+ */
+BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type);
+
+/** Iterate over all running block filter indexes, invoking fn on each. */
+void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn);
+
+/**
+ * Initialize a block filter index for the given type if one does not already exist. Returns true if
+ * a new index is created and false if one has already been initialized.
+ */
+bool InitBlockFilterIndex(BlockFilterType filter_type,
+ size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
+
+/**
+ * Destroy the block filter index with the given type. Returns false if no such index exists. This
+ * just releases the allocated memory and closes the database connection, it does not delete the
+ * index data.
+ */
+bool DestroyBlockFilterIndex(BlockFilterType filter_type);
+
+/** Destroy all open block filter indexes. */
+void DestroyAllBlockFilterIndexes();
+
+#endif // BITCOIN_INDEX_BLOCKFILTERINDEX_H
diff --git a/src/init.cpp b/src/init.cpp
index 2b538fb836..70459994c7 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -12,6 +12,7 @@
#include <addrman.h>
#include <amount.h>
#include <banman.h>
+#include <blockfilter.h>
#include <chain.h>
#include <chainparams.h>
#include <checkpoints.h>
@@ -20,6 +21,7 @@
#include <fs.h>
#include <httpserver.h>
#include <httprpc.h>
+#include <index/blockfilterindex.h>
#include <interfaces/chain.h>
#include <index/txindex.h>
#include <key.h>
@@ -191,6 +193,7 @@ void Interrupt()
if (g_txindex) {
g_txindex->Interrupt();
}
+ ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); });
}
void Shutdown(InitInterfaces& interfaces)
@@ -222,6 +225,7 @@ void Shutdown(InitInterfaces& interfaces)
if (peerLogic) UnregisterValidationInterface(peerLogic.get());
if (g_connman) g_connman->Stop();
if (g_txindex) g_txindex->Stop();
+ ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); });
StopTorControl();
@@ -236,6 +240,7 @@ void Shutdown(InitInterfaces& interfaces)
g_connman.reset();
g_banman.reset();
g_txindex.reset();
+ DestroyAllBlockFilterIndexes();
if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
DumpMempool();
@@ -406,6 +411,10 @@ void SetupServerArgs()
hidden_args.emplace_back("-sysperms");
#endif
gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS);
+ gArgs.AddArg("-blockfilterindex=<type>",
+ strprintf("Maintain an index of compact filters by block (default: %s, values: %s).", DEFAULT_BLOCKFILTERINDEX, ListBlockFilterTypes()) +
+ " If <type> is not supplied or if <type> = 1, indexes for all known types are enabled.",
+ false, OptionsCategory::OPTIONS);
gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION);
@@ -875,6 +884,7 @@ int nUserMaxConnections;
int nFD;
ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED);
int64_t peer_connect_timeout;
+std::vector<BlockFilterType> g_enabled_filter_types;
} // namespace
@@ -956,10 +966,29 @@ bool AppInitParameterInteraction()
return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", "").c_str()));
}
+ // parse and validate enabled filter types
+ std::string blockfilterindex_value = gArgs.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX);
+ if (blockfilterindex_value == "" || blockfilterindex_value == "1") {
+ g_enabled_filter_types = AllBlockFilterTypes();
+ } else if (blockfilterindex_value != "0") {
+ const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex");
+ g_enabled_filter_types.reserve(names.size());
+ for (const auto& name : names) {
+ BlockFilterType filter_type;
+ if (!BlockFilterTypeByName(name, filter_type)) {
+ return InitError(strprintf(_("Unknown -blockfilterindex value %s."), name));
+ }
+ g_enabled_filter_types.push_back(filter_type);
+ }
+ }
+
// if using block pruning, then disallow txindex
if (gArgs.GetArg("-prune", 0)) {
if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX))
return InitError(_("Prune mode is incompatible with -txindex."));
+ if (!g_enabled_filter_types.empty()) {
+ return InitError(_("Prune mode is incompatible with -blockfilterindex."));
+ }
}
// -bind and -whitebind can't be set when not listening
@@ -1450,6 +1479,13 @@ bool AppInitMain(InitInterfaces& interfaces)
nTotalCache -= nBlockTreeDBCache;
int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
nTotalCache -= nTxIndexCache;
+ int64_t filter_index_cache = 0;
+ if (!g_enabled_filter_types.empty()) {
+ size_t n_indexes = g_enabled_filter_types.size();
+ int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20);
+ filter_index_cache = max_cache / n_indexes;
+ nTotalCache -= filter_index_cache * n_indexes;
+ }
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
nTotalCache -= nCoinDBCache;
@@ -1460,6 +1496,10 @@ bool AppInitMain(InitInterfaces& interfaces)
if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
LogPrintf("* Using %.1f MiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024));
}
+ for (BlockFilterType filter_type : g_enabled_filter_types) {
+ LogPrintf("* Using %.1f MiB for %s block filter index database\n",
+ filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
+ }
LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
@@ -1647,6 +1687,11 @@ bool AppInitMain(InitInterfaces& interfaces)
g_txindex->Start();
}
+ for (const auto& filter_type : g_enabled_filter_types) {
+ InitBlockFilterIndex(filter_type, filter_index_cache, false, fReindex);
+ GetBlockFilterIndex(filter_type)->Start();
+ }
+
// ********************************************************* Step 9: load wallet
for (const auto& client : interfaces.chain_clients) {
if (!client->load()) {
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 1cc496c733..c85c2bcc92 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -7,6 +7,7 @@
#include <amount.h>
#include <base58.h>
+#include <blockfilter.h>
#include <chain.h>
#include <chainparams.h>
#include <checkpoints.h>
@@ -14,6 +15,7 @@
#include <consensus/validation.h>
#include <core_io.h>
#include <hash.h>
+#include <index/blockfilterindex.h>
#include <index/txindex.h>
#include <key_io.h>
#include <policy/feerate.h>
@@ -2299,6 +2301,85 @@ UniValue scantxoutset(const JSONRPCRequest& request)
return result;
}
+static UniValue getblockfilter(const JSONRPCRequest& request)
+{
+ if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) {
+ throw std::runtime_error(
+ RPCHelpMan{"getblockfilter",
+ "\nRetrieve a BIP 157 content filter for a particular block.\n",
+ {
+ {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hash of the block"},
+ {"filtertype", RPCArg::Type::STR, /*default*/ "basic", "The type name of the filter"},
+ },
+ RPCResult{
+ "{\n"
+ " \"filter\" : (string) the hex-encoded filter data\n"
+ " \"header\" : (string) the hex-encoded filter header\n"
+ "}\n"
+ },
+ RPCExamples{
+ HelpExampleCli("getblockfilter", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\" \"basic\"")
+ }
+ }.ToString()
+ );
+ }
+
+ uint256 block_hash = ParseHashV(request.params[0], "blockhash");
+ std::string filtertype_name = "basic";
+ if (!request.params[1].isNull()) {
+ filtertype_name = request.params[1].get_str();
+ }
+
+ BlockFilterType filtertype;
+ if (!BlockFilterTypeByName(filtertype_name, filtertype)) {
+ throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype");
+ }
+
+ BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
+ if (!index) {
+ throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name);
+ }
+
+ const CBlockIndex* block_index;
+ bool block_was_connected;
+ {
+ LOCK(cs_main);
+ block_index = LookupBlockIndex(block_hash);
+ if (!block_index) {
+ throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
+ }
+ block_was_connected = block_index->IsValid(BLOCK_VALID_SCRIPTS);
+ }
+
+ bool index_ready = index->BlockUntilSyncedToCurrentChain();
+
+ BlockFilter filter;
+ uint256 filter_header;
+ if (!index->LookupFilter(block_index, filter) ||
+ !index->LookupFilterHeader(block_index, filter_header)) {
+ int err_code;
+ std::string errmsg = "Filter not found.";
+
+ if (!block_was_connected) {
+ err_code = RPC_INVALID_ADDRESS_OR_KEY;
+ errmsg += " Block was not connected to active chain.";
+ } else if (!index_ready) {
+ err_code = RPC_MISC_ERROR;
+ errmsg += " Block filters are still in the process of being indexed.";
+ } else {
+ err_code = RPC_INTERNAL_ERROR;
+ errmsg += " This error is unexpected and indicates index corruption.";
+ }
+
+ throw JSONRPCError(err_code, errmsg);
+ }
+
+ UniValue ret(UniValue::VOBJ);
+ ret.pushKV("filter", HexStr(filter.GetEncodedFilter()));
+ ret.pushKV("header", filter_header.GetHex());
+ return ret;
+}
+
// clang-format off
static const CRPCCommand commands[] =
{ // category name actor (function) argNames
@@ -2326,6 +2407,7 @@ static const CRPCCommand commands[] =
{ "blockchain", "preciousblock", &preciousblock, {"blockhash"} },
{ "blockchain", "scantxoutset", &scantxoutset, {"action", "scanobjects"} },
+ { "blockchain", "getblockfilter", &getblockfilter, {"blockhash", "filtertype"} },
/* Not shown in help */
{ "hidden", "invalidateblock", &invalidateblock, {"blockhash"} },
diff --git a/src/serialize.h b/src/serialize.h
index 2d0cfbbbf0..b001ee1324 100644
--- a/src/serialize.h
+++ b/src/serialize.h
@@ -89,6 +89,11 @@ template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
obj = htole32(obj);
s.write((char*)&obj, 4);
}
+template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
+{
+ obj = htobe32(obj);
+ s.write((char*)&obj, 4);
+}
template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
{
obj = htole64(obj);
@@ -118,6 +123,12 @@ template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
s.read((char*)&obj, 4);
return le32toh(obj);
}
+template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
+{
+ uint32_t obj;
+ s.read((char*)&obj, 4);
+ return be32toh(obj);
+}
template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
{
uint64_t obj;
diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp
new file mode 100644
index 0000000000..f0a24777a6
--- /dev/null
+++ b/src/test/blockfilter_index_tests.cpp
@@ -0,0 +1,307 @@
+// Copyright (c) 2017-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <blockfilter.h>
+#include <chainparams.h>
+#include <consensus/validation.h>
+#include <index/blockfilterindex.h>
+#include <miner.h>
+#include <pow.h>
+#include <test/test_bitcoin.h>
+#include <script/standard.h>
+#include <validation.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(blockfilter_index_tests)
+
+static bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index,
+ BlockFilter& filter)
+{
+ CBlock block;
+ if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) {
+ return false;
+ }
+
+ CBlockUndo block_undo;
+ if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) {
+ return false;
+ }
+
+ filter = BlockFilter(filter_type, block, block_undo);
+ return true;
+}
+
+static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index,
+ uint256& last_header)
+{
+ BlockFilter expected_filter;
+ if (!ComputeFilter(filter_index.GetFilterType(), block_index, expected_filter)) {
+ BOOST_ERROR("ComputeFilter failed on block " << block_index->nHeight);
+ return false;
+ }
+
+ BlockFilter filter;
+ uint256 filter_header;
+ std::vector<BlockFilter> filters;
+ std::vector<uint256> filter_hashes;
+
+ BOOST_CHECK(filter_index.LookupFilter(block_index, filter));
+ BOOST_CHECK(filter_index.LookupFilterHeader(block_index, filter_header));
+ BOOST_CHECK(filter_index.LookupFilterRange(block_index->nHeight, block_index, filters));
+ BOOST_CHECK(filter_index.LookupFilterHashRange(block_index->nHeight, block_index,
+ filter_hashes));
+
+ BOOST_CHECK_EQUAL(filters.size(), 1);
+ BOOST_CHECK_EQUAL(filter_hashes.size(), 1);
+
+ BOOST_CHECK_EQUAL(filter.GetHash(), expected_filter.GetHash());
+ BOOST_CHECK_EQUAL(filter_header, expected_filter.ComputeHeader(last_header));
+ BOOST_CHECK_EQUAL(filters[0].GetHash(), expected_filter.GetHash());
+ BOOST_CHECK_EQUAL(filter_hashes[0], expected_filter.GetHash());
+
+ filters.clear();
+ filter_hashes.clear();
+ last_header = filter_header;
+ return true;
+}
+
+static CBlock CreateBlock(const CBlockIndex* prev,
+ const std::vector<CMutableTransaction>& txns,
+ const CScript& scriptPubKey)
+{
+ const CChainParams& chainparams = Params();
+ std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey);
+ CBlock& block = pblocktemplate->block;
+ block.hashPrevBlock = prev->GetBlockHash();
+ block.nTime = prev->nTime + 1;
+
+ // Replace mempool-selected txns with just coinbase plus passed-in txns:
+ block.vtx.resize(1);
+ for (const CMutableTransaction& tx : txns) {
+ block.vtx.push_back(MakeTransactionRef(tx));
+ }
+ // IncrementExtraNonce creates a valid coinbase and merkleRoot
+ unsigned int extraNonce = 0;
+ IncrementExtraNonce(&block, prev, extraNonce);
+
+ while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce;
+
+ return block;
+}
+
+static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key,
+ size_t length, std::vector<std::shared_ptr<CBlock>>& chain)
+{
+ std::vector<CMutableTransaction> no_txns;
+
+ chain.resize(length);
+ for (auto& block : chain) {
+ block = std::make_shared<CBlock>(CreateBlock(pindex, no_txns, coinbase_script_pub_key));
+ CBlockHeader header = block->GetBlockHeader();
+
+ CValidationState state;
+ if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex, nullptr)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, TestChain100Setup)
+{
+ BlockFilterIndex filter_index(BlockFilterType::BASIC, 1 << 20, true);
+
+ uint256 last_header;
+
+ // Filter should not be found in the index before it is started.
+ {
+ LOCK(cs_main);
+
+ BlockFilter filter;
+ uint256 filter_header;
+ std::vector<BlockFilter> filters;
+ std::vector<uint256> filter_hashes;
+
+ for (const CBlockIndex* block_index = chainActive.Genesis();
+ block_index != nullptr;
+ block_index = chainActive.Next(block_index)) {
+ BOOST_CHECK(!filter_index.LookupFilter(block_index, filter));
+ BOOST_CHECK(!filter_index.LookupFilterHeader(block_index, filter_header));
+ BOOST_CHECK(!filter_index.LookupFilterRange(block_index->nHeight, block_index, filters));
+ BOOST_CHECK(!filter_index.LookupFilterHashRange(block_index->nHeight, block_index,
+ filter_hashes));
+ }
+ }
+
+ // BlockUntilSyncedToCurrentChain should return false before index is started.
+ BOOST_CHECK(!filter_index.BlockUntilSyncedToCurrentChain());
+
+ filter_index.Start();
+
+ // Allow filter index to catch up with the block index.
+ constexpr int64_t timeout_ms = 10 * 1000;
+ int64_t time_start = GetTimeMillis();
+ while (!filter_index.BlockUntilSyncedToCurrentChain()) {
+ BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis());
+ MilliSleep(100);
+ }
+
+ // Check that filter index has all blocks that were in the chain before it started.
+ {
+ LOCK(cs_main);
+ const CBlockIndex* block_index;
+ for (block_index = chainActive.Genesis();
+ block_index != nullptr;
+ block_index = chainActive.Next(block_index)) {
+ CheckFilterLookups(filter_index, block_index, last_header);
+ }
+ }
+
+ // Create two forks.
+ const CBlockIndex* tip;
+ {
+ LOCK(cs_main);
+ tip = chainActive.Tip();
+ }
+ CScript coinbase_script_pub_key = GetScriptForDestination(coinbaseKey.GetPubKey().GetID());
+ std::vector<std::shared_ptr<CBlock>> chainA, chainB;
+ BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key, 10, chainA));
+ BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key, 10, chainB));
+
+ // Check that new blocks on chain A get indexed.
+ uint256 chainA_last_header = last_header;
+ for (size_t i = 0; i < 2; i++) {
+ const auto& block = chainA[i];
+ BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr));
+
+ const CBlockIndex* block_index;
+ {
+ LOCK(cs_main);
+ block_index = LookupBlockIndex(block->GetHash());
+ }
+
+ BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+ CheckFilterLookups(filter_index, block_index, chainA_last_header);
+ }
+
+ // Reorg to chain B.
+ uint256 chainB_last_header = last_header;
+ for (size_t i = 0; i < 3; i++) {
+ const auto& block = chainB[i];
+ BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr));
+
+ const CBlockIndex* block_index;
+ {
+ LOCK(cs_main);
+ block_index = LookupBlockIndex(block->GetHash());
+ }
+
+ BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+ CheckFilterLookups(filter_index, block_index, chainB_last_header);
+ }
+
+ // Check that filters for stale blocks on A can be retrieved.
+ chainA_last_header = last_header;
+ for (size_t i = 0; i < 2; i++) {
+ const auto& block = chainA[i];
+ const CBlockIndex* block_index;
+ {
+ LOCK(cs_main);
+ block_index = LookupBlockIndex(block->GetHash());
+ }
+
+ BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+ CheckFilterLookups(filter_index, block_index, chainA_last_header);
+ }
+
+ // Reorg back to chain A.
+ for (size_t i = 2; i < 4; i++) {
+ const auto& block = chainA[i];
+ BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr));
+ }
+
+ // Check that chain A and B blocks can be retrieved.
+ chainA_last_header = last_header;
+ chainB_last_header = last_header;
+ for (size_t i = 0; i < 3; i++) {
+ const CBlockIndex* block_index;
+
+ {
+ LOCK(cs_main);
+ block_index = LookupBlockIndex(chainA[i]->GetHash());
+ }
+ BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+ CheckFilterLookups(filter_index, block_index, chainA_last_header);
+
+ {
+ LOCK(cs_main);
+ block_index = LookupBlockIndex(chainB[i]->GetHash());
+ }
+ BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+ CheckFilterLookups(filter_index, block_index, chainB_last_header);
+ }
+
+ // Test lookups for a range of filters/hashes.
+ std::vector<BlockFilter> filters;
+ std::vector<uint256> filter_hashes;
+
+ {
+ LOCK(cs_main);
+ tip = chainActive.Tip();
+ }
+ BOOST_CHECK(filter_index.LookupFilterRange(0, tip, filters));
+ BOOST_CHECK(filter_index.LookupFilterHashRange(0, tip, filter_hashes));
+
+ BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1);
+ BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1);
+
+ filters.clear();
+ filter_hashes.clear();
+
+ filter_index.Interrupt();
+ filter_index.Stop();
+}
+
+BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup)
+{
+ SetDataDir("tempdir");
+
+ BlockFilterIndex* filter_index;
+
+ filter_index = GetBlockFilterIndex(BlockFilterType::BASIC);
+ BOOST_CHECK(filter_index == nullptr);
+
+ BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false));
+
+ filter_index = GetBlockFilterIndex(BlockFilterType::BASIC);
+ BOOST_CHECK(filter_index != nullptr);
+ BOOST_CHECK(filter_index->GetFilterType() == BlockFilterType::BASIC);
+
+ // Initialize returns false if index already exists.
+ BOOST_CHECK(!InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false));
+
+ int iter_count = 0;
+ ForEachBlockFilterIndex([&iter_count](BlockFilterIndex& _index) { iter_count++; });
+ BOOST_CHECK_EQUAL(iter_count, 1);
+
+ BOOST_CHECK(DestroyBlockFilterIndex(BlockFilterType::BASIC));
+
+ // Destroy returns false because index was already destroyed.
+ BOOST_CHECK(!DestroyBlockFilterIndex(BlockFilterType::BASIC));
+
+ filter_index = GetBlockFilterIndex(BlockFilterType::BASIC);
+ BOOST_CHECK(filter_index == nullptr);
+
+ // Reinitialize index.
+ BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false));
+
+ DestroyAllBlockFilterIndexes();
+
+ filter_index = GetBlockFilterIndex(BlockFilterType::BASIC);
+ BOOST_CHECK(filter_index == nullptr);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp
index f4c74af098..df0a041e0e 100644
--- a/src/test/blockfilter_tests.cpp
+++ b/src/test/blockfilter_tests.cpp
@@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(gcsfilter_default_constructor)
BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
{
- CScript included_scripts[5], excluded_scripts[3];
+ CScript included_scripts[5], excluded_scripts[4];
// First two are outputs on a single transaction.
included_scripts[0] << std::vector<unsigned char>(0, 65) << OP_CHECKSIG;
@@ -73,14 +73,19 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
// This script is not related to the block at all.
excluded_scripts[1] << std::vector<unsigned char>(5, 33) << OP_CHECKSIG;
+ // OP_RETURN is non-standard since it's not followed by a data push, but is still excluded from
+ // filter.
+ excluded_scripts[2] << OP_RETURN << OP_4 << OP_ADD << OP_8 << OP_EQUAL;
+
CMutableTransaction tx_1;
tx_1.vout.emplace_back(100, included_scripts[0]);
tx_1.vout.emplace_back(200, included_scripts[1]);
+ tx_1.vout.emplace_back(0, excluded_scripts[0]);
CMutableTransaction tx_2;
tx_2.vout.emplace_back(300, included_scripts[2]);
- tx_2.vout.emplace_back(0, excluded_scripts[0]);
- tx_2.vout.emplace_back(400, excluded_scripts[2]); // Script is empty
+ tx_2.vout.emplace_back(0, excluded_scripts[2]);
+ tx_2.vout.emplace_back(400, excluded_scripts[3]); // Script is empty
CBlock block;
block.vtx.push_back(MakeTransactionRef(tx_1));
@@ -90,7 +95,7 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
block_undo.vtxundo.emplace_back();
block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(500, included_scripts[3]), 1000, true);
block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(600, included_scripts[4]), 10000, false);
- block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(700, excluded_scripts[2]), 100000, false);
+ block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(700, excluded_scripts[3]), 100000, false);
BlockFilter block_filter(BlockFilterType::BASIC, block, block_undo);
const GCSFilter& filter = block_filter.GetFilter();
@@ -174,4 +179,16 @@ BOOST_AUTO_TEST_CASE(blockfilters_json_test)
}
}
+BOOST_AUTO_TEST_CASE(blockfilter_type_names)
+{
+ BOOST_CHECK_EQUAL(BlockFilterTypeName(BlockFilterType::BASIC), "basic");
+ BOOST_CHECK_EQUAL(BlockFilterTypeName(static_cast<BlockFilterType>(255)), "");
+
+ BlockFilterType filter_type;
+ BOOST_CHECK(BlockFilterTypeByName("basic", filter_type));
+ BOOST_CHECK_EQUAL(filter_type, BlockFilterType::BASIC);
+
+ BOOST_CHECK(!BlockFilterTypeByName("unknown", filter_type));
+}
+
BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/test/data/blockfilters.json b/src/test/data/blockfilters.json
index 134b788eed..8945296a07 100644
--- a/src/test/data/blockfilters.json
+++ b/src/test/data/blockfilters.json
@@ -3,9 +3,11 @@
[0,"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff001d1aa4ae180101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000",[],"0000000000000000000000000000000000000000000000000000000000000000","019dfca8","21584579b7eb08997773e5aeff3a7f932700042d0ed2a6129012b7d7ae81b750","Genesis block"],
[2,"000000006c02c8ea6e4ff69651f7fcde348fb9d557a06e6957b65552002a7820","0100000006128e87be8b1b4dea47a7247d5528d2702c96826c7a648497e773b800000000e241352e3bec0a95a6217e10c3abb54adfa05abb12c126695595580fb92e222032e7494dffff001d00d235340101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0432e7494d010e062f503253482fffffffff0100f2052a010000002321038a7f6ef1c8ca0c588aa53fa860128077c9e6c11e6830f4d7ee4e763a56b7718fac00000000",[],"d7bdac13a59d745b1add0d2ce852f1a0442e8945fc1bf3848d3cbffd88c24fe1","0174a170","186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0",""],
[3,"000000008b896e272758da5297bcd98fdc6d97c9b765ecec401e286dc1fdbe10","0100000020782a005255b657696ea057d5b98f34defcf75196f64f6eeac8026c0000000041ba5afc532aae03151b8aa87b65e1594f97504a768e010c98c0add79216247186e7494dffff001d058dc2b60101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0486e7494d0151062f503253482fffffffff0100f2052a01000000232103f6d9ff4c12959445ca5549c811683bf9c88e637b222dd2e0311154c4c85cf423ac00000000",[],"186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0","016cf7a0","8d63aadf5ab7257cb6d2316a57b16f517bff1c6388f124ec4c04af1212729d2a",""],
+[15007,"0000000038c44c703bae0f98cdd6bf30922326340a5996cc692aaae8bacf47ad","0100000002394092aa378fe35d7e9ac79c869b975c4de4374cd75eb5484b0e1e00000000eb9b8670abd44ad6c55cee18e3020fb0c6519e7004b01a16e9164867531b67afc33bc94fffff001d123f10050101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e04c33bc94f0115062f503253482fffffffff0100f2052a01000000232103f268e9ae07e0f8cb2f6e901d87c510d650b97230c0365b021df8f467363cafb1ac00000000",[],"18b5c2b0146d2d09d24fb00ff5b52bd0742f36c9e65527abdb9de30c027a4748","013c3710","07384b01311867949e0c046607c66b7a766d338474bb67f66c8ae9dbd454b20e","Tx has non-standard OP_RETURN output followed by opcodes"],
[49291,"0000000018b07dca1b28b4b5a119f6d6e71698ce1ed96f143f54179ce177a19c","02000000abfaf47274223ca2fea22797e44498240e482cb4c2f2baea088962f800000000604b5b52c32305b15d7542071d8b04e750a547500005d4010727694b6e72a776e55d0d51ffff001d211806480201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0d038bc0000102062f503253482fffffffff01a078072a01000000232102971dd6034ed0cf52450b608d196c07d6345184fcb14deb277a6b82d526a6163dac0000000001000000081cefd96060ecb1c4fbe675ad8a4f8bdc61d634c52b3a1c4116dee23749fe80ff000000009300493046022100866859c21f306538152e83f115bcfbf59ab4bb34887a88c03483a5dff9895f96022100a6dfd83caa609bf0516debc2bf65c3df91813a4842650a1858b3f61cfa8af249014730440220296d4b818bb037d0f83f9f7111665f49532dfdcbec1e6b784526e9ac4046eaa602204acf3a5cb2695e8404d80bf49ab04828bcbe6fc31d25a2844ced7a8d24afbdff01ffffffff1cefd96060ecb1c4fbe675ad8a4f8bdc61d634c52b3a1c4116dee23749fe80ff020000009400483045022100e87899175991aa008176cb553c6f2badbb5b741f328c9845fcab89f8b18cae2302200acce689896dc82933015e7230e5230d5cff8a1ffe82d334d60162ac2c5b0c9601493046022100994ad29d1e7b03e41731a4316e5f4992f0d9b6e2efc40a1ccd2c949b461175c502210099b69fdc2db00fbba214f16e286f6a49e2d8a0d5ffc6409d87796add475478d601ffffffff1e4a6d2d280ea06680d6cf8788ac90344a9c67cca9b06005bbd6d3f6945c8272010000009500493046022100a27400ba52fd842ce07398a1de102f710a10c5599545e6c95798934352c2e4df022100f6383b0b14c9f64b6718139f55b6b9494374755b86bae7d63f5d3e583b57255a01493046022100fdf543292f34e1eeb1703b264965339ec4a450ec47585009c606b3edbc5b617b022100a5fbb1c8de8aaaa582988cdb23622838e38de90bebcaab3928d949aa502a65d401ffffffff1e4a6d2d280ea06680d6cf8788ac90344a9c67cca9b06005bbd6d3f6945c8272020000009400493046022100ac626ac3051f875145b4fe4cfe089ea895aac73f65ab837b1ac30f5d875874fa022100bc03e79fa4b7eb707fb735b95ff6613ca33adeaf3a0607cdcead4cfd3b51729801483045022100b720b04a5c5e2f61b7df0fcf334ab6fea167b7aaede5695d3f7c6973496adbf1022043328c4cc1cdc3e5db7bb895ccc37133e960b2fd3ece98350f774596badb387201ffffffff23a8733e349c97d6cd90f520fdd084ba15ce0a395aad03cd51370602bb9e5db3010000004a00483045022100e8556b72c5e9c0da7371913a45861a61c5df434dfd962de7b23848e1a28c86ca02205d41ceda00136267281be0974be132ac4cda1459fe2090ce455619d8b91045e901ffffffff6856d609b881e875a5ee141c235e2a82f6b039f2b9babe82333677a5570285a6000000006a473044022040a1c631554b8b210fbdf2a73f191b2851afb51d5171fb53502a3a040a38d2c0022040d11cf6e7b41fe1b66c3d08f6ada1aee07a047cb77f242b8ecc63812c832c9a012102bcfad931b502761e452962a5976c79158a0f6d307ad31b739611dac6a297c256ffffffff6856d609b881e875a5ee141c235e2a82f6b039f2b9babe82333677a5570285a601000000930048304502205b109df098f7e932fbf71a45869c3f80323974a826ee2770789eae178a21bfc8022100c0e75615e53ee4b6e32b9bb5faa36ac539e9c05fa2ae6b6de5d09c08455c8b9601483045022009fb7d27375c47bea23b24818634df6a54ecf72d52e0c1268fb2a2c84f1885de022100e0ed4f15d62e7f537da0d0f1863498f9c7c0c0a4e00e4679588c8d1a9eb20bb801ffffffffa563c3722b7b39481836d5edfc1461f97335d5d1e9a23ade13680d0e2c1c371f030000006c493046022100ecc38ae2b1565643dc3c0dad5e961a5f0ea09cab28d024f92fa05c922924157e022100ebc166edf6fbe4004c72bfe8cf40130263f98ddff728c8e67b113dbd621906a601210211a4ed241174708c07206601b44a4c1c29e5ad8b1f731c50ca7e1d4b2a06dc1fffffffff02d0223a00000000001976a91445db0b779c0b9fa207f12a8218c94fc77aff504588ac80f0fa02000000000000000000",["5221033423007d8f263819a2e42becaaf5b06f34cb09919e06304349d950668209eaed21021d69e2b68c3960903b702af7829fadcd80bd89b158150c85c4a75b2c8cb9c39452ae","52210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821021d69e2b68c3960903b702af7829fadcd80bd89b158150c85c4a75b2c8cb9c39452ae","522102a7ae1e0971fc1689bd66d2a7296da3a1662fd21a53c9e38979e0f090a375c12d21022adb62335f41eb4e27056ac37d462cda5ad783fa8e0e526ed79c752475db285d52ae","52210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821022adb62335f41eb4e27056ac37d462cda5ad783fa8e0e526ed79c752475db285d52ae","512103b9d1d0e2b4355ec3cdef7c11a5c0beff9e8b8d8372ab4b4e0aaf30e80173001951ae","76a9149144761ebaccd5b4bbdc2a35453585b5637b2f8588ac","522103f1848b40621c5d48471d9784c8174ca060555891ace6d2b03c58eece946b1a9121020ee5d32b54d429c152fdc7b1db84f2074b0564d35400d89d11870f9273ec140c52ae","76a914f4fa1cc7de742d135ea82c17adf0bb9cf5f4fb8388ac"],"ed47705334f4643892ca46396eb3f4196a5e30880589e4009ef38eae895d4a13","0afbc2920af1b027f31f87b592276eb4c32094bb4d3697021b4c6380","b6d98692cec5145f67585f3434ec3c2b3030182e1cb3ec58b855c5c164dfaaa3","Tx pays to empty output script"],
-[180480,"00000000fd3ceb2404ff07a785c7fdcc76619edc8ed61bd25134eaa22084366a","020000006058aa080a655aa991a444bd7d1f2defd9a3bbe68aabb69030cf3b4e00000000d2e826bfd7ef0beaa891a7eedbc92cd6a544a6cb61c7bdaa436762eb2123ef9790f5f552ffff001d0002c90f0501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0300c102024608062f503253482fffffffff01c0c6072a01000000232102e769e60137a4df6b0df8ebd387cca44c4c57ae74cc0114a8e8317c8f3bfd85e9ac00000000010000000381a0802911a01ffb025c4dea0bc77963e8c1bb46313b71164c53f72f37fe5248010000000151ffffffffc904b267833d215e2128bd9575242232ac2bc311550c7fc1f0ef6f264b40d14c010000000151ffffffffdf0915666649dba81886519c531649b7b02180b4af67d6885e871299e9d5f775000000000151ffffffff0180817dcb00000000232103bb52138972c48a132fc1f637858c5189607dd0f7fe40c4f20f6ad65f2d389ba4ac0000000001000000018da38b434fba82d66052af74fc5e4e94301b114d9bc03f819dc876398404c8b4010000006c493046022100fe738b7580dc5fb5168e51fc61b5aed211125eb71068031009a22d9bbad752c5022100be5086baa384d40bcab0fa586e4f728397388d86e18b66cc417dc4f7fa4f9878012103f233299455134caa2687bdf15cb0becdfb03bd0ff2ff38e65ec6b7834295c34fffffffff022ebc1400000000001976a9147779b7fba1c1e06b717069b80ca170e8b04458a488ac9879c40f000000001976a9142a0307cd925dbb66b534c4db33003dd18c57015788ac0000000001000000026139a62e3422a602de36c873a225c1d3ca5aeee598539ceecb9f0dc8d1ad0f83010000006b483045022100ad9f32b4a0a2ddc19b5a74eba78123e57616f1b3cfd72ce68c03ea35a3dda1f002200dbd22aa6da17213df5e70dfc3b2611d40f70c98ed9626aa5e2cde9d97461f0a012103ddb295d2f1e8319187738fb4b230fdd9aa29d0e01647f69f6d770b9ab24eea90ffffffff983c82c87cf020040d671956525014d5c2b28c6d948c85e1a522362c0059eeae010000006b4830450221009ca544274c786d30a5d5d25e17759201ea16d3aedddf0b9e9721246f7ef6b32e02202cfa5564b6e87dfd9fd98957820e4d4e6238baeb0f65fe305d91506bb13f5f4f012103c99113deac0d5d044e3ac0346abc02501542af8c8d3759f1382c72ff84e704f7ffffffff02c0c62d00000000001976a914ae19d27efe12f5a886dc79af37ad6805db6f922d88ac70ce2000000000001976a9143b8d051d37a07ea1042067e93efe63dbf73920b988ac000000000100000002be566e8cd9933f0c75c4a82c027f7d0c544d5c101d0607ef6ae5d07b98e7f1dc000000006b483045022036a8cdfd5ea7ebc06c2bfb6e4f942bbf9a1caeded41680d11a3a9f5d8284abad022100cacb92a5be3f39e8bc14db1710910ef7b395fa1e18f45d41c28d914fcdde33be012102bf59abf110b5131fae0a3ce1ec379329b4c896a6ae5d443edb68529cc2bc7816ffffffff96cf67645b76ceb23fe922874847456a15feee1655082ff32d25a6bf2c0dfc90000000006a47304402203471ca2001784a5ac0abab583581f2613523da47ec5f53df833c117b5abd81500220618a2847723d57324f2984678db556dbca1a72230fc7e39df04c2239942ba942012102925c9794fd7bb9f8b29e207d5fc491b1150135a21f505041858889fa4edf436fffffffff026c840f00000000001976a914797fb8777d7991d8284d88bfd421ce520f0f843188ac00ca9a3b000000001976a9146d10f3f592699265d10b106eda37c3ce793f7a8588ac00000000",["","","","76a9142903b138c24be9e070b3e73ec495d77a204615e788ac","76a91433a1941fd9a37b9821d376f5a51bd4b52fa50e2888ac","76a914e4374e8155d0865742ca12b8d4d14d41b57d682f88ac","76a914001fa7459a6cfc64bdc178ba7e7a21603bb2568f88ac","76a914f6039952bc2b307aeec5371bfb96b66078ec17f688ac"],"b109139671dbedc2b6fcd499a5480a7461ae458af8ff9411d819aa64ba6995d1","0db414c859a07e8205876354a210a75042d0463404913d61a8e068e58a3ae2aa080026","a0af77e0a7ed20ea78d2def3200cc24f08217dcd51755c7c7feb0e2ba8316c2d","Tx spends from empty output script"],
-[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"da49977ba1ee0d620a2c4f8f646b03cd0d230f5c6c994722e3ba884889f0be1a","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","4cd9dd007a325199102f1fc0b7d77ca25ee3c84d46018c4353ecfcb56c0d3e7a","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"],
-[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"e9d729b72d533c29abe5276d5cf6c152f3723f10efe000b1e0c9ca5265a8beb6","010c0b40","e6137ae5a8424c40da1e5023c16975cc97b09300b4c050e6b1c713add3836c40","Coinbase tx has unparseable output script"],
-[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"a4a4d6c6034da8aa06f01fe71f1fffbd79e032006b07f6c7a2c60a66aa310c01","0385acb4f0fe889ef0","3588f34fbbc11640f9ed40b2a66a4e096215d50389691309c1dac74d4268aa81","Includes witness data"]
+[180480,"00000000fd3ceb2404ff07a785c7fdcc76619edc8ed61bd25134eaa22084366a","020000006058aa080a655aa991a444bd7d1f2defd9a3bbe68aabb69030cf3b4e00000000d2e826bfd7ef0beaa891a7eedbc92cd6a544a6cb61c7bdaa436762eb2123ef9790f5f552ffff001d0002c90f0501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0300c102024608062f503253482fffffffff01c0c6072a01000000232102e769e60137a4df6b0df8ebd387cca44c4c57ae74cc0114a8e8317c8f3bfd85e9ac00000000010000000381a0802911a01ffb025c4dea0bc77963e8c1bb46313b71164c53f72f37fe5248010000000151ffffffffc904b267833d215e2128bd9575242232ac2bc311550c7fc1f0ef6f264b40d14c010000000151ffffffffdf0915666649dba81886519c531649b7b02180b4af67d6885e871299e9d5f775000000000151ffffffff0180817dcb00000000232103bb52138972c48a132fc1f637858c5189607dd0f7fe40c4f20f6ad65f2d389ba4ac0000000001000000018da38b434fba82d66052af74fc5e4e94301b114d9bc03f819dc876398404c8b4010000006c493046022100fe738b7580dc5fb5168e51fc61b5aed211125eb71068031009a22d9bbad752c5022100be5086baa384d40bcab0fa586e4f728397388d86e18b66cc417dc4f7fa4f9878012103f233299455134caa2687bdf15cb0becdfb03bd0ff2ff38e65ec6b7834295c34fffffffff022ebc1400000000001976a9147779b7fba1c1e06b717069b80ca170e8b04458a488ac9879c40f000000001976a9142a0307cd925dbb66b534c4db33003dd18c57015788ac0000000001000000026139a62e3422a602de36c873a225c1d3ca5aeee598539ceecb9f0dc8d1ad0f83010000006b483045022100ad9f32b4a0a2ddc19b5a74eba78123e57616f1b3cfd72ce68c03ea35a3dda1f002200dbd22aa6da17213df5e70dfc3b2611d40f70c98ed9626aa5e2cde9d97461f0a012103ddb295d2f1e8319187738fb4b230fdd9aa29d0e01647f69f6d770b9ab24eea90ffffffff983c82c87cf020040d671956525014d5c2b28c6d948c85e1a522362c0059eeae010000006b4830450221009ca544274c786d30a5d5d25e17759201ea16d3aedddf0b9e9721246f7ef6b32e02202cfa5564b6e87dfd9fd98957820e4d4e6238baeb0f65fe305d91506bb13f5f4f012103c99113deac0d5d044e3ac0346abc02501542af8c8d3759f1382c72ff84e704f7ffffffff02c0c62d00000000001976a914ae19d27efe12f5a886dc79af37ad6805db6f922d88ac70ce2000000000001976a9143b8d051d37a07ea1042067e93efe63dbf73920b988ac000000000100000002be566e8cd9933f0c75c4a82c027f7d0c544d5c101d0607ef6ae5d07b98e7f1dc000000006b483045022036a8cdfd5ea7ebc06c2bfb6e4f942bbf9a1caeded41680d11a3a9f5d8284abad022100cacb92a5be3f39e8bc14db1710910ef7b395fa1e18f45d41c28d914fcdde33be012102bf59abf110b5131fae0a3ce1ec379329b4c896a6ae5d443edb68529cc2bc7816ffffffff96cf67645b76ceb23fe922874847456a15feee1655082ff32d25a6bf2c0dfc90000000006a47304402203471ca2001784a5ac0abab583581f2613523da47ec5f53df833c117b5abd81500220618a2847723d57324f2984678db556dbca1a72230fc7e39df04c2239942ba942012102925c9794fd7bb9f8b29e207d5fc491b1150135a21f505041858889fa4edf436fffffffff026c840f00000000001976a914797fb8777d7991d8284d88bfd421ce520f0f843188ac00ca9a3b000000001976a9146d10f3f592699265d10b106eda37c3ce793f7a8588ac00000000",["","","","76a9142903b138c24be9e070b3e73ec495d77a204615e788ac","76a91433a1941fd9a37b9821d376f5a51bd4b52fa50e2888ac","76a914e4374e8155d0865742ca12b8d4d14d41b57d682f88ac","76a914001fa7459a6cfc64bdc178ba7e7a21603bb2568f88ac","76a914f6039952bc2b307aeec5371bfb96b66078ec17f688ac"],"d34ef98386f413769502808d4bac5f20f8dfd5bffc9eedafaa71de0eb1f01489","0db414c859a07e8205876354a210a75042d0463404913d61a8e068e58a3ae2aa080026","c582d51c0ca365e3fcf36c51cb646d7f83a67e867cb4743fd2128e3e022b700c","Tx spends from empty output script"],
+[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"8f13b9a9c85611635b47906c3053ac53cfcec7211455d4cb0d63dc9acc13d472","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","546c574a0472144bcaf9b6aeabf26372ad87c7af7d1ee0dbfae5e099abeae49c","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"],
+[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"fe4d230dbb0f4fec9bed23a5283e08baf996e3f32b93f52c7de1f641ddfd04ad","010c0b40","0965a544743bbfa36f254446e75630c09404b3d164a261892372977538928ed5","Coinbase tx has unparseable output script"],
+[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"31d66d516a9eda7de865df29f6ef6cb8e4bf9309e5dac899968a9a62a5df61e3","0385acb4f0fe889ef0","4e6d564c2a2452065c205dd7eb2791124e0c4e0dbb064c410c24968572589dec","Includes witness data"],
+[1414221,"0000000000000027b2b3b3381f114f674f481544ff2be37ae3788d7e078383b1","000000204ea88307a7959d8207968f152bedca5a93aefab253f1fb2cfb032a400000000070cebb14ec6dbc27a9dfd066d9849a4d3bac5f674665f73a5fe1de01a022a0c851fda85bf05f4c19a779d1450102000000010000000000000000000000000000000000000000000000000000000000000000ffffffff18034d94154d696e6572476174653030310d000000f238f401ffffffff01c817a804000000000000000000",[],"5e5e12d90693c8e936f01847859404c67482439681928353ca1296982042864e","00","021e8882ef5a0ed932edeebbecfeda1d7ce528ec7b3daa27641acf1189d7b5dc","Empty data"]
]
diff --git a/src/txdb.h b/src/txdb.h
index d5a4bfaeb1..c4ece11503 100644
--- a/src/txdb.h
+++ b/src/txdb.h
@@ -37,6 +37,8 @@ static const int64_t nMaxBlockDBCache = 2;
// Unlike for the UTXO database, for the txindex scenario the leveldb cache make
// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
static const int64_t nMaxTxIndexCache = 1024;
+//! Max memory allocated to all block filter index caches combined in MiB.
+static const int64_t max_filter_index_cache = 1024;
//! Max memory allocated to coin DB specific cache (MiB)
static const int64_t nMaxCoinsDBCache = 8;
diff --git a/src/validation.h b/src/validation.h
index 9f00cab49e..041d8860b0 100644
--- a/src/validation.h
+++ b/src/validation.h
@@ -116,6 +116,7 @@ static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60;
static const bool DEFAULT_CHECKPOINTS_ENABLED = true;
static const bool DEFAULT_TXINDEX = false;
+static const char* const DEFAULT_BLOCKFILTERINDEX = "0";
static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100;
/** Default for -persistmempool */
static const bool DEFAULT_PERSIST_MEMPOOL = true;
diff --git a/test/functional/rpc_getblockfilter.py b/test/functional/rpc_getblockfilter.py
new file mode 100755
index 0000000000..bd93b6f7a4
--- /dev/null
+++ b/test/functional/rpc_getblockfilter.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the getblockfilter RPC."""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+ assert_equal, assert_is_hex_string, assert_raises_rpc_error,
+ connect_nodes, disconnect_nodes, sync_blocks
+ )
+
+FILTER_TYPES = ["basic"]
+
+class GetBlockFilterTest(BitcoinTestFramework):
+ def set_test_params(self):
+ self.setup_clean_chain = True
+ self.num_nodes = 2
+ self.extra_args = [["-blockfilterindex"], []]
+
+ def run_test(self):
+ # Create two chains by disconnecting nodes 0 & 1, mining, then reconnecting
+ disconnect_nodes(self.nodes[0], 1)
+
+ self.nodes[0].generate(3)
+ self.nodes[1].generate(4)
+
+ assert_equal(self.nodes[0].getblockcount(), 3)
+ chain0_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)]
+
+ # Reorg node 0 to a new chain
+ connect_nodes(self.nodes[0], 1)
+ sync_blocks(self.nodes)
+
+ assert_equal(self.nodes[0].getblockcount(), 4)
+ chain1_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)]
+
+ # Test getblockfilter returns a filter for all blocks and filter types on active chain
+ for block_hash in chain1_hashes:
+ for filter_type in FILTER_TYPES:
+ result = self.nodes[0].getblockfilter(block_hash, filter_type)
+ assert_is_hex_string(result['filter'])
+
+ # Test getblockfilter returns a filter for all blocks and filter types on stale chain
+ for block_hash in chain0_hashes:
+ for filter_type in FILTER_TYPES:
+ result = self.nodes[0].getblockfilter(block_hash, filter_type)
+ assert_is_hex_string(result['filter'])
+
+ # Test getblockfilter with unknown block
+ bad_block_hash = "0123456789abcdef" * 4
+ assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockfilter, bad_block_hash, "basic")
+
+ # Test getblockfilter with undefined filter type
+ genesis_hash = self.nodes[0].getblockhash(0)
+ assert_raises_rpc_error(-5, "Unknown filtertype", self.nodes[0].getblockfilter, genesis_hash, "unknown")
+
+if __name__ == '__main__':
+ GetBlockFilterTest().main()
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 61c5c9aefb..86f334e942 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -144,6 +144,7 @@ BASE_SCRIPTS = [
'wallet_txn_doublespend.py',
'wallet_txn_clone.py --mineblock',
'feature_notifications.py',
+ 'rpc_getblockfilter.py',
'rpc_invalidateblock.py',
'feature_rbf.py',
'mempool_packages.py',