Merge #12254: BIP 158: Compact Block Filters for Light Clients

254c85b68794ada713dbdae415db72adf5fcbaf3 bench: Benchmark GCS filter creation and matching. (Jim Posen)
f33b717a85363e067316c133a542559d2f4aaeca blockfilter: Optimization on compilers with int128 support. (Jim Posen)
97b64d67daf0336dfb64b132f3e4d6a4c1967da4 blockfilter: Unit test against BIP 158 test vectors. (Jim Posen)
a4afb9cadbaecb0676e6475ab8d32a52faecb47a blockfilter: Additional helper methods to compute hash and header. (Jim Posen)
cd09c7925b5af4104834971cfe072251e3ac2bda blockfilter: Serialization methods on BlockFilter. (Jim Posen)
c1855f6052aca806fdb51be01b30dfeee8b55f40 blockfilter: Construction of basic block filters. (Jim Posen)
53e7874e079f9ddfe8b176f11d46e6b59c7283d5 blockfilter: Simple test for GCSFilter construction and Match. (Jim Posen)
558c536e35a25594881693e6ff01d275c88d7af1 blockfilter: Implement GCSFilter Match methods. (Jim Posen)
cf70b550054eed36f194eaa13f4a9cb31e32df38 blockfilter: Implement GCSFilter constructors. (Jim Posen)
c454f0ac63c6028f54c7eb51683b3ccdb475b19b blockfilter: Declare GCSFilter class for BIP 158 impl. (Jim Posen)
9b622dc72279b027c59d6541cddff53800fc689b streams: Unit tests for BitStreamReader and BitStreamWriter. (Jim Posen)
fe943f99bf0a2bbb12e30bc4803c0337e3c95b93 streams: Implement BitStreamReader/Writer classes. (Jim Posen)
87f2d9ee43a9220076b1959d1ca65245d9591be9 streams: Unit test for VectorReader class. (Jim Posen)
947133dec92cd25ec2b3358c09b8614ba6fb40d4 streams: Create VectorReader stream interface for vectors. (Jim Posen)

Pull request description:

  This implements the compact block filter construction in [BIP 158](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki). The code is not used anywhere in the Bitcoin Core code base yet. The next step towards [BIP 157](https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki) support would be to create an indexing module similar to `TxIndex` that constructs the basic and extended filters for each validated block.

  ### Filter Sizes

  [Here](https://gateway.ipfs.io/ipfs/QmRqaAAQZ5ZX5eqxP7J2R1MzFrc2WDdKSWJEKtQzyawqog) is a CSV of filter sizes for blocks in the main chain.

  As you can see below, the ratio of filter size to block size drops after the first ~150,000 blocks:

  ![filter_sizes](https://user-images.githubusercontent.com/881253/42900589-299772d4-8a7e-11e8-886d-0d4f3f4fbe44.png)

  The reason for the relatively large filter sizes is that Golomb-coded sets only achieve good compression with a sufficient number of elements. Empirically, the average element size with 100 elements is 14% larger than with 10,000 elements.

  The ratio of filter size to block size is computed without witness data for basic filters. Here is a summary table of filter size ratios *for blocks after height 150,000*:

  | Stat | Filter Type |
  |-------|--------------|
  | Weighted Size Ratio Mean | 0.0198 |
  | Size Ratio Mean | 0.0224 |
  | Size Ratio Std Deviation | 0.0202 |
  | Mean Element Size (bits) | 21.145 |
  | Approx Theoretical Min Element Size (bits) | 21.025 |

Tree-SHA512: 2d045fbfc3fc45490ecb9b08d2f7e4dbbe7cd8c1c939f06bbdb8e8aacfe4c495cdb67c820e52520baebbf8a8305a0efd8e59d3fa8e367574a4b830509a39223f

3 files changed, 231 insertions, 0 deletions

diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp
new file mode 100644
index 0000000000..e3cb05f09e
--- /dev/null
+++ b/src/test/blockfilter_tests.cpp
@@ -0,0 +1,142 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <test/data/blockfilters.json.h>
+#include <test/test_bitcoin.h>
+
+#include <blockfilter.h>
+#include <core_io.h>
+#include <serialize.h>
+#include <streams.h>
+#include <univalue.h>
+#include <utilstrencodings.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(blockfilter_tests)
+
+BOOST_AUTO_TEST_CASE(gcsfilter_test)
+{
+    GCSFilter::ElementSet included_elements, excluded_elements;
+    for (int i = 0; i < 100; ++i) {
+        GCSFilter::Element element1(32);
+        element1[0] = i;
+        included_elements.insert(std::move(element1));
+
+        GCSFilter::Element element2(32);
+        element2[1] = i;
+        excluded_elements.insert(std::move(element2));
+    }
+
+    GCSFilter filter(0, 0, 10, 1 << 10, included_elements);
+    for (const auto& element : included_elements) {
+        BOOST_CHECK(filter.Match(element));
+
+        auto insertion = excluded_elements.insert(element);
+        BOOST_CHECK(filter.MatchAny(excluded_elements));
+        excluded_elements.erase(insertion.first);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
+{
+    CScript included_scripts[5], excluded_scripts[2];
+
+    // First two are outputs on a single transaction.
+    included_scripts[0] << std::vector<unsigned char>(0, 65) << OP_CHECKSIG;
+    included_scripts[1] << OP_DUP << OP_HASH160 << std::vector<unsigned char>(1, 20) << OP_EQUALVERIFY << OP_CHECKSIG;
+
+    // Third is an output on in a second transaction.
+    included_scripts[2] << OP_1 << std::vector<unsigned char>(2, 33) << OP_1 << OP_CHECKMULTISIG;
+
+    // Last two are spent by a single transaction.
+    included_scripts[3] << OP_0 << std::vector<unsigned char>(3, 32);
+    included_scripts[4] << OP_4 << OP_ADD << OP_8 << OP_EQUAL;
+
+    // OP_RETURN output is an output on the second transaction.
+    excluded_scripts[0] << OP_RETURN << std::vector<unsigned char>(4, 40);
+
+    // This script is not related to the block at all.
+    excluded_scripts[1] << std::vector<unsigned char>(5, 33) << OP_CHECKSIG;
+
+    CMutableTransaction tx_1;
+    tx_1.vout.emplace_back(100, included_scripts[0]);
+    tx_1.vout.emplace_back(200, included_scripts[1]);
+
+    CMutableTransaction tx_2;
+    tx_2.vout.emplace_back(300, included_scripts[2]);
+    tx_2.vout.emplace_back(0, excluded_scripts[0]);
+
+    CBlock block;
+    block.vtx.push_back(MakeTransactionRef(tx_1));
+    block.vtx.push_back(MakeTransactionRef(tx_2));
+
+    CBlockUndo block_undo;
+    block_undo.vtxundo.emplace_back();
+    block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(400, included_scripts[3]), 1000, true);
+    block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(500, included_scripts[4]), 10000, false);
+
+    BlockFilter block_filter(BlockFilterType::BASIC, block, block_undo);
+    const GCSFilter& filter = block_filter.GetFilter();
+
+    for (const CScript& script : included_scripts) {
+        BOOST_CHECK(filter.Match(GCSFilter::Element(script.begin(), script.end())));
+    }
+    for (const CScript& script : excluded_scripts) {
+        BOOST_CHECK(!filter.Match(GCSFilter::Element(script.begin(), script.end())));
+    }
+}
+
+BOOST_AUTO_TEST_CASE(blockfilters_json_test)
+{
+    UniValue json;
+    std::string json_data(json_tests::blockfilters,
+                          json_tests::blockfilters + sizeof(json_tests::blockfilters));
+    if (!json.read(json_data) || !json.isArray()) {
+        BOOST_ERROR("Parse error.");
+        return;
+    }
+
+    const UniValue& tests = json.get_array();
+    for (unsigned int i = 0; i < tests.size(); i++) {
+        UniValue test = tests[i];
+        std::string strTest = test.write();
+
+        if (test.size() == 1) {
+            continue;
+        } else if (test.size() < 7) {
+            BOOST_ERROR("Bad test: " << strTest);
+            continue;
+        }
+
+        unsigned int pos = 0;
+        /*int block_height =*/ test[pos++].get_int();
+        /*uint256 block_hash =*/ ParseHashStr(test[pos++].get_str(), "block_hash");
+
+        CBlock block;
+        BOOST_REQUIRE(DecodeHexBlk(block, test[pos++].get_str()));
+
+        CBlockUndo block_undo;
+        block_undo.vtxundo.emplace_back();
+        CTxUndo& tx_undo = block_undo.vtxundo.back();
+        const UniValue& prev_scripts = test[pos++].get_array();
+        for (unsigned int ii = 0; ii < prev_scripts.size(); ii++) {
+            std::vector<unsigned char> raw_script = ParseHex(prev_scripts[ii].get_str());
+            CTxOut txout(0, CScript(raw_script.begin(), raw_script.end()));
+            tx_undo.vprevout.emplace_back(txout, 0, false);
+        }
+
+        uint256 prev_filter_header_basic = ParseHashStr(test[pos++].get_str(), "prev_filter_header_basic");
+        std::vector<unsigned char> filter_basic = ParseHex(test[pos++].get_str());
+        uint256 filter_header_basic = ParseHashStr(test[pos++].get_str(), "filter_header_basic");
+
+        BlockFilter computed_filter_basic(BlockFilterType::BASIC, block, block_undo);
+        BOOST_CHECK(computed_filter_basic.GetFilter().GetEncoded() == filter_basic);
+
+        uint256 computed_header_basic = computed_filter_basic.ComputeHeader(prev_filter_header_basic);
+        BOOST_CHECK(computed_header_basic == filter_header_basic);
+    }
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/test/data/blockfilters.json b/src/test/data/blockfilters.json
new file mode 100644
index 0000000000..2f00728ff6
--- /dev/null
+++ b/src/test/data/blockfilters.json
@@ -0,0 +1,9 @@
+[
+["Block Height,Block Hash,Block,[Prev Output Scripts for Block],Previous Basic Header,Basic Filter,Basic Header,Notes"],
+[0,"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff001d1aa4ae180101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000",[],"0000000000000000000000000000000000000000000000000000000000000000","019dfca8","21584579b7eb08997773e5aeff3a7f932700042d0ed2a6129012b7d7ae81b750","Genesis block"],
+[2,"000000006c02c8ea6e4ff69651f7fcde348fb9d557a06e6957b65552002a7820","0100000006128e87be8b1b4dea47a7247d5528d2702c96826c7a648497e773b800000000e241352e3bec0a95a6217e10c3abb54adfa05abb12c126695595580fb92e222032e7494dffff001d00d235340101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0432e7494d010e062f503253482fffffffff0100f2052a010000002321038a7f6ef1c8ca0c588aa53fa860128077c9e6c11e6830f4d7ee4e763a56b7718fac00000000",[],"d7bdac13a59d745b1add0d2ce852f1a0442e8945fc1bf3848d3cbffd88c24fe1","0174a170","186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0",""],
+[3,"000000008b896e272758da5297bcd98fdc6d97c9b765ecec401e286dc1fdbe10","0100000020782a005255b657696ea057d5b98f34defcf75196f64f6eeac8026c0000000041ba5afc532aae03151b8aa87b65e1594f97504a768e010c98c0add79216247186e7494dffff001d058dc2b60101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0486e7494d0151062f503253482fffffffff0100f2052a01000000232103f6d9ff4c12959445ca5549c811683bf9c88e637b222dd2e0311154c4c85cf423ac00000000",[],"186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0","016cf7a0","8d63aadf5ab7257cb6d2316a57b16f517bff1c6388f124ec4c04af1212729d2a",""],
+[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"da49977ba1ee0d620a2c4f8f646b03cd0d230f5c6c994722e3ba884889f0be1a","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","4cd9dd007a325199102f1fc0b7d77ca25ee3c84d46018c4353ecfcb56c0d3e7a","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"],
+[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"e9d729b72d533c29abe5276d5cf6c152f3723f10efe000b1e0c9ca5265a8beb6","010c0b40","e6137ae5a8424c40da1e5023c16975cc97b09300b4c050e6b1c713add3836c40","Coinbase tx has unparseable output script"],
+[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"a4a4d6c6034da8aa06f01fe71f1fffbd79e032006b07f6c7a2c60a66aa310c01","0385acb4f0fe889ef0","3588f34fbbc11640f9ed40b2a66a4e096215d50389691309c1dac74d4268aa81","Includes witness data"]
+]
diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp
index 6a0aa864d9..26cf74830d 100644
--- a/src/test/streams_tests.cpp
+++ b/src/test/streams_tests.cpp
@@ -68,6 +68,86 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer)
     vch.clear();
 }
 
+BOOST_AUTO_TEST_CASE(streams_vector_reader)
+{
+    std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6};
+
+    VectorReader reader(SER_NETWORK, INIT_PROTO_VERSION, vch, 0);
+    BOOST_CHECK_EQUAL(reader.size(), 6);
+    BOOST_CHECK(!reader.empty());
+
+    // Read a single byte as an unsigned char.
+    unsigned char a;
+    reader >> a;
+    BOOST_CHECK_EQUAL(a, 1);
+    BOOST_CHECK_EQUAL(reader.size(), 5);
+    BOOST_CHECK(!reader.empty());
+
+    // Read a single byte as a signed char.
+    signed char b;
+    reader >> b;
+    BOOST_CHECK_EQUAL(b, -1);
+    BOOST_CHECK_EQUAL(reader.size(), 4);
+    BOOST_CHECK(!reader.empty());
+
+    // Read a 4 bytes as an unsigned int.
+    unsigned int c;
+    reader >> c;
+    BOOST_CHECK_EQUAL(c, 100992003); // 3,4,5,6 in little-endian base-256
+    BOOST_CHECK_EQUAL(reader.size(), 0);
+    BOOST_CHECK(reader.empty());
+
+    // Reading after end of byte vector throws an error.
+    signed int d;
+    BOOST_CHECK_THROW(reader >> d, std::ios_base::failure);
+
+    // Read a 4 bytes as a signed int from the beginning of the buffer.
+    reader.seek(-6);
+    reader >> d;
+    BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256
+    BOOST_CHECK_EQUAL(reader.size(), 2);
+    BOOST_CHECK(!reader.empty());
+
+    // Reading after end of byte vector throws an error even if the reader is
+    // not totally empty.
+    BOOST_CHECK_THROW(reader >> d, std::ios_base::failure);
+}
+
+BOOST_AUTO_TEST_CASE(bitstream_reader_writer)
+{
+    CDataStream data(SER_NETWORK, INIT_PROTO_VERSION);
+
+    BitStreamWriter<CDataStream> bit_writer(data);
+    bit_writer.Write(0, 1);
+    bit_writer.Write(2, 2);
+    bit_writer.Write(6, 3);
+    bit_writer.Write(11, 4);
+    bit_writer.Write(1, 5);
+    bit_writer.Write(32, 6);
+    bit_writer.Write(7, 7);
+    bit_writer.Write(30497, 16);
+    bit_writer.Flush();
+
+    CDataStream data_copy(data);
+    uint32_t serialized_int1;
+    data >> serialized_int1;
+    BOOST_CHECK_EQUAL(serialized_int1, (uint32_t)0x7700C35A); // NOTE: Serialized as LE
+    uint16_t serialized_int2;
+    data >> serialized_int2;
+    BOOST_CHECK_EQUAL(serialized_int2, (uint16_t)0x1072); // NOTE: Serialized as LE
+
+    BitStreamReader<CDataStream> bit_reader(data_copy);
+    BOOST_CHECK_EQUAL(bit_reader.Read(1), 0);
+    BOOST_CHECK_EQUAL(bit_reader.Read(2), 2);
+    BOOST_CHECK_EQUAL(bit_reader.Read(3), 6);
+    BOOST_CHECK_EQUAL(bit_reader.Read(4), 11);
+    BOOST_CHECK_EQUAL(bit_reader.Read(5), 1);
+    BOOST_CHECK_EQUAL(bit_reader.Read(6), 32);
+    BOOST_CHECK_EQUAL(bit_reader.Read(7), 7);
+    BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497);
+    BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure);
+}
+
 BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
 {
     std::vector<char> in;