Merge pull request #6777

dcd8e27 Refer to obfuscate_key via pointer in peripheral CLevelDB classes (James O'Beirne)
1488506 Add tests for gettxoutsetinfo, CLevelDBBatch, CLevelDBIterator (James O'Beirne)
0fdf8c8 Handle obfuscation in CLevelDBIterator (James O'Beirne)
3499ce1 Encapsulate CLevelDB iterators cleanly (Pieter Wuille)

6 files changed, 245 insertions, 61 deletions

diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py
index 58098bdec5..db97767272 100755
--- a/qa/pull-tester/rpc-tests.py
+++ b/qa/pull-tester/rpc-tests.py
@@ -38,7 +38,7 @@ for i in range(1,len(sys.argv)):
 buildDir = BUILDDIR
 os.environ["BITCOIND"] = buildDir + '/src/bitcoind' + EXEEXT
 os.environ["BITCOINCLI"] = buildDir + '/src/bitcoin-cli' + EXEEXT
- 
+
 #Disable Windows tests by default
 if EXEEXT == ".exe" and "-win" not in opts:
     print "Win tests currently disabled.  Use -win option to enable"
@@ -67,6 +67,7 @@ testScripts = [
     'reindex.py',
     'decodescript.py',
     'p2p-fullblocktest.py',
+    'blockchain.py',
 ]
 testScriptsExt = [
     'bipdersig-p2p.py',
@@ -98,10 +99,10 @@ if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
     rpcTestDir = buildDir + '/qa/rpc-tests/'
     #Run Tests
     for i in range(len(testScripts)):
-       if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts 
+       if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts
            or testScripts[i] in opts or  re.sub(".py$", "", testScripts[i]) in opts ):
-            print  "Running testscript " + testScripts[i] + "..." 
-            subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) 
+            print  "Running testscript " + testScripts[i] + "..."
+            subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
 	    #exit if help is called so we print just one set of instructions
             p = re.compile(" -h| --help")
             if p.match(passOn):
@@ -109,9 +110,9 @@ if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
 
     #Run Extended Tests
     for i in range(len(testScriptsExt)):
-        if ('-extended' in opts or testScriptsExt[i] in opts 
+        if ('-extended' in opts or testScriptsExt[i] in opts
            or re.sub(".py$", "", testScriptsExt[i]) in opts):
-            print  "Running 2nd level testscript " + testScriptsExt[i] + "..." 
-            subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) 
+            print  "Running 2nd level testscript " + testScriptsExt[i] + "..."
+            subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
 else:
     print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled"
diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py
new file mode 100755
index 0000000000..a5c98b777e
--- /dev/null
+++ b/qa/rpc-tests/blockchain.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python2
+# Copyright (c) 2014 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Test RPC calls related to blockchain state.
+#
+
+import decimal
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    initialize_chain,
+    assert_equal,
+    start_nodes,
+    connect_nodes_bi,
+)
+
+class BlockchainTest(BitcoinTestFramework):
+    """
+    Test blockchain-related RPC calls:
+
+        - gettxoutsetinfo
+
+    """
+
+    def setup_chain(self):
+        print("Initializing test directory " + self.options.tmpdir)
+        initialize_chain(self.options.tmpdir)
+
+    def setup_network(self, split=False):
+        self.nodes = start_nodes(2, self.options.tmpdir)
+        connect_nodes_bi(self.nodes, 0, 1)
+        self.is_network_split = False
+        self.sync_all()
+
+    def run_test(self):
+        node = self.nodes[0]
+        res = node.gettxoutsetinfo()
+
+        assert_equal(res[u'total_amount'], decimal.Decimal('8725.00000000'))
+        assert_equal(res[u'transactions'], 200)
+        assert_equal(res[u'height'], 200)
+        assert_equal(res[u'txouts'], 200)
+        assert_equal(res[u'bytes_serialized'], 13000),
+        assert_equal(len(res[u'bestblock']), 64)
+        assert_equal(len(res[u'hash_serialized']), 64)
+
+
+if __name__ == '__main__':
+    BlockchainTest().main()
diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp
index ce96b5c8aa..32c9345be5 100644
--- a/src/leveldbwrapper.cpp
+++ b/src/leveldbwrapper.cpp
@@ -131,7 +131,7 @@ std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const
 
 bool CLevelDBWrapper::IsEmpty()
 {
-    boost::scoped_ptr<leveldb::Iterator> it(NewIterator());
+    boost::scoped_ptr<CLevelDBIterator> it(NewIterator());
     it->SeekToFirst();
     return !(it->Valid());
 }
@@ -145,3 +145,10 @@ std::string CLevelDBWrapper::GetObfuscateKeyHex() const
 { 
     return HexStr(obfuscate_key); 
 }
+
+CLevelDBIterator::~CLevelDBIterator() { delete piter; }
+bool CLevelDBIterator::Valid() { return piter->Valid(); }
+void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); }
+void CLevelDBIterator::SeekToLast() { piter->SeekToLast(); }
+void CLevelDBIterator::Next() { piter->Next(); }
+void CLevelDBIterator::Prev() { piter->Prev(); }
diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h
index f5c463830c..60101e18cc 100644
--- a/src/leveldbwrapper.h
+++ b/src/leveldbwrapper.h
@@ -32,13 +32,13 @@ class CLevelDBBatch
 
 private:
     leveldb::WriteBatch batch;
-    const std::vector<unsigned char> obfuscate_key;
+    const std::vector<unsigned char> *obfuscate_key;
 
 public:
     /**
      * @param[in] obfuscate_key    If passed, XOR data with this key.
      */
-    CLevelDBBatch(const std::vector<unsigned char>& obfuscate_key) : obfuscate_key(obfuscate_key) { };
+    CLevelDBBatch(const std::vector<unsigned char> *obfuscate_key) : obfuscate_key(obfuscate_key) { };
 
     template <typename K, typename V>
     void Write(const K& key, const V& value)
@@ -51,7 +51,7 @@ public:
         CDataStream ssValue(SER_DISK, CLIENT_VERSION);
         ssValue.reserve(ssValue.GetSerializeSize(value));
         ssValue << value;
-        ssValue.Xor(obfuscate_key);
+        ssValue.Xor(*obfuscate_key);
         leveldb::Slice slValue(&ssValue[0], ssValue.size());
 
         batch.Put(slKey, slValue);
@@ -68,7 +68,72 @@ public:
         batch.Delete(slKey);
     }
 };
+ 
+class CLevelDBIterator
+{
+private:
+    leveldb::Iterator *piter;
+    const std::vector<unsigned char> *obfuscate_key;
+
+public:
+
+    /**
+     * @param[in] piterIn          The original leveldb iterator.
+     * @param[in] obfuscate_key    If passed, XOR data with this key.
+     */
+    CLevelDBIterator(leveldb::Iterator *piterIn, const std::vector<unsigned char>* obfuscate_key) :
+        piter(piterIn), obfuscate_key(obfuscate_key) { };
+    ~CLevelDBIterator();
+
+    bool Valid();
 
+    void SeekToFirst();
+    void SeekToLast();
+
+    template<typename K> void Seek(const K& key) {
+        CDataStream ssKey(SER_DISK, CLIENT_VERSION);
+        ssKey.reserve(ssKey.GetSerializeSize(key));
+        ssKey << key;
+        leveldb::Slice slKey(&ssKey[0], ssKey.size());
+        piter->Seek(slKey);
+    }
+
+    void Next();
+    void Prev();
+
+    template<typename K> bool GetKey(K& key) {
+        leveldb::Slice slKey = piter->key();
+        try {
+            CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION);
+            ssKey >> key;
+        } catch(std::exception &e) {
+            return false;
+        }
+        return true;
+    }
+
+    unsigned int GetKeySize() {
+        return piter->key().size();
+    }
+
+    template<typename V> bool GetValue(V& value) {
+        leveldb::Slice slValue = piter->value();
+        try {
+            CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION);
+            ssValue.Xor(*obfuscate_key);
+            ssValue >> value;
+        } catch(std::exception &e) {
+            return false;
+        }
+        return true;
+    }
+
+    unsigned int GetValueSize() {
+        return piter->value().size();
+    }
+
+};
+ 
 class CLevelDBWrapper
 {
 private:
@@ -145,7 +210,7 @@ public:
     template <typename K, typename V>
     bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error)
     {
-        CLevelDBBatch batch(obfuscate_key);
+        CLevelDBBatch batch(&obfuscate_key);
         batch.Write(key, value);
         return WriteBatch(batch, fSync);
     }
@@ -172,7 +237,7 @@ public:
     template <typename K>
     bool Erase(const K& key, bool fSync = false) throw(leveldb_error)
     {
-        CLevelDBBatch batch(obfuscate_key);
+        CLevelDBBatch batch(&obfuscate_key);
         batch.Erase(key);
         return WriteBatch(batch, fSync);
     }
@@ -187,14 +252,13 @@ public:
 
     bool Sync() throw(leveldb_error)
     {
-        CLevelDBBatch batch(obfuscate_key);
+        CLevelDBBatch batch(&obfuscate_key);
         return WriteBatch(batch, true);
     }
 
-    // not exactly clean encapsulation, but it's easiest for now
-    leveldb::Iterator* NewIterator()
+    CLevelDBIterator *NewIterator() 
     {
-        return pdb->NewIterator(iteroptions);
+        return new CLevelDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key);
     }
 
     /**
diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp
index db04f3ea48..606313b004 100644
--- a/src/test/leveldbwrapper_tests.cpp
+++ b/src/test/leveldbwrapper_tests.cpp
@@ -46,7 +46,86 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper)
         BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
     }
 }
-                       
+
+// Test batch operations
+BOOST_AUTO_TEST_CASE(leveldbwrapper_batch)
+{
+    // Perform tests both obfuscated and non-obfuscated.
+    for (int i = 0; i < 2; i++) {
+        bool obfuscate = (bool)i;
+        path ph = temp_directory_path() / unique_path();
+        CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
+
+        char key = 'i';
+        uint256 in = GetRandHash();
+        char key2 = 'j';
+        uint256 in2 = GetRandHash();
+        char key3 = 'k';
+        uint256 in3 = GetRandHash();
+
+        uint256 res;
+        CLevelDBBatch batch(&dbw.GetObfuscateKey());
+
+        batch.Write(key, in);
+        batch.Write(key2, in2);
+        batch.Write(key3, in3);
+
+        // Remove key3 before it's even been written
+        batch.Erase(key3);
+
+        dbw.WriteBatch(batch);
+
+        BOOST_CHECK(dbw.Read(key, res));
+        BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
+        BOOST_CHECK(dbw.Read(key2, res));
+        BOOST_CHECK_EQUAL(res.ToString(), in2.ToString());
+
+        // key3 never should've been written
+        BOOST_CHECK(dbw.Read(key3, res) == false);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator)
+{
+    // Perform tests both obfuscated and non-obfuscated.
+    for (int i = 0; i < 2; i++) {
+        bool obfuscate = (bool)i;
+        path ph = temp_directory_path() / unique_path();
+        CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
+
+        // The two keys are intentionally chosen for ordering
+        char key = 'j';
+        uint256 in = GetRandHash();
+        BOOST_CHECK(dbw.Write(key, in));
+        char key2 = 'k';
+        uint256 in2 = GetRandHash();
+        BOOST_CHECK(dbw.Write(key2, in2));
+
+        boost::scoped_ptr<CLevelDBIterator> it(const_cast<CLevelDBWrapper*>(&dbw)->NewIterator());
+
+        // Be sure to seek past the obfuscation key (if it exists)
+        it->Seek(key);
+
+        char key_res;
+        uint256 val_res;
+
+        it->GetKey(key_res);
+        it->GetValue(val_res);
+        BOOST_CHECK_EQUAL(key_res, key);
+        BOOST_CHECK_EQUAL(val_res.ToString(), in.ToString());
+
+        it->Next();
+
+        it->GetKey(key_res);
+        it->GetValue(val_res);
+        BOOST_CHECK_EQUAL(key_res, key2);
+        BOOST_CHECK_EQUAL(val_res.ToString(), in2.ToString());
+
+        it->Next();
+        BOOST_CHECK_EQUAL(it->Valid(), false);
+    }
+}
+
 // Test that we do not obfuscation if there is existing data.
 BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
 {
diff --git a/src/txdb.cpp b/src/txdb.cpp
index 9738dea03d..5723c92440 100644
--- a/src/txdb.cpp
+++ b/src/txdb.cpp
@@ -49,7 +49,7 @@ uint256 CCoinsViewDB::GetBestBlock() const {
 }
 
 bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
-    CLevelDBBatch batch(db.GetObfuscateKey());
+    CLevelDBBatch batch(&db.GetObfuscateKey());
     size_t count = 0;
     size_t changed = 0;
     for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
@@ -98,8 +98,8 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
     /* It seems that there are no "const iterators" for LevelDB.  Since we
        only need read operations on it, use a const-cast to get around
        that restriction.  */
-    boost::scoped_ptr<leveldb::Iterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator());
-    pcursor->SeekToFirst();
+    boost::scoped_ptr<CLevelDBIterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator());
+    pcursor->Seek('c');
 
     CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION);
     stats.hashBlock = GetBestBlock();
@@ -107,22 +107,10 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
     CAmount nTotalAmount = 0;
     while (pcursor->Valid()) {
         boost::this_thread::interruption_point();
-        try {
-            leveldb::Slice slKey = pcursor->key();
-            CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION);
-            char chType;
-            ssKey >> chType;
-            if (chType == DB_COINS) {
-                leveldb::Slice slValue = pcursor->value();
-                CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
-                CCoins coins;
-                ssValue >> coins;
-                uint256 txhash;
-                ssKey >> txhash;
-                ss << txhash;
-                ss << VARINT(coins.nVersion);
-                ss << (coins.fCoinBase ? 'c' : 'n');
-                ss << VARINT(coins.nHeight);
+        std::pair<char, uint256> key;
+        CCoins coins;
+        if (pcursor->GetKey(key) && key.first == 'c') {
+            if (pcursor->GetValue(coins)) {
                 stats.nTransactions++;
                 for (unsigned int i=0; i<coins.vout.size(); i++) {
                     const CTxOut &out = coins.vout[i];
@@ -133,13 +121,15 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
                         nTotalAmount += out.nValue;
                     }
                 }
-                stats.nSerializedSize += 32 + slValue.size();
+                stats.nSerializedSize += 32 + pcursor->GetKeySize();
                 ss << VARINT(0);
+            } else {
+                return error("CCoinsViewDB::GetStats() : unable to read value");
             }
-            pcursor->Next();
-        } catch (const std::exception& e) {
-            return error("%s: Deserialize or I/O error - %s", __func__, e.what());
+        } else {
+            break;
         }
+        pcursor->Next();
     }
     {
         LOCK(cs_main);
@@ -151,7 +141,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
 }
 
 bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
-    CLevelDBBatch batch(GetObfuscateKey());
+    CLevelDBBatch batch(&GetObfuscateKey());
     for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
         batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second);
     }
@@ -167,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) {
 }
 
 bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) {
-    CLevelDBBatch batch(GetObfuscateKey());
+    CLevelDBBatch batch(&GetObfuscateKey());
     for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++)
         batch.Write(make_pair(DB_TXINDEX, it->first), it->second);
     return WriteBatch(batch);
@@ -187,26 +177,17 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
 
 bool CBlockTreeDB::LoadBlockIndexGuts()
 {
-    boost::scoped_ptr<leveldb::Iterator> pcursor(NewIterator());
+    boost::scoped_ptr<CLevelDBIterator> pcursor(NewIterator());
 
-    CDataStream ssKeySet(SER_DISK, CLIENT_VERSION);
-    ssKeySet << make_pair(DB_BLOCK_INDEX, uint256());
-    pcursor->Seek(ssKeySet.str());
+    pcursor->Seek(make_pair('b', uint256()));
 
     // Load mapBlockIndex
     while (pcursor->Valid()) {
         boost::this_thread::interruption_point();
-        try {
-            leveldb::Slice slKey = pcursor->key();
-            CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION);
-            char chType;
-            ssKey >> chType;
-            if (chType == DB_BLOCK_INDEX) {
-                leveldb::Slice slValue = pcursor->value();
-                CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
-                CDiskBlockIndex diskindex;
-                ssValue >> diskindex;
-
+        std::pair<char, uint256> key;
+        if (pcursor->GetKey(key) && key.first == 'b') {
+            CDiskBlockIndex diskindex;
+            if (pcursor->GetValue(diskindex)) {
                 // Construct block index object
                 CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash());
                 pindexNew->pprev          = InsertBlockIndex(diskindex.hashPrev);
@@ -227,10 +208,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts()
 
                 pcursor->Next();
             } else {
-                break; // if shutdown requested or finished loading block index
+                return error("LoadBlockIndex() : failed to read value");
             }
-        } catch (const std::exception& e) {
-            return error("%s: Deserialize or I/O error - %s", __func__, e.what());
+        } else {
+            break;
         }
     }