From 3499ce1e1ad87a86598d00b7124072c91ddad833 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 7 Oct 2015 17:12:24 -0700 Subject: Encapsulate CLevelDB iterators cleanly Conflicts: src/leveldb.cpp src/leveldb.h src/txdb.cpp --- src/leveldbwrapper.cpp | 9 ++++++- src/leveldbwrapper.h | 65 ++++++++++++++++++++++++++++++++++++++++++++++---- src/txdb.cpp | 59 ++++++++++++++++----------------------------- 3 files changed, 89 insertions(+), 44 deletions(-) (limited to 'src') diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index ce96b5c8aa..a94cfd8a30 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -131,7 +131,7 @@ std::vector CLevelDBWrapper::CreateObfuscateKey() const bool CLevelDBWrapper::IsEmpty() { - boost::scoped_ptr it(NewIterator()); + boost::scoped_ptr it(NewIterator()); it->SeekToFirst(); return !(it->Valid()); } @@ -145,3 +145,10 @@ std::string CLevelDBWrapper::GetObfuscateKeyHex() const { return HexStr(obfuscate_key); } + +CLevelDBIterator::~CLevelDBIterator() { delete piter; } +bool CLevelDBIterator::Valid() { return piter->Valid(); } +void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); } +void CLevelDBIterator::SeekToLast() { piter->SeekToLast(); } +void CLevelDBIterator::Next() { piter->Next(); } +void CLevelDBIterator::Prev() { piter->Prev(); } diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index f5c463830c..ece4e49694 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -68,7 +68,65 @@ public: batch.Delete(slKey); } }; + +class CLevelDBIterator +{ +private: + leveldb::Iterator *piter; + +public: + CLevelDBIterator(leveldb::Iterator *piterIn) : piter(piterIn) {} + ~CLevelDBIterator(); + + bool Valid(); + void SeekToFirst(); + void SeekToLast(); + + template void Seek(const K& key) { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey << key; + leveldb::Slice slKey(&ssKey[0], ssKey.size()); + piter->Seek(slKey); + } + + void Next(); + void Prev(); + + template bool GetKey(K& key) { + leveldb::Slice slKey = piter->key(); + try { + CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + ssKey >> key; + } catch(std::exception &e) { + return false; + } + return true; + } + + unsigned int GetKeySize() { + return piter->key().size(); + } + + template bool GetValue(V& value) { + leveldb::Slice slValue = piter->value(); + try { + CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); + ssValue.Xor(db.GetObfuscateKey()); + ssValue >> value; + } catch(std::exception &e) { + return false; + } + return true; + } + + unsigned int GetValueSize() { + return piter->value().size(); + } + +}; + class CLevelDBWrapper { private: @@ -191,11 +249,10 @@ public: return WriteBatch(batch, true); } - // not exactly clean encapsulation, but it's easiest for now - leveldb::Iterator* NewIterator() + CLevelDBIterator *NewIterator() + { + return new CLevelDBIterator(pdb->NewIterator(iteroptions)); { - return pdb->NewIterator(iteroptions); - } /** * Return true if the database managed by this class contains no entries. diff --git a/src/txdb.cpp b/src/txdb.cpp index 9738dea03d..3d20508912 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -98,8 +98,8 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); - pcursor->SeekToFirst(); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + pcursor->Seek('c'); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = GetBestBlock(); @@ -107,22 +107,10 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { CAmount nTotalAmount = 0; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - leveldb::Slice slKey = pcursor->key(); - CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION); - char chType; - ssKey >> chType; - if (chType == DB_COINS) { - leveldb::Slice slValue = pcursor->value(); - CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); - CCoins coins; - ssValue >> coins; - uint256 txhash; - ssKey >> txhash; - ss << txhash; - ss << VARINT(coins.nVersion); - ss << (coins.fCoinBase ? 'c' : 'n'); - ss << VARINT(coins.nHeight); + std::pair key; + CCoins coins; + if (pcursor->GetKey(key) && key.first == 'c') { + if (pcursor->GetValue(coins)) { stats.nTransactions++; for (unsigned int i=0; iGetKeySize(); ss << VARINT(0); + } else { + return error("CCoinsViewDB::GetStats() : unable to read value"); } - pcursor->Next(); - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } else { + break; } + pcursor->Next(); } { LOCK(cs_main); @@ -189,24 +179,15 @@ bool CBlockTreeDB::LoadBlockIndexGuts() { boost::scoped_ptr pcursor(NewIterator()); - CDataStream ssKeySet(SER_DISK, CLIENT_VERSION); - ssKeySet << make_pair(DB_BLOCK_INDEX, uint256()); - pcursor->Seek(ssKeySet.str()); + pcursor->Seek(make_pair('b', uint256(0))); // Load mapBlockIndex while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - leveldb::Slice slKey = pcursor->key(); - CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION); - char chType; - ssKey >> chType; - if (chType == DB_BLOCK_INDEX) { - leveldb::Slice slValue = pcursor->value(); - CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); - CDiskBlockIndex diskindex; - ssValue >> diskindex; - + std::pair key; + if (pcursor->GetKey(key) && key.first == 'b') { + CDiskBlockIndex diskindex; + if (pcursor->GetValue(diskindex)) { // Construct block index object CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); @@ -227,10 +208,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pcursor->Next(); } else { - break; // if shutdown requested or finished loading block index + return error("LoadBlockIndex() : failed to read value"); } - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } else { + break; } } -- cgit v1.2.3 From 0fdf8c80ee322ab747321d61faf9c72af4a51445 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 8 Oct 2015 00:44:10 -0700 Subject: Handle obfuscation in CLevelDBIterator --- src/leveldbwrapper.cpp | 2 +- src/leveldbwrapper.h | 15 +++++++++++---- src/txdb.cpp | 6 +++--- 3 files changed, 15 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index a94cfd8a30..32c9345be5 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -145,7 +145,7 @@ std::string CLevelDBWrapper::GetObfuscateKeyHex() const { return HexStr(obfuscate_key); } - + CLevelDBIterator::~CLevelDBIterator() { delete piter; } bool CLevelDBIterator::Valid() { return piter->Valid(); } void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); } diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index ece4e49694..891381c5f2 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -73,9 +73,16 @@ class CLevelDBIterator { private: leveldb::Iterator *piter; + const std::vector obfuscate_key; public: - CLevelDBIterator(leveldb::Iterator *piterIn) : piter(piterIn) {} + + /** + * @param[in] piterIn The original leveldb iterator. + * @param[in] obfuscate_key If passed, XOR data with this key. + */ + CLevelDBIterator(leveldb::Iterator *piterIn, const std::vector& obfuscate_key) : + piter(piterIn), obfuscate_key(obfuscate_key) { }; ~CLevelDBIterator(); bool Valid(); @@ -113,7 +120,7 @@ public: leveldb::Slice slValue = piter->value(); try { CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); - ssValue.Xor(db.GetObfuscateKey()); + ssValue.Xor(obfuscate_key); ssValue >> value; } catch(std::exception &e) { return false; @@ -251,8 +258,8 @@ public: CLevelDBIterator *NewIterator() { - return new CLevelDBIterator(pdb->NewIterator(iteroptions)); - { + return new CLevelDBIterator(pdb->NewIterator(iteroptions), obfuscate_key); + } /** * Return true if the database managed by this class contains no entries. diff --git a/src/txdb.cpp b/src/txdb.cpp index 3d20508912..4d7ec27aae 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -98,7 +98,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); pcursor->Seek('c'); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); @@ -177,9 +177,9 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { bool CBlockTreeDB::LoadBlockIndexGuts() { - boost::scoped_ptr pcursor(NewIterator()); + boost::scoped_ptr pcursor(NewIterator()); - pcursor->Seek(make_pair('b', uint256(0))); + pcursor->Seek(make_pair('b', uint256())); // Load mapBlockIndex while (pcursor->Valid()) { -- cgit v1.2.3 From 14885068726a8e0dc73ffa127225ab80be3e3612 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 8 Oct 2015 01:22:50 -0700 Subject: Add tests for gettxoutsetinfo, CLevelDBBatch, CLevelDBIterator Thanks @dexX7. --- src/test/leveldbwrapper_tests.cpp | 81 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp index db04f3ea48..36975548c2 100644 --- a/src/test/leveldbwrapper_tests.cpp +++ b/src/test/leveldbwrapper_tests.cpp @@ -46,7 +46,86 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper) BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); } } - + +// Test batch operations +BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + + char key = 'i'; + uint256 in = GetRandHash(); + char key2 = 'j'; + uint256 in2 = GetRandHash(); + char key3 = 'k'; + uint256 in3 = GetRandHash(); + + uint256 res; + CLevelDBBatch batch(dbw.GetObfuscateKey()); + + batch.Write(key, in); + batch.Write(key2, in2); + batch.Write(key3, in3); + + // Remove key3 before it's even been written + batch.Erase(key3); + + dbw.WriteBatch(batch); + + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + BOOST_CHECK(dbw.Read(key2, res)); + BOOST_CHECK_EQUAL(res.ToString(), in2.ToString()); + + // key3 never should've been written + BOOST_CHECK(dbw.Read(key3, res) == false); + } +} + +BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + + // The two keys are intentionally chosen for ordering + char key = 'j'; + uint256 in = GetRandHash(); + BOOST_CHECK(dbw.Write(key, in)); + char key2 = 'k'; + uint256 in2 = GetRandHash(); + BOOST_CHECK(dbw.Write(key2, in2)); + + boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); + + // Be sure to seek past the obfuscation key (if it exists) + it->Seek(key); + + char key_res; + uint256 val_res; + + it->GetKey(key_res); + it->GetValue(val_res); + BOOST_CHECK_EQUAL(key_res, key); + BOOST_CHECK_EQUAL(val_res.ToString(), in.ToString()); + + it->Next(); + + it->GetKey(key_res); + it->GetValue(val_res); + BOOST_CHECK_EQUAL(key_res, key2); + BOOST_CHECK_EQUAL(val_res.ToString(), in2.ToString()); + + it->Next(); + BOOST_CHECK_EQUAL(it->Valid(), false); + } +} + // Test that we do not obfuscation if there is existing data. BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) { -- cgit v1.2.3 From dcd8e27c65de0d2cb972588a6e811ca7ccd1b3bd Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Fri, 9 Oct 2015 10:55:27 -0700 Subject: Refer to obfuscate_key via pointer in peripheral CLevelDB classes cc @sipa --- src/leveldbwrapper.h | 20 ++++++++++---------- src/test/leveldbwrapper_tests.cpp | 2 +- src/txdb.cpp | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index 891381c5f2..60101e18cc 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -32,13 +32,13 @@ class CLevelDBBatch private: leveldb::WriteBatch batch; - const std::vector obfuscate_key; + const std::vector *obfuscate_key; public: /** * @param[in] obfuscate_key If passed, XOR data with this key. */ - CLevelDBBatch(const std::vector& obfuscate_key) : obfuscate_key(obfuscate_key) { }; + CLevelDBBatch(const std::vector *obfuscate_key) : obfuscate_key(obfuscate_key) { }; template void Write(const K& key, const V& value) @@ -51,7 +51,7 @@ public: CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(ssValue.GetSerializeSize(value)); ssValue << value; - ssValue.Xor(obfuscate_key); + ssValue.Xor(*obfuscate_key); leveldb::Slice slValue(&ssValue[0], ssValue.size()); batch.Put(slKey, slValue); @@ -73,7 +73,7 @@ class CLevelDBIterator { private: leveldb::Iterator *piter; - const std::vector obfuscate_key; + const std::vector *obfuscate_key; public: @@ -81,7 +81,7 @@ public: * @param[in] piterIn The original leveldb iterator. * @param[in] obfuscate_key If passed, XOR data with this key. */ - CLevelDBIterator(leveldb::Iterator *piterIn, const std::vector& obfuscate_key) : + CLevelDBIterator(leveldb::Iterator *piterIn, const std::vector* obfuscate_key) : piter(piterIn), obfuscate_key(obfuscate_key) { }; ~CLevelDBIterator(); @@ -120,7 +120,7 @@ public: leveldb::Slice slValue = piter->value(); try { CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); - ssValue.Xor(obfuscate_key); + ssValue.Xor(*obfuscate_key); ssValue >> value; } catch(std::exception &e) { return false; @@ -210,7 +210,7 @@ public: template bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error) { - CLevelDBBatch batch(obfuscate_key); + CLevelDBBatch batch(&obfuscate_key); batch.Write(key, value); return WriteBatch(batch, fSync); } @@ -237,7 +237,7 @@ public: template bool Erase(const K& key, bool fSync = false) throw(leveldb_error) { - CLevelDBBatch batch(obfuscate_key); + CLevelDBBatch batch(&obfuscate_key); batch.Erase(key); return WriteBatch(batch, fSync); } @@ -252,13 +252,13 @@ public: bool Sync() throw(leveldb_error) { - CLevelDBBatch batch(obfuscate_key); + CLevelDBBatch batch(&obfuscate_key); return WriteBatch(batch, true); } CLevelDBIterator *NewIterator() { - return new CLevelDBIterator(pdb->NewIterator(iteroptions), obfuscate_key); + return new CLevelDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key); } /** diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp index 36975548c2..606313b004 100644 --- a/src/test/leveldbwrapper_tests.cpp +++ b/src/test/leveldbwrapper_tests.cpp @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) uint256 in3 = GetRandHash(); uint256 res; - CLevelDBBatch batch(dbw.GetObfuscateKey()); + CLevelDBBatch batch(&dbw.GetObfuscateKey()); batch.Write(key, in); batch.Write(key2, in2); diff --git a/src/txdb.cpp b/src/txdb.cpp index 4d7ec27aae..5723c92440 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -49,7 +49,7 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CLevelDBBatch batch(db.GetObfuscateKey()); + CLevelDBBatch batch(&db.GetObfuscateKey()); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -141,7 +141,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CLevelDBBatch batch(GetObfuscateKey()); + CLevelDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -157,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CLevelDBBatch batch(GetObfuscateKey()); + CLevelDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); -- cgit v1.2.3