diff options
| -rw-r--r-- | src/Makefile.test.include | 2 | ||||
| -rw-r--r-- | src/leveldbwrapper.cpp | 60 | ||||
| -rw-r--r-- | src/leveldbwrapper.h | 53 | ||||
| -rw-r--r-- | src/streams.h | 23 | ||||
| -rw-r--r-- | src/test/leveldbwrapper_tests.cpp | 128 | ||||
| -rw-r--r-- | src/test/streams_tests.cpp | 67 | ||||
| -rw-r--r-- | src/txdb.cpp | 27 |
7 files changed, 338 insertions, 22 deletions
diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9a6e43631b..a3b6c4d9b1 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -53,6 +53,7 @@ BITCOIN_TESTS =\ test/hash_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ + test/leveldbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ test/miner_tests.cpp \ @@ -73,6 +74,7 @@ BITCOIN_TESTS =\ test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ + test/streams_tests.cpp \ test/test_bitcoin.cpp \ test/test_bitcoin.h \ test/timedata_tests.cpp \ diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index 26cacf95ae..ce96b5c8aa 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -5,6 +5,7 @@ #include "leveldbwrapper.h" #include "util.h" +#include "random.h" #include <boost/filesystem.hpp> @@ -12,6 +13,7 @@ #include <leveldb/env.h> #include <leveldb/filter_policy.h> #include <memenv.h> +#include <stdint.h> void HandleError(const leveldb::Status& status) throw(leveldb_error) { @@ -43,7 +45,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe) +CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) { penv = NULL; readoptions.verify_checksums = true; @@ -67,6 +69,25 @@ CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCa leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); HandleError(status); LogPrintf("Opened LevelDB successfully\n"); + + // The base-case obfuscation key, which is a noop. + obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000'); + + bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key); + + if (!key_exists && obfuscate && IsEmpty()) { + // Initialize non-degenerate obfuscation if it won't upset + // existing, non-obfuscated data. + std::vector<unsigned char> new_key = CreateObfuscateKey(); + + // Write `new_key` so we don't obfuscate the key with itself + Write(OBFUSCATE_KEY_KEY, new_key); + obfuscate_key = new_key; + + LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex()); + } + + LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex()); } CLevelDBWrapper::~CLevelDBWrapper() @@ -87,3 +108,40 @@ bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) throw(leveldb HandleError(status); return true; } + +// Prefixed with null character to avoid collisions with other keys +// +// We must use a string constructor which specifies length so that we copy +// past the null-terminator. +const std::string CLevelDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14); + +const unsigned int CLevelDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; + +/** + * Returns a string (consisting of 8 random bytes) suitable for use as an + * obfuscating XOR key. + */ +std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const +{ + unsigned char buff[OBFUSCATE_KEY_NUM_BYTES]; + GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES); + return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]); + +} + +bool CLevelDBWrapper::IsEmpty() +{ + boost::scoped_ptr<leveldb::Iterator> it(NewIterator()); + it->SeekToFirst(); + return !(it->Valid()); +} + +const std::vector<unsigned char>& CLevelDBWrapper::GetObfuscateKey() const +{ + return obfuscate_key; +} + +std::string CLevelDBWrapper::GetObfuscateKeyHex() const +{ + return HexStr(obfuscate_key); +} diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index c65e842704..f5c463830c 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -9,6 +9,7 @@ #include "serialize.h" #include "streams.h" #include "util.h" +#include "utilstrencodings.h" #include "version.h" #include <boost/filesystem/path.hpp> @@ -31,8 +32,14 @@ class CLevelDBBatch private: leveldb::WriteBatch batch; + const std::vector<unsigned char> obfuscate_key; public: + /** + * @param[in] obfuscate_key If passed, XOR data with this key. + */ + CLevelDBBatch(const std::vector<unsigned char>& obfuscate_key) : obfuscate_key(obfuscate_key) { }; + template <typename K, typename V> void Write(const K& key, const V& value) { @@ -44,6 +51,7 @@ public: CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(ssValue.GetSerializeSize(value)); ssValue << value; + ssValue.Xor(obfuscate_key); leveldb::Slice slValue(&ssValue[0], ssValue.size()); batch.Put(slKey, slValue); @@ -85,8 +93,27 @@ private: //! the database itself leveldb::DB* pdb; + //! a key used for optional XOR-obfuscation of the database + std::vector<unsigned char> obfuscate_key; + + //! the key under which the obfuscation key is stored + static const std::string OBFUSCATE_KEY_KEY; + + //! the length of the obfuscate key in number of bytes + static const unsigned int OBFUSCATE_KEY_NUM_BYTES; + + std::vector<unsigned char> CreateObfuscateKey() const; + public: - CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false); + /** + * @param[in] path Location in the filesystem where leveldb data will be stored. + * @param[in] nCacheSize Configures various leveldb cache settings. + * @param[in] fMemory If true, use leveldb's memory environment. + * @param[in] fWipe If true, remove all existing data. + * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR + * with a zero'd byte array. + */ + CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); ~CLevelDBWrapper(); template <typename K, typename V> @@ -107,6 +134,7 @@ public: } try { CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); + ssValue.Xor(obfuscate_key); ssValue >> value; } catch (const std::exception&) { return false; @@ -117,7 +145,7 @@ public: template <typename K, typename V> bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error) { - CLevelDBBatch batch; + CLevelDBBatch batch(obfuscate_key); batch.Write(key, value); return WriteBatch(batch, fSync); } @@ -144,7 +172,7 @@ public: template <typename K> bool Erase(const K& key, bool fSync = false) throw(leveldb_error) { - CLevelDBBatch batch; + CLevelDBBatch batch(obfuscate_key); batch.Erase(key); return WriteBatch(batch, fSync); } @@ -159,7 +187,7 @@ public: bool Sync() throw(leveldb_error) { - CLevelDBBatch batch; + CLevelDBBatch batch(obfuscate_key); return WriteBatch(batch, true); } @@ -168,6 +196,23 @@ public: { return pdb->NewIterator(iteroptions); } + + /** + * Return true if the database managed by this class contains no entries. + */ + bool IsEmpty(); + + /** + * Accessor for obfuscate_key. + */ + const std::vector<unsigned char>& GetObfuscateKey() const; + + /** + * Return the obfuscate_key as a hex-formatted string. + */ + std::string GetObfuscateKeyHex() const; + }; #endif // BITCOIN_LEVELDBWRAPPER_H + diff --git a/src/streams.h b/src/streams.h index fa1e18defe..8610e4d18e 100644 --- a/src/streams.h +++ b/src/streams.h @@ -296,6 +296,29 @@ public: data.insert(data.end(), begin(), end()); clear(); } + + /** + * XOR the contents of this stream with a certain key. + * + * @param[in] key The key used to XOR the data in this stream. + */ + void Xor(const std::vector<unsigned char>& key) + { + if (key.size() == 0) { + return; + } + + for (size_type i = 0, j = 0; i != size(); i++) { + vch[i] ^= key[j++]; + + // This potentially acts on very many bytes of data, so it's + // important that we calculate `j`, i.e. the `key` index in this + // way instead of doing a %, which would effectively be a division + // for each byte Xor'd -- much slower than need be. + if (j == key.size()) + j = 0; + } + } }; diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp new file mode 100644 index 0000000000..db04f3ea48 --- /dev/null +++ b/src/test/leveldbwrapper_tests.cpp @@ -0,0 +1,128 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "leveldbwrapper.h" +#include "uint256.h" +#include "random.h" +#include "test/test_bitcoin.h" + +#include <boost/assign/std/vector.hpp> // for 'operator+=()' +#include <boost/assert.hpp> +#include <boost/test/unit_test.hpp> + +using namespace std; +using namespace boost::assign; // bring 'operator+=()' into scope +using namespace boost::filesystem; + +// Test if a string consists entirely of null characters +bool is_null_key(const vector<unsigned char>& key) { + bool isnull = true; + + for (unsigned int i = 0; i < key.size(); i++) + isnull &= (key[i] == '\x00'); + + return isnull; +} + +BOOST_FIXTURE_TEST_SUITE(leveldbwrapper_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(leveldbwrapper) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + // Ensure that we're doing real obfuscation when obfuscate=true + BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey())); + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + } +} + +// Test that we do not obfuscation if there is existing data. +BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) +{ + // We're going to share this path between two wrappers + path ph = temp_directory_path() / unique_path(); + create_directories(ph); + + // Set up a non-obfuscated wrapper to write some initial data. + CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw->Write(key, in)); + BOOST_CHECK(dbw->Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + + // Call the destructor to free leveldb LOCK + delete dbw; + + // Now, set up another wrapper that wants to obfuscate the same directory + CLevelDBWrapper odbw(ph, (1 << 10), false, false, true); + + // Check that the key/val we wrote with unobfuscated wrapper exists and + // is readable. + uint256 res2; + BOOST_CHECK(odbw.Read(key, res2)); + BOOST_CHECK_EQUAL(res2.ToString(), in.ToString()); + + BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data + BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string + + uint256 in2 = GetRandHash(); + uint256 res3; + + // Check that we can write successfully + BOOST_CHECK(odbw.Write(key, in2)); + BOOST_CHECK(odbw.Read(key, res3)); + BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); +} + +// Ensure that we start obfuscating during a reindex. +BOOST_AUTO_TEST_CASE(existing_data_reindex) +{ + // We're going to share this path between two wrappers + path ph = temp_directory_path() / unique_path(); + create_directories(ph); + + // Set up a non-obfuscated wrapper to write some initial data. + CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw->Write(key, in)); + BOOST_CHECK(dbw->Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + + // Call the destructor to free leveldb LOCK + delete dbw; + + // Simulate a -reindex by wiping the existing data store + CLevelDBWrapper odbw(ph, (1 << 10), false, true, true); + + // Check that the key/val we wrote with unobfuscated wrapper doesn't exist + uint256 res2; + BOOST_CHECK(!odbw.Read(key, res2)); + BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey())); + + uint256 in2 = GetRandHash(); + uint256 res3; + + // Check that we can write successfully + BOOST_CHECK(odbw.Write(key, in2)); + BOOST_CHECK(odbw.Read(key, res3)); + BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp new file mode 100644 index 0000000000..0ed8f363d7 --- /dev/null +++ b/src/test/streams_tests.cpp @@ -0,0 +1,67 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "streams.h" +#include "support/allocators/zeroafterfree.h" +#include "test/test_bitcoin.h" + +#include <boost/assign/std/vector.hpp> // for 'operator+=()' +#include <boost/assert.hpp> +#include <boost/test/unit_test.hpp> + +using namespace std; +using namespace boost::assign; // bring 'operator+=()' into scope + +BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(streams_serializedata_xor) +{ + std::vector<char> in; + std::vector<char> expected_xor; + std::vector<unsigned char> key; + CDataStream ds(in, 0, 0); + + // Degenerate case + + key += '\x00','\x00'; + ds.Xor(key); + BOOST_CHECK_EQUAL( + std::string(expected_xor.begin(), expected_xor.end()), + std::string(ds.begin(), ds.end())); + + in += '\x0f','\xf0'; + expected_xor += '\xf0','\x0f'; + + // Single character key + + ds.clear(); + ds.insert(ds.begin(), in.begin(), in.end()); + key.clear(); + + key += '\xff'; + ds.Xor(key); + BOOST_CHECK_EQUAL( + std::string(expected_xor.begin(), expected_xor.end()), + std::string(ds.begin(), ds.end())); + + // Multi character key + + in.clear(); + expected_xor.clear(); + in += '\xf0','\x0f'; + expected_xor += '\x0f','\x00'; + + ds.clear(); + ds.insert(ds.begin(), in.begin(), in.end()); + + key.clear(); + key += '\xff','\x0f'; + + ds.Xor(key); + BOOST_CHECK_EQUAL( + std::string(expected_xor.begin(), expected_xor.end()), + std::string(ds.begin(), ds.end())); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.cpp b/src/txdb.cpp index 21ecd65238..9738dea03d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -29,18 +29,8 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -void static BatchWriteCoins(CLevelDBBatch &batch, const uint256 &hash, const CCoins &coins) { - if (coins.IsPruned()) - batch.Erase(make_pair(DB_COINS, hash)); - else - batch.Write(make_pair(DB_COINS, hash), coins); -} - -void static BatchWriteHashBestChain(CLevelDBBatch &batch, const uint256 &hash) { - batch.Write(DB_BEST_BLOCK, hash); -} - -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) { +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) +{ } bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { @@ -59,12 +49,15 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CLevelDBBatch batch; + CLevelDBBatch batch(db.GetObfuscateKey()); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - BatchWriteCoins(batch, it->first, it->second.coins); + if (it->second.coins.IsPruned()) + batch.Erase(make_pair(DB_COINS, it->first)); + else + batch.Write(make_pair(DB_COINS, it->first), it->second.coins); changed++; } count++; @@ -72,7 +65,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { mapCoins.erase(itOld); } if (!hashBlock.IsNull()) - BatchWriteHashBestChain(batch, hashBlock); + batch.Write(DB_BEST_BLOCK, hashBlock); LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return db.WriteBatch(batch); @@ -158,7 +151,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) { - CLevelDBBatch batch; + CLevelDBBatch batch(GetObfuscateKey()); for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -174,7 +167,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) { - CLevelDBBatch batch; + CLevelDBBatch batch(GetObfuscateKey()); for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); |