From 3795e8152b678b9f805a395b144190a9f2fa2af4 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 22 Oct 2015 21:33:06 -0400 Subject: leveldbwrapper file rename to dbwrapper.* --- src/Makefile.am | 4 +- src/Makefile.test.include | 2 +- src/dbwrapper.cpp | 152 +++++++++++++++++++++ src/dbwrapper.h | 280 ++++++++++++++++++++++++++++++++++++++ src/leveldbwrapper.cpp | 152 --------------------- src/leveldbwrapper.h | 280 -------------------------------------- src/test/dbwrapper_tests.cpp | 207 ++++++++++++++++++++++++++++ src/test/leveldbwrapper_tests.cpp | 207 ---------------------------- src/txdb.h | 2 +- 9 files changed, 643 insertions(+), 643 deletions(-) create mode 100644 src/dbwrapper.cpp create mode 100644 src/dbwrapper.h delete mode 100644 src/leveldbwrapper.cpp delete mode 100644 src/leveldbwrapper.h create mode 100644 src/test/dbwrapper_tests.cpp delete mode 100644 src/test/leveldbwrapper_tests.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 0a16a863eb..312643cec3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -110,7 +110,7 @@ BITCOIN_CORE_H = \ init.h \ key.h \ keystore.h \ - leveldbwrapper.h \ + dbwrapper.h \ limitedmap.h \ main.h \ memusage.h \ @@ -188,7 +188,7 @@ libbitcoin_server_a_SOURCES = \ httprpc.cpp \ httpserver.cpp \ init.cpp \ - leveldbwrapper.cpp \ + dbwrapper.cpp \ main.cpp \ merkleblock.cpp \ miner.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 0fe8ea42ff..f23a8f41fc 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -54,7 +54,7 @@ BITCOIN_TESTS =\ test/hash_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ - test/leveldbwrapper_tests.cpp \ + test/dbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ test/miner_tests.cpp \ diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp new file mode 100644 index 0000000000..b6307cf0bf --- /dev/null +++ b/src/dbwrapper.cpp @@ -0,0 +1,152 @@ +// Copyright (c) 2012-2014 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "dbwrapper.h" + +#include "util.h" +#include "random.h" + +#include + +#include +#include +#include +#include +#include + +void HandleError(const leveldb::Status& status) throw(dbwrapper_error) +{ + if (status.ok()) + return; + LogPrintf("%s\n", status.ToString()); + if (status.IsCorruption()) + throw dbwrapper_error("Database corrupted"); + if (status.IsIOError()) + throw dbwrapper_error("Database I/O error"); + if (status.IsNotFound()) + throw dbwrapper_error("Database entry missing"); + throw dbwrapper_error("Unknown database error"); +} + +static leveldb::Options GetOptions(size_t nCacheSize) +{ + leveldb::Options options; + options.block_cache = leveldb::NewLRUCache(nCacheSize / 2); + options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously + options.filter_policy = leveldb::NewBloomFilterPolicy(10); + options.compression = leveldb::kNoCompression; + options.max_open_files = 64; + if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) { + // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error + // on corruption in later versions. + options.paranoid_checks = true; + } + return options; +} + +CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) +{ + penv = NULL; + readoptions.verify_checksums = true; + iteroptions.verify_checksums = true; + iteroptions.fill_cache = false; + syncoptions.sync = true; + options = GetOptions(nCacheSize); + options.create_if_missing = true; + if (fMemory) { + penv = leveldb::NewMemEnv(leveldb::Env::Default()); + options.env = penv; + } else { + if (fWipe) { + LogPrintf("Wiping LevelDB in %s\n", path.string()); + leveldb::Status result = leveldb::DestroyDB(path.string(), options); + HandleError(result); + } + TryCreateDirectory(path); + LogPrintf("Opening LevelDB in %s\n", path.string()); + } + leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); + HandleError(status); + LogPrintf("Opened LevelDB successfully\n"); + + // The base-case obfuscation key, which is a noop. + obfuscate_key = std::vector(OBFUSCATE_KEY_NUM_BYTES, '\000'); + + bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key); + + if (!key_exists && obfuscate && IsEmpty()) { + // Initialize non-degenerate obfuscation if it won't upset + // existing, non-obfuscated data. + std::vector new_key = CreateObfuscateKey(); + + // Write `new_key` so we don't obfuscate the key with itself + Write(OBFUSCATE_KEY_KEY, new_key); + obfuscate_key = new_key; + + LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex()); + } + + LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex()); +} + +CDBWrapper::~CDBWrapper() +{ + delete pdb; + pdb = NULL; + delete options.filter_policy; + options.filter_policy = NULL; + delete options.block_cache; + options.block_cache = NULL; + delete penv; + options.env = NULL; +} + +bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) throw(dbwrapper_error) +{ + leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); + HandleError(status); + return true; +} + +// Prefixed with null character to avoid collisions with other keys +// +// We must use a string constructor which specifies length so that we copy +// past the null-terminator. +const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14); + +const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; + +/** + * Returns a string (consisting of 8 random bytes) suitable for use as an + * obfuscating XOR key. + */ +std::vector CDBWrapper::CreateObfuscateKey() const +{ + unsigned char buff[OBFUSCATE_KEY_NUM_BYTES]; + GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES); + return std::vector(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]); + +} + +bool CDBWrapper::IsEmpty() +{ + boost::scoped_ptr it(NewIterator()); + it->SeekToFirst(); + return !(it->Valid()); +} + +const std::vector& CDBWrapper::GetObfuscateKey() const +{ + return obfuscate_key; +} + +std::string CDBWrapper::GetObfuscateKeyHex() const +{ + return HexStr(obfuscate_key); +} + +CDBIterator::~CDBIterator() { delete piter; } +bool CDBIterator::Valid() { return piter->Valid(); } +void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } +void CDBIterator::Next() { piter->Next(); } diff --git a/src/dbwrapper.h b/src/dbwrapper.h new file mode 100644 index 0000000000..aa28767508 --- /dev/null +++ b/src/dbwrapper.h @@ -0,0 +1,280 @@ +// Copyright (c) 2012-2014 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_DBWRAPPER_H +#define BITCOIN_DBWRAPPER_H + +#include "clientversion.h" +#include "serialize.h" +#include "streams.h" +#include "util.h" +#include "utilstrencodings.h" +#include "version.h" + +#include + +#include +#include + +class dbwrapper_error : public std::runtime_error +{ +public: + dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {} +}; + +void HandleError(const leveldb::Status& status) throw(dbwrapper_error); + +/** Batch of changes queued to be written to a CDBWrapper */ +class CDBBatch +{ + friend class CDBWrapper; + +private: + leveldb::WriteBatch batch; + const std::vector *obfuscate_key; + +public: + /** + * @param[in] obfuscate_key If passed, XOR data with this key. + */ + CDBBatch(const std::vector *obfuscate_key) : obfuscate_key(obfuscate_key) { }; + + template + void Write(const K& key, const V& value) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey << key; + leveldb::Slice slKey(&ssKey[0], ssKey.size()); + + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + ssValue.reserve(ssValue.GetSerializeSize(value)); + ssValue << value; + ssValue.Xor(*obfuscate_key); + leveldb::Slice slValue(&ssValue[0], ssValue.size()); + + batch.Put(slKey, slValue); + } + + template + void Erase(const K& key) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey << key; + leveldb::Slice slKey(&ssKey[0], ssKey.size()); + + batch.Delete(slKey); + } +}; + +class CDBIterator +{ +private: + leveldb::Iterator *piter; + const std::vector *obfuscate_key; + +public: + + /** + * @param[in] piterIn The original leveldb iterator. + * @param[in] obfuscate_key If passed, XOR data with this key. + */ + CDBIterator(leveldb::Iterator *piterIn, const std::vector* obfuscate_key) : + piter(piterIn), obfuscate_key(obfuscate_key) { }; + ~CDBIterator(); + + bool Valid(); + + void SeekToFirst(); + + template void Seek(const K& key) { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey << key; + leveldb::Slice slKey(&ssKey[0], ssKey.size()); + piter->Seek(slKey); + } + + void Next(); + + template bool GetKey(K& key) { + leveldb::Slice slKey = piter->key(); + try { + CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + ssKey >> key; + } catch(std::exception &e) { + return false; + } + return true; + } + + unsigned int GetKeySize() { + return piter->key().size(); + } + + template bool GetValue(V& value) { + leveldb::Slice slValue = piter->value(); + try { + CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); + ssValue.Xor(*obfuscate_key); + ssValue >> value; + } catch(std::exception &e) { + return false; + } + return true; + } + + unsigned int GetValueSize() { + return piter->value().size(); + } + +}; + +class CDBWrapper +{ +private: + //! custom environment this database is using (may be NULL in case of default environment) + leveldb::Env* penv; + + //! database options used + leveldb::Options options; + + //! options used when reading from the database + leveldb::ReadOptions readoptions; + + //! options used when iterating over values of the database + leveldb::ReadOptions iteroptions; + + //! options used when writing to the database + leveldb::WriteOptions writeoptions; + + //! options used when sync writing to the database + leveldb::WriteOptions syncoptions; + + //! the database itself + leveldb::DB* pdb; + + //! a key used for optional XOR-obfuscation of the database + std::vector obfuscate_key; + + //! the key under which the obfuscation key is stored + static const std::string OBFUSCATE_KEY_KEY; + + //! the length of the obfuscate key in number of bytes + static const unsigned int OBFUSCATE_KEY_NUM_BYTES; + + std::vector CreateObfuscateKey() const; + +public: + /** + * @param[in] path Location in the filesystem where leveldb data will be stored. + * @param[in] nCacheSize Configures various leveldb cache settings. + * @param[in] fMemory If true, use leveldb's memory environment. + * @param[in] fWipe If true, remove all existing data. + * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR + * with a zero'd byte array. + */ + CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); + ~CDBWrapper(); + + template + bool Read(const K& key, V& value) const throw(dbwrapper_error) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey << key; + leveldb::Slice slKey(&ssKey[0], ssKey.size()); + + std::string strValue; + leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); + if (!status.ok()) { + if (status.IsNotFound()) + return false; + LogPrintf("LevelDB read failure: %s\n", status.ToString()); + HandleError(status); + } + try { + CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); + ssValue.Xor(obfuscate_key); + ssValue >> value; + } catch (const std::exception&) { + return false; + } + return true; + } + + template + bool Write(const K& key, const V& value, bool fSync = false) throw(dbwrapper_error) + { + CDBBatch batch(&obfuscate_key); + batch.Write(key, value); + return WriteBatch(batch, fSync); + } + + template + bool Exists(const K& key) const throw(dbwrapper_error) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey << key; + leveldb::Slice slKey(&ssKey[0], ssKey.size()); + + std::string strValue; + leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); + if (!status.ok()) { + if (status.IsNotFound()) + return false; + LogPrintf("LevelDB read failure: %s\n", status.ToString()); + HandleError(status); + } + return true; + } + + template + bool Erase(const K& key, bool fSync = false) throw(dbwrapper_error) + { + CDBBatch batch(&obfuscate_key); + batch.Erase(key); + return WriteBatch(batch, fSync); + } + + bool WriteBatch(CDBBatch& batch, bool fSync = false) throw(dbwrapper_error); + + // not available for LevelDB; provide for compatibility with BDB + bool Flush() + { + return true; + } + + bool Sync() throw(dbwrapper_error) + { + CDBBatch batch(&obfuscate_key); + return WriteBatch(batch, true); + } + + CDBIterator *NewIterator() + { + return new CDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key); + } + + /** + * Return true if the database managed by this class contains no entries. + */ + bool IsEmpty(); + + /** + * Accessor for obfuscate_key. + */ + const std::vector& GetObfuscateKey() const; + + /** + * Return the obfuscate_key as a hex-formatted string. + */ + std::string GetObfuscateKeyHex() const; + +}; + +#endif // BITCOIN_DBWRAPPER_H + diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp deleted file mode 100644 index 6ecf7c7f0e..0000000000 --- a/src/leveldbwrapper.cpp +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) 2012-2014 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "leveldbwrapper.h" - -#include "util.h" -#include "random.h" - -#include - -#include -#include -#include -#include -#include - -void HandleError(const leveldb::Status& status) throw(dbwrapper_error) -{ - if (status.ok()) - return; - LogPrintf("%s\n", status.ToString()); - if (status.IsCorruption()) - throw dbwrapper_error("Database corrupted"); - if (status.IsIOError()) - throw dbwrapper_error("Database I/O error"); - if (status.IsNotFound()) - throw dbwrapper_error("Database entry missing"); - throw dbwrapper_error("Unknown database error"); -} - -static leveldb::Options GetOptions(size_t nCacheSize) -{ - leveldb::Options options; - options.block_cache = leveldb::NewLRUCache(nCacheSize / 2); - options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously - options.filter_policy = leveldb::NewBloomFilterPolicy(10); - options.compression = leveldb::kNoCompression; - options.max_open_files = 64; - if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) { - // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error - // on corruption in later versions. - options.paranoid_checks = true; - } - return options; -} - -CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) -{ - penv = NULL; - readoptions.verify_checksums = true; - iteroptions.verify_checksums = true; - iteroptions.fill_cache = false; - syncoptions.sync = true; - options = GetOptions(nCacheSize); - options.create_if_missing = true; - if (fMemory) { - penv = leveldb::NewMemEnv(leveldb::Env::Default()); - options.env = penv; - } else { - if (fWipe) { - LogPrintf("Wiping LevelDB in %s\n", path.string()); - leveldb::Status result = leveldb::DestroyDB(path.string(), options); - HandleError(result); - } - TryCreateDirectory(path); - LogPrintf("Opening LevelDB in %s\n", path.string()); - } - leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); - HandleError(status); - LogPrintf("Opened LevelDB successfully\n"); - - // The base-case obfuscation key, which is a noop. - obfuscate_key = std::vector(OBFUSCATE_KEY_NUM_BYTES, '\000'); - - bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key); - - if (!key_exists && obfuscate && IsEmpty()) { - // Initialize non-degenerate obfuscation if it won't upset - // existing, non-obfuscated data. - std::vector new_key = CreateObfuscateKey(); - - // Write `new_key` so we don't obfuscate the key with itself - Write(OBFUSCATE_KEY_KEY, new_key); - obfuscate_key = new_key; - - LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex()); - } - - LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex()); -} - -CDBWrapper::~CDBWrapper() -{ - delete pdb; - pdb = NULL; - delete options.filter_policy; - options.filter_policy = NULL; - delete options.block_cache; - options.block_cache = NULL; - delete penv; - options.env = NULL; -} - -bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) throw(dbwrapper_error) -{ - leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); - HandleError(status); - return true; -} - -// Prefixed with null character to avoid collisions with other keys -// -// We must use a string constructor which specifies length so that we copy -// past the null-terminator. -const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14); - -const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; - -/** - * Returns a string (consisting of 8 random bytes) suitable for use as an - * obfuscating XOR key. - */ -std::vector CDBWrapper::CreateObfuscateKey() const -{ - unsigned char buff[OBFUSCATE_KEY_NUM_BYTES]; - GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES); - return std::vector(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]); - -} - -bool CDBWrapper::IsEmpty() -{ - boost::scoped_ptr it(NewIterator()); - it->SeekToFirst(); - return !(it->Valid()); -} - -const std::vector& CDBWrapper::GetObfuscateKey() const -{ - return obfuscate_key; -} - -std::string CDBWrapper::GetObfuscateKeyHex() const -{ - return HexStr(obfuscate_key); -} - -CDBIterator::~CDBIterator() { delete piter; } -bool CDBIterator::Valid() { return piter->Valid(); } -void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } -void CDBIterator::Next() { piter->Next(); } diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h deleted file mode 100644 index c8fc457d90..0000000000 --- a/src/leveldbwrapper.h +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright (c) 2012-2014 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_LEVELDBWRAPPER_H -#define BITCOIN_LEVELDBWRAPPER_H - -#include "clientversion.h" -#include "serialize.h" -#include "streams.h" -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" - -#include - -#include -#include - -class dbwrapper_error : public std::runtime_error -{ -public: - dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {} -}; - -void HandleError(const leveldb::Status& status) throw(dbwrapper_error); - -/** Batch of changes queued to be written to a CDBWrapper */ -class CDBBatch -{ - friend class CDBWrapper; - -private: - leveldb::WriteBatch batch; - const std::vector *obfuscate_key; - -public: - /** - * @param[in] obfuscate_key If passed, XOR data with this key. - */ - CDBBatch(const std::vector *obfuscate_key) : obfuscate_key(obfuscate_key) { }; - - template - void Write(const K& key, const V& value) - { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); - ssKey << key; - leveldb::Slice slKey(&ssKey[0], ssKey.size()); - - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - ssValue.reserve(ssValue.GetSerializeSize(value)); - ssValue << value; - ssValue.Xor(*obfuscate_key); - leveldb::Slice slValue(&ssValue[0], ssValue.size()); - - batch.Put(slKey, slValue); - } - - template - void Erase(const K& key) - { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); - ssKey << key; - leveldb::Slice slKey(&ssKey[0], ssKey.size()); - - batch.Delete(slKey); - } -}; - -class CDBIterator -{ -private: - leveldb::Iterator *piter; - const std::vector *obfuscate_key; - -public: - - /** - * @param[in] piterIn The original leveldb iterator. - * @param[in] obfuscate_key If passed, XOR data with this key. - */ - CDBIterator(leveldb::Iterator *piterIn, const std::vector* obfuscate_key) : - piter(piterIn), obfuscate_key(obfuscate_key) { }; - ~CDBIterator(); - - bool Valid(); - - void SeekToFirst(); - - template void Seek(const K& key) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); - ssKey << key; - leveldb::Slice slKey(&ssKey[0], ssKey.size()); - piter->Seek(slKey); - } - - void Next(); - - template bool GetKey(K& key) { - leveldb::Slice slKey = piter->key(); - try { - CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); - ssKey >> key; - } catch(std::exception &e) { - return false; - } - return true; - } - - unsigned int GetKeySize() { - return piter->key().size(); - } - - template bool GetValue(V& value) { - leveldb::Slice slValue = piter->value(); - try { - CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); - ssValue.Xor(*obfuscate_key); - ssValue >> value; - } catch(std::exception &e) { - return false; - } - return true; - } - - unsigned int GetValueSize() { - return piter->value().size(); - } - -}; - -class CDBWrapper -{ -private: - //! custom environment this database is using (may be NULL in case of default environment) - leveldb::Env* penv; - - //! database options used - leveldb::Options options; - - //! options used when reading from the database - leveldb::ReadOptions readoptions; - - //! options used when iterating over values of the database - leveldb::ReadOptions iteroptions; - - //! options used when writing to the database - leveldb::WriteOptions writeoptions; - - //! options used when sync writing to the database - leveldb::WriteOptions syncoptions; - - //! the database itself - leveldb::DB* pdb; - - //! a key used for optional XOR-obfuscation of the database - std::vector obfuscate_key; - - //! the key under which the obfuscation key is stored - static const std::string OBFUSCATE_KEY_KEY; - - //! the length of the obfuscate key in number of bytes - static const unsigned int OBFUSCATE_KEY_NUM_BYTES; - - std::vector CreateObfuscateKey() const; - -public: - /** - * @param[in] path Location in the filesystem where leveldb data will be stored. - * @param[in] nCacheSize Configures various leveldb cache settings. - * @param[in] fMemory If true, use leveldb's memory environment. - * @param[in] fWipe If true, remove all existing data. - * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR - * with a zero'd byte array. - */ - CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); - ~CDBWrapper(); - - template - bool Read(const K& key, V& value) const throw(dbwrapper_error) - { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); - ssKey << key; - leveldb::Slice slKey(&ssKey[0], ssKey.size()); - - std::string strValue; - leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); - if (!status.ok()) { - if (status.IsNotFound()) - return false; - LogPrintf("LevelDB read failure: %s\n", status.ToString()); - HandleError(status); - } - try { - CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); - ssValue.Xor(obfuscate_key); - ssValue >> value; - } catch (const std::exception&) { - return false; - } - return true; - } - - template - bool Write(const K& key, const V& value, bool fSync = false) throw(dbwrapper_error) - { - CDBBatch batch(&obfuscate_key); - batch.Write(key, value); - return WriteBatch(batch, fSync); - } - - template - bool Exists(const K& key) const throw(dbwrapper_error) - { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); - ssKey << key; - leveldb::Slice slKey(&ssKey[0], ssKey.size()); - - std::string strValue; - leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); - if (!status.ok()) { - if (status.IsNotFound()) - return false; - LogPrintf("LevelDB read failure: %s\n", status.ToString()); - HandleError(status); - } - return true; - } - - template - bool Erase(const K& key, bool fSync = false) throw(dbwrapper_error) - { - CDBBatch batch(&obfuscate_key); - batch.Erase(key); - return WriteBatch(batch, fSync); - } - - bool WriteBatch(CDBBatch& batch, bool fSync = false) throw(dbwrapper_error); - - // not available for LevelDB; provide for compatibility with BDB - bool Flush() - { - return true; - } - - bool Sync() throw(dbwrapper_error) - { - CDBBatch batch(&obfuscate_key); - return WriteBatch(batch, true); - } - - CDBIterator *NewIterator() - { - return new CDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key); - } - - /** - * Return true if the database managed by this class contains no entries. - */ - bool IsEmpty(); - - /** - * Accessor for obfuscate_key. - */ - const std::vector& GetObfuscateKey() const; - - /** - * Return the obfuscate_key as a hex-formatted string. - */ - std::string GetObfuscateKeyHex() const; - -}; - -#endif // BITCOIN_LEVELDBWRAPPER_H - diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp new file mode 100644 index 0000000000..8b6b0697ab --- /dev/null +++ b/src/test/dbwrapper_tests.cpp @@ -0,0 +1,207 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "dbwrapper.h" +#include "uint256.h" +#include "random.h" +#include "test/test_bitcoin.h" + +#include // for 'operator+=()' +#include +#include + +using namespace std; +using namespace boost::assign; // bring 'operator+=()' into scope +using namespace boost::filesystem; + +// Test if a string consists entirely of null characters +bool is_null_key(const vector& key) { + bool isnull = true; + + for (unsigned int i = 0; i < key.size(); i++) + isnull &= (key[i] == '\x00'); + + return isnull; +} + +BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(dbwrapper) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + // Ensure that we're doing real obfuscation when obfuscate=true + BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey())); + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + } +} + +// Test batch operations +BOOST_AUTO_TEST_CASE(dbwrapper_batch) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + + char key = 'i'; + uint256 in = GetRandHash(); + char key2 = 'j'; + uint256 in2 = GetRandHash(); + char key3 = 'k'; + uint256 in3 = GetRandHash(); + + uint256 res; + CDBBatch batch(&dbw.GetObfuscateKey()); + + batch.Write(key, in); + batch.Write(key2, in2); + batch.Write(key3, in3); + + // Remove key3 before it's even been written + batch.Erase(key3); + + dbw.WriteBatch(batch); + + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + BOOST_CHECK(dbw.Read(key2, res)); + BOOST_CHECK_EQUAL(res.ToString(), in2.ToString()); + + // key3 never should've been written + BOOST_CHECK(dbw.Read(key3, res) == false); + } +} + +BOOST_AUTO_TEST_CASE(dbwrapper_iterator) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + + // The two keys are intentionally chosen for ordering + char key = 'j'; + uint256 in = GetRandHash(); + BOOST_CHECK(dbw.Write(key, in)); + char key2 = 'k'; + uint256 in2 = GetRandHash(); + BOOST_CHECK(dbw.Write(key2, in2)); + + boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); + + // Be sure to seek past the obfuscation key (if it exists) + it->Seek(key); + + char key_res; + uint256 val_res; + + it->GetKey(key_res); + it->GetValue(val_res); + BOOST_CHECK_EQUAL(key_res, key); + BOOST_CHECK_EQUAL(val_res.ToString(), in.ToString()); + + it->Next(); + + it->GetKey(key_res); + it->GetValue(val_res); + BOOST_CHECK_EQUAL(key_res, key2); + BOOST_CHECK_EQUAL(val_res.ToString(), in2.ToString()); + + it->Next(); + BOOST_CHECK_EQUAL(it->Valid(), false); + } +} + +// Test that we do not obfuscation if there is existing data. +BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) +{ + // We're going to share this path between two wrappers + path ph = temp_directory_path() / unique_path(); + create_directories(ph); + + // Set up a non-obfuscated wrapper to write some initial data. + CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw->Write(key, in)); + BOOST_CHECK(dbw->Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + + // Call the destructor to free leveldb LOCK + delete dbw; + + // Now, set up another wrapper that wants to obfuscate the same directory + CDBWrapper odbw(ph, (1 << 10), false, false, true); + + // Check that the key/val we wrote with unobfuscated wrapper exists and + // is readable. + uint256 res2; + BOOST_CHECK(odbw.Read(key, res2)); + BOOST_CHECK_EQUAL(res2.ToString(), in.ToString()); + + BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data + BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string + + uint256 in2 = GetRandHash(); + uint256 res3; + + // Check that we can write successfully + BOOST_CHECK(odbw.Write(key, in2)); + BOOST_CHECK(odbw.Read(key, res3)); + BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); +} + +// Ensure that we start obfuscating during a reindex. +BOOST_AUTO_TEST_CASE(existing_data_reindex) +{ + // We're going to share this path between two wrappers + path ph = temp_directory_path() / unique_path(); + create_directories(ph); + + // Set up a non-obfuscated wrapper to write some initial data. + CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw->Write(key, in)); + BOOST_CHECK(dbw->Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + + // Call the destructor to free leveldb LOCK + delete dbw; + + // Simulate a -reindex by wiping the existing data store + CDBWrapper odbw(ph, (1 << 10), false, true, true); + + // Check that the key/val we wrote with unobfuscated wrapper doesn't exist + uint256 res2; + BOOST_CHECK(!odbw.Read(key, res2)); + BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey())); + + uint256 in2 = GetRandHash(); + uint256 res3; + + // Check that we can write successfully + BOOST_CHECK(odbw.Write(key, in2)); + BOOST_CHECK(odbw.Read(key, res3)); + BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp deleted file mode 100644 index 8defb8a8ff..0000000000 --- a/src/test/leveldbwrapper_tests.cpp +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) 2012-2013 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "leveldbwrapper.h" -#include "uint256.h" -#include "random.h" -#include "test/test_bitcoin.h" - -#include // for 'operator+=()' -#include -#include - -using namespace std; -using namespace boost::assign; // bring 'operator+=()' into scope -using namespace boost::filesystem; - -// Test if a string consists entirely of null characters -bool is_null_key(const vector& key) { - bool isnull = true; - - for (unsigned int i = 0; i < key.size(); i++) - isnull &= (key[i] == '\x00'); - - return isnull; -} - -BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(dbwrapper) -{ - // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; - path ph = temp_directory_path() / unique_path(); - CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); - char key = 'k'; - uint256 in = GetRandHash(); - uint256 res; - - // Ensure that we're doing real obfuscation when obfuscate=true - BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey())); - - BOOST_CHECK(dbw.Write(key, in)); - BOOST_CHECK(dbw.Read(key, res)); - BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); - } -} - -// Test batch operations -BOOST_AUTO_TEST_CASE(dbwrapper_batch) -{ - // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; - path ph = temp_directory_path() / unique_path(); - CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); - - char key = 'i'; - uint256 in = GetRandHash(); - char key2 = 'j'; - uint256 in2 = GetRandHash(); - char key3 = 'k'; - uint256 in3 = GetRandHash(); - - uint256 res; - CDBBatch batch(&dbw.GetObfuscateKey()); - - batch.Write(key, in); - batch.Write(key2, in2); - batch.Write(key3, in3); - - // Remove key3 before it's even been written - batch.Erase(key3); - - dbw.WriteBatch(batch); - - BOOST_CHECK(dbw.Read(key, res)); - BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); - BOOST_CHECK(dbw.Read(key2, res)); - BOOST_CHECK_EQUAL(res.ToString(), in2.ToString()); - - // key3 never should've been written - BOOST_CHECK(dbw.Read(key3, res) == false); - } -} - -BOOST_AUTO_TEST_CASE(dbwrapper_iterator) -{ - // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; - path ph = temp_directory_path() / unique_path(); - CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); - - // The two keys are intentionally chosen for ordering - char key = 'j'; - uint256 in = GetRandHash(); - BOOST_CHECK(dbw.Write(key, in)); - char key2 = 'k'; - uint256 in2 = GetRandHash(); - BOOST_CHECK(dbw.Write(key2, in2)); - - boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); - - // Be sure to seek past the obfuscation key (if it exists) - it->Seek(key); - - char key_res; - uint256 val_res; - - it->GetKey(key_res); - it->GetValue(val_res); - BOOST_CHECK_EQUAL(key_res, key); - BOOST_CHECK_EQUAL(val_res.ToString(), in.ToString()); - - it->Next(); - - it->GetKey(key_res); - it->GetValue(val_res); - BOOST_CHECK_EQUAL(key_res, key2); - BOOST_CHECK_EQUAL(val_res.ToString(), in2.ToString()); - - it->Next(); - BOOST_CHECK_EQUAL(it->Valid(), false); - } -} - -// Test that we do not obfuscation if there is existing data. -BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) -{ - // We're going to share this path between two wrappers - path ph = temp_directory_path() / unique_path(); - create_directories(ph); - - // Set up a non-obfuscated wrapper to write some initial data. - CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); - char key = 'k'; - uint256 in = GetRandHash(); - uint256 res; - - BOOST_CHECK(dbw->Write(key, in)); - BOOST_CHECK(dbw->Read(key, res)); - BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); - - // Call the destructor to free leveldb LOCK - delete dbw; - - // Now, set up another wrapper that wants to obfuscate the same directory - CDBWrapper odbw(ph, (1 << 10), false, false, true); - - // Check that the key/val we wrote with unobfuscated wrapper exists and - // is readable. - uint256 res2; - BOOST_CHECK(odbw.Read(key, res2)); - BOOST_CHECK_EQUAL(res2.ToString(), in.ToString()); - - BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data - BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string - - uint256 in2 = GetRandHash(); - uint256 res3; - - // Check that we can write successfully - BOOST_CHECK(odbw.Write(key, in2)); - BOOST_CHECK(odbw.Read(key, res3)); - BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); -} - -// Ensure that we start obfuscating during a reindex. -BOOST_AUTO_TEST_CASE(existing_data_reindex) -{ - // We're going to share this path between two wrappers - path ph = temp_directory_path() / unique_path(); - create_directories(ph); - - // Set up a non-obfuscated wrapper to write some initial data. - CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); - char key = 'k'; - uint256 in = GetRandHash(); - uint256 res; - - BOOST_CHECK(dbw->Write(key, in)); - BOOST_CHECK(dbw->Read(key, res)); - BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); - - // Call the destructor to free leveldb LOCK - delete dbw; - - // Simulate a -reindex by wiping the existing data store - CDBWrapper odbw(ph, (1 << 10), false, true, true); - - // Check that the key/val we wrote with unobfuscated wrapper doesn't exist - uint256 res2; - BOOST_CHECK(!odbw.Read(key, res2)); - BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey())); - - uint256 in2 = GetRandHash(); - uint256 res3; - - // Check that we can write successfully - BOOST_CHECK(odbw.Write(key, in2)); - BOOST_CHECK(odbw.Read(key, res3)); - BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.h b/src/txdb.h index 1e8fccea45..586ab55d0d 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -7,7 +7,7 @@ #define BITCOIN_TXDB_H #include "coins.h" -#include "leveldbwrapper.h" +#include "dbwrapper.h" #include #include -- cgit v1.2.3