diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | configure.ac | 44 | ||||
| -rw-r--r-- | doc/release-notes.md | 10 | ||||
| -rw-r--r-- | doc/zmq.md | 12 | ||||
| -rw-r--r--[-rwxr-xr-x] | qa/pull-tester/run-bitcoind-for-test.sh.in | 0 | ||||
| -rw-r--r-- | src/Makefile.am | 5 | ||||
| -rw-r--r-- | src/Makefile.bench.include | 45 | ||||
| -rw-r--r-- | src/Makefile.test.include | 2 | ||||
| -rw-r--r-- | src/bench/Examples.cpp | 34 | ||||
| -rw-r--r-- | src/bench/bench.cpp | 68 | ||||
| -rw-r--r-- | src/bench/bench.h | 71 | ||||
| -rw-r--r-- | src/bench/bench_bitcoin.cpp | 21 | ||||
| -rw-r--r-- | src/leveldbwrapper.cpp | 60 | ||||
| -rw-r--r-- | src/leveldbwrapper.h | 53 | ||||
| -rw-r--r-- | src/streams.h | 23 | ||||
| -rw-r--r-- | src/test/leveldbwrapper_tests.cpp | 128 | ||||
| -rw-r--r-- | src/test/streams_tests.cpp | 67 | ||||
| -rw-r--r-- | src/txdb.cpp | 27 | ||||
| -rw-r--r-- | src/util.cpp | 7 |
20 files changed, 635 insertions, 48 deletions
diff --git a/.gitignore b/.gitignore index 9f1e522803..a8035731d1 100644 --- a/.gitignore +++ b/.gitignore @@ -104,7 +104,7 @@ linux-coverage-build linux-build win32-build qa/pull-tester/run-bitcoind-for-test.sh -qa/pull-tester/tests-config.sh +qa/pull-tester/tests_config.py qa/pull-tester/cache/* qa/pull-tester/test.*/* @@ -1,4 +1,4 @@ -Bitcoin Core integration/staging tree +Bitcoin Core integration/staging tree ===================================== [](https://travis-ci.org/bitcoin/bitcoin) @@ -28,7 +28,7 @@ Development Process The `master` branch is regularly built and tested, but is not guaranteed to be completely stable. [Tags](https://github.com/bitcoin/bitcoin/tags) are created -regularly to indicate new official, stable release versions of Bitcoin. +regularly to indicate new official, stable release versions of Bitcoin Core. The contribution workflow is described in [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/configure.ac b/configure.ac index d8c7e8b779..4318aafa55 100644 --- a/configure.ac +++ b/configure.ac @@ -91,6 +91,11 @@ AC_ARG_ENABLE(tests, [use_tests=$enableval], [use_tests=yes]) +AC_ARG_ENABLE(bench, + AS_HELP_STRING([--enable-bench],[compile benchmarks (default is yes)]), + [use_bench=$enableval], + [use_bench=yes]) + AC_ARG_WITH([comparison-tool], AS_HELP_STRING([--with-comparison-tool],[path to java comparison tool (requires --enable-tests)]), [use_comparison_tool=$withval], @@ -676,6 +681,16 @@ if test x$use_pkgconfig = xyes; then PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) fi fi + + if test "x$use_zmq" = "xyes"; then + PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], + [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], + [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) + AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) + use_zmq=no]) + else + AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) + fi ] ) else @@ -693,6 +708,20 @@ else fi fi + if test "x$use_zmq" = "xyes"; then + AC_CHECK_HEADER([zmq.h], + [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], + [AC_MSG_WARN([zmq.h not found, disabling zmq support]) + use_zmq=no + AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) + AC_CHECK_LIB([zmq],[zmq_ctx_shutdown],ZMQ_LIBS=-lzmq, + [AC_MSG_WARN([libzmq >= 4.0 not found, disabling zmq support]) + use_zmq=no + AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) + else + AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) + fi + BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], BITCOIN_QT_FAIL(libprotobuf not found))) if test x$use_qr != xno; then BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) @@ -824,20 +853,6 @@ if test x$bitcoin_enable_qt != xno; then fi fi -# conditional search for and use libzmq -AC_MSG_CHECKING([whether to build ZMQ support]) -if test "x$use_zmq" = "xyes"; then - AC_MSG_RESULT([yes]) - PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) - use_zmq=no]) -else - AC_MSG_RESULT([no, --disable-zmq used]) - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) -fi - AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) AC_MSG_CHECKING([whether to build test_bitcoin]) @@ -866,6 +881,7 @@ AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$use_tests = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$use_tests$bitcoin_enable_qt_test = xyesyes]) +AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([USE_COMPARISON_TOOL],[test x$use_comparison_tool != xno]) diff --git a/doc/release-notes.md b/doc/release-notes.md index 70623a3939..78ab3516f6 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -105,6 +105,16 @@ In this version, it is only enforced for peers that send protocol versions removed. It is recommended to update SPV clients to check for the `NODE_BLOOM` service bit for nodes that report versions newer than 70011. +Any sequence of pushdatas in OP_RETURN outputs now allowed +---------------------------------------------------------- + +Previously OP_RETURN outputs with a payload were only relayed and mined if they +had a single pushdata. This restriction has been lifted to allow any +combination of data pushes and numeric constant opcodes (OP_1 to OP_16). The +limit on OP_RETURN output size is now applied to the entire serialized +scriptPubKey, 83 bytes by default. (the previous 80 byte default plus three +bytes overhead) + Merkle branches removed from wallet ----------------------------------- diff --git a/doc/zmq.md b/doc/zmq.md index 358d29d046..484b005cf4 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -43,14 +43,14 @@ operation. ## Enabling -By default, the ZeroMQ port functionality is enabled. Two steps are -required to enable--compiling in the ZeroMQ code, and configuring -runtime operation on the command-line or configuration file. +By default, the ZeroMQ feature is automatically compiled in if the +necessary prerequisites are found. To disable, use --disable-zmq +during the *configure* step of building bitcoind: - $ ./configure --enable-zmq (other options) + $ ./configure --disable-zmq (other options) -This will produce a binary that is capable of providing the ZeroMQ -facility, but will not do so until also configured properly. +To actually enable operation, one must set the appropriate options on +the commandline or in the configuration file. ## Usage diff --git a/qa/pull-tester/run-bitcoind-for-test.sh.in b/qa/pull-tester/run-bitcoind-for-test.sh.in index 14ae08e4e5..14ae08e4e5 100755..100644 --- a/qa/pull-tester/run-bitcoind-for-test.sh.in +++ b/qa/pull-tester/run-bitcoind-for-test.sh.in diff --git a/src/Makefile.am b/src/Makefile.am index 462774389a..c3f19926df 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -64,6 +64,7 @@ endif bin_PROGRAMS = TESTS = +BENCHMARKS = if BUILD_BITCOIND bin_PROGRAMS += bitcoind @@ -439,6 +440,10 @@ if ENABLE_TESTS include Makefile.test.include endif +if ENABLE_BENCH +include Makefile.bench.include +endif + if ENABLE_QT include Makefile.qt.include endif diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include new file mode 100644 index 0000000000..61fe9e287d --- /dev/null +++ b/src/Makefile.bench.include @@ -0,0 +1,45 @@ +bin_PROGRAMS += bench/bench_bitcoin +BENCH_SRCDIR = bench +BENCH_BINARY = bench/bench_bitcoin$(EXEEXT) + + +bench_bench_bitcoin_SOURCES = \ + bench/bench_bitcoin.cpp \ + bench/bench.cpp \ + bench/bench.h \ + bench/Examples.cpp + +bench_bench_bitcoin_CPPFLAGS = $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ +bench_bench_bitcoin_LDADD = \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UNIVALUE) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBLEVELDB) \ + $(LIBMEMENV) \ + $(LIBSECP256K1) + +if ENABLE_ZMQ +bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +endif + +if ENABLE_WALLET +bench_bench_bitcoin_LDADD += $(LIBBITCOIN_WALLET) +endif + +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) +bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + + +CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno + +CLEANFILES += $(CLEAN_BITCOIN_BENCH) + +bitcoin_bench: $(BENCH_BINARY) + +bench: $(BENCH_BINARY) FORCE + $(BENCH_BINARY) + +bitcoin_bench_clean : FORCE + rm -f $(CLEAN_BITCOIN_BENCH) $(bench_bench_bitcoin_OBJECTS) $(BENCH_BINARY) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9a6e43631b..a3b6c4d9b1 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -53,6 +53,7 @@ BITCOIN_TESTS =\ test/hash_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ + test/leveldbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ test/miner_tests.cpp \ @@ -73,6 +74,7 @@ BITCOIN_TESTS =\ test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ + test/streams_tests.cpp \ test/test_bitcoin.cpp \ test/test_bitcoin.h \ test/timedata_tests.cpp \ diff --git a/src/bench/Examples.cpp b/src/bench/Examples.cpp new file mode 100644 index 0000000000..b6b020a971 --- /dev/null +++ b/src/bench/Examples.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" +#include "main.h" +#include "utiltime.h" + +// Sanity test: this should loop ten times, and +// min/max/average should be close to 100ms. +static void Sleep100ms(benchmark::State& state) +{ + while (state.KeepRunning()) { + MilliSleep(100); + } +} + +BENCHMARK(Sleep100ms); + +// Extremely fast-running benchmark: +#include <math.h> + +volatile double sum = 0.0; // volatile, global so not optimized away + +static void Trig(benchmark::State& state) +{ + double d = 0.01; + while (state.KeepRunning()) { + sum += sin(d); + d += 0.000001; + } +} + +BENCHMARK(Trig); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp new file mode 100644 index 0000000000..89c3b0cc2a --- /dev/null +++ b/src/bench/bench.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "bench.h" +#include <iostream> +#include <sys/time.h> + +using namespace benchmark; + +std::map<std::string, BenchFunction> BenchRunner::benchmarks; + +static double gettimedouble(void) { + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_usec * 0.000001 + tv.tv_sec; +} + +BenchRunner::BenchRunner(std::string name, BenchFunction func) +{ + benchmarks.insert(std::make_pair(name, func)); +} + +void +BenchRunner::RunAll(double elapsedTimeForOne) +{ + std::cout << "Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "\n"; + + for (std::map<std::string,BenchFunction>::iterator it = benchmarks.begin(); + it != benchmarks.end(); ++it) { + + State state(it->first, elapsedTimeForOne); + BenchFunction& func = it->second; + func(state); + } +} + +bool State::KeepRunning() +{ + double now; + if (count == 0) { + beginTime = now = gettimedouble(); + } + else { + // timeCheckCount is used to avoid calling gettime most of the time, + // so benchmarks that run very quickly get consistent results. + if ((count+1)%timeCheckCount != 0) { + ++count; + return true; // keep going + } + now = gettimedouble(); + double elapsedOne = (now - lastTime)/timeCheckCount; + if (elapsedOne < minTime) minTime = elapsedOne; + if (elapsedOne > maxTime) maxTime = elapsedOne; + if (elapsedOne*timeCheckCount < maxElapsed/16) timeCheckCount *= 2; + } + lastTime = now; + ++count; + + if (now - beginTime < maxElapsed) return true; // Keep going + + --count; + + // Output results + double average = (now-beginTime)/count; + std::cout << name << "," << count << "," << minTime << "," << maxTime << "," << average << "\n"; + + return false; +} diff --git a/src/bench/bench.h b/src/bench/bench.h new file mode 100644 index 0000000000..bf591a2be6 --- /dev/null +++ b/src/bench/bench.h @@ -0,0 +1,71 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef BITCOIN_BENCH_H +#define BITCOIN_BENCH_H + +// Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark +// framework (see https://github.com/google/benchmark) +// Wny not use the Google Benchmark framework? Because adding Yet Another Dependency +// (that uses cmake as its build system and has lots of features we don't need) isn't +// worth it. + +/* + * Usage: + +static void CODE_TO_TIME(benchmark::State& state) +{ + ... do any setup needed... + while (state.KeepRunning()) { + ... do stuff you want to time... + } + ... do any cleanup needed... +} + +BENCHMARK(CODE_TO_TIME); + + */ + + +#include <boost/function.hpp> +#include <boost/preprocessor/cat.hpp> +#include <boost/preprocessor/stringize.hpp> +#include <map> +#include <string> + +namespace benchmark { + + class State { + std::string name; + double maxElapsed; + double beginTime; + double lastTime, minTime, maxTime; + int64_t count; + int64_t timeCheckCount; + public: + State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) { + minTime = std::numeric_limits<double>::max(); + maxTime = std::numeric_limits<double>::min(); + timeCheckCount = 1; + } + bool KeepRunning(); + }; + + typedef boost::function<void(State&)> BenchFunction; + + class BenchRunner + { + static std::map<std::string, BenchFunction> benchmarks; + + public: + BenchRunner(std::string name, BenchFunction func); + + static void RunAll(double elapsedTimeForOne=1.0); + }; +} + +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); +#define BENCHMARK(n) \ + benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n); + +#endif // BITCOIN_BENCH_H diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp new file mode 100644 index 0000000000..db1402216d --- /dev/null +++ b/src/bench/bench_bitcoin.cpp @@ -0,0 +1,21 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" + +#include "key.h" +#include "main.h" +#include "util.h" + +int +main(int argc, char** argv) +{ + ECC_Start(); + SetupEnvironment(); + fPrintToDebugLog = false; // don't want to write to debug.log file + + benchmark::BenchRunner::RunAll(); + + ECC_Stop(); +} diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index 26cacf95ae..ce96b5c8aa 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -5,6 +5,7 @@ #include "leveldbwrapper.h" #include "util.h" +#include "random.h" #include <boost/filesystem.hpp> @@ -12,6 +13,7 @@ #include <leveldb/env.h> #include <leveldb/filter_policy.h> #include <memenv.h> +#include <stdint.h> void HandleError(const leveldb::Status& status) throw(leveldb_error) { @@ -43,7 +45,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe) +CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) { penv = NULL; readoptions.verify_checksums = true; @@ -67,6 +69,25 @@ CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCa leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); HandleError(status); LogPrintf("Opened LevelDB successfully\n"); + + // The base-case obfuscation key, which is a noop. + obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000'); + + bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key); + + if (!key_exists && obfuscate && IsEmpty()) { + // Initialize non-degenerate obfuscation if it won't upset + // existing, non-obfuscated data. + std::vector<unsigned char> new_key = CreateObfuscateKey(); + + // Write `new_key` so we don't obfuscate the key with itself + Write(OBFUSCATE_KEY_KEY, new_key); + obfuscate_key = new_key; + + LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex()); + } + + LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex()); } CLevelDBWrapper::~CLevelDBWrapper() @@ -87,3 +108,40 @@ bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) throw(leveldb HandleError(status); return true; } + +// Prefixed with null character to avoid collisions with other keys +// +// We must use a string constructor which specifies length so that we copy +// past the null-terminator. +const std::string CLevelDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14); + +const unsigned int CLevelDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; + +/** + * Returns a string (consisting of 8 random bytes) suitable for use as an + * obfuscating XOR key. + */ +std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const +{ + unsigned char buff[OBFUSCATE_KEY_NUM_BYTES]; + GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES); + return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]); + +} + +bool CLevelDBWrapper::IsEmpty() +{ + boost::scoped_ptr<leveldb::Iterator> it(NewIterator()); + it->SeekToFirst(); + return !(it->Valid()); +} + +const std::vector<unsigned char>& CLevelDBWrapper::GetObfuscateKey() const +{ + return obfuscate_key; +} + +std::string CLevelDBWrapper::GetObfuscateKeyHex() const +{ + return HexStr(obfuscate_key); +} diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index c65e842704..f5c463830c 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -9,6 +9,7 @@ #include "serialize.h" #include "streams.h" #include "util.h" +#include "utilstrencodings.h" #include "version.h" #include <boost/filesystem/path.hpp> @@ -31,8 +32,14 @@ class CLevelDBBatch private: leveldb::WriteBatch batch; + const std::vector<unsigned char> obfuscate_key; public: + /** + * @param[in] obfuscate_key If passed, XOR data with this key. + */ + CLevelDBBatch(const std::vector<unsigned char>& obfuscate_key) : obfuscate_key(obfuscate_key) { }; + template <typename K, typename V> void Write(const K& key, const V& value) { @@ -44,6 +51,7 @@ public: CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(ssValue.GetSerializeSize(value)); ssValue << value; + ssValue.Xor(obfuscate_key); leveldb::Slice slValue(&ssValue[0], ssValue.size()); batch.Put(slKey, slValue); @@ -85,8 +93,27 @@ private: //! the database itself leveldb::DB* pdb; + //! a key used for optional XOR-obfuscation of the database + std::vector<unsigned char> obfuscate_key; + + //! the key under which the obfuscation key is stored + static const std::string OBFUSCATE_KEY_KEY; + + //! the length of the obfuscate key in number of bytes + static const unsigned int OBFUSCATE_KEY_NUM_BYTES; + + std::vector<unsigned char> CreateObfuscateKey() const; + public: - CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false); + /** + * @param[in] path Location in the filesystem where leveldb data will be stored. + * @param[in] nCacheSize Configures various leveldb cache settings. + * @param[in] fMemory If true, use leveldb's memory environment. + * @param[in] fWipe If true, remove all existing data. + * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR + * with a zero'd byte array. + */ + CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); ~CLevelDBWrapper(); template <typename K, typename V> @@ -107,6 +134,7 @@ public: } try { CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); + ssValue.Xor(obfuscate_key); ssValue >> value; } catch (const std::exception&) { return false; @@ -117,7 +145,7 @@ public: template <typename K, typename V> bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error) { - CLevelDBBatch batch; + CLevelDBBatch batch(obfuscate_key); batch.Write(key, value); return WriteBatch(batch, fSync); } @@ -144,7 +172,7 @@ public: template <typename K> bool Erase(const K& key, bool fSync = false) throw(leveldb_error) { - CLevelDBBatch batch; + CLevelDBBatch batch(obfuscate_key); batch.Erase(key); return WriteBatch(batch, fSync); } @@ -159,7 +187,7 @@ public: bool Sync() throw(leveldb_error) { - CLevelDBBatch batch; + CLevelDBBatch batch(obfuscate_key); return WriteBatch(batch, true); } @@ -168,6 +196,23 @@ public: { return pdb->NewIterator(iteroptions); } + + /** + * Return true if the database managed by this class contains no entries. + */ + bool IsEmpty(); + + /** + * Accessor for obfuscate_key. + */ + const std::vector<unsigned char>& GetObfuscateKey() const; + + /** + * Return the obfuscate_key as a hex-formatted string. + */ + std::string GetObfuscateKeyHex() const; + }; #endif // BITCOIN_LEVELDBWRAPPER_H + diff --git a/src/streams.h b/src/streams.h index fa1e18defe..8610e4d18e 100644 --- a/src/streams.h +++ b/src/streams.h @@ -296,6 +296,29 @@ public: data.insert(data.end(), begin(), end()); clear(); } + + /** + * XOR the contents of this stream with a certain key. + * + * @param[in] key The key used to XOR the data in this stream. + */ + void Xor(const std::vector<unsigned char>& key) + { + if (key.size() == 0) { + return; + } + + for (size_type i = 0, j = 0; i != size(); i++) { + vch[i] ^= key[j++]; + + // This potentially acts on very many bytes of data, so it's + // important that we calculate `j`, i.e. the `key` index in this + // way instead of doing a %, which would effectively be a division + // for each byte Xor'd -- much slower than need be. + if (j == key.size()) + j = 0; + } + } }; diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp new file mode 100644 index 0000000000..db04f3ea48 --- /dev/null +++ b/src/test/leveldbwrapper_tests.cpp @@ -0,0 +1,128 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "leveldbwrapper.h" +#include "uint256.h" +#include "random.h" +#include "test/test_bitcoin.h" + +#include <boost/assign/std/vector.hpp> // for 'operator+=()' +#include <boost/assert.hpp> +#include <boost/test/unit_test.hpp> + +using namespace std; +using namespace boost::assign; // bring 'operator+=()' into scope +using namespace boost::filesystem; + +// Test if a string consists entirely of null characters +bool is_null_key(const vector<unsigned char>& key) { + bool isnull = true; + + for (unsigned int i = 0; i < key.size(); i++) + isnull &= (key[i] == '\x00'); + + return isnull; +} + +BOOST_FIXTURE_TEST_SUITE(leveldbwrapper_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(leveldbwrapper) +{ + // Perform tests both obfuscated and non-obfuscated. + for (int i = 0; i < 2; i++) { + bool obfuscate = (bool)i; + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + // Ensure that we're doing real obfuscation when obfuscate=true + BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey())); + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + } +} + +// Test that we do not obfuscation if there is existing data. +BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) +{ + // We're going to share this path between two wrappers + path ph = temp_directory_path() / unique_path(); + create_directories(ph); + + // Set up a non-obfuscated wrapper to write some initial data. + CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw->Write(key, in)); + BOOST_CHECK(dbw->Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + + // Call the destructor to free leveldb LOCK + delete dbw; + + // Now, set up another wrapper that wants to obfuscate the same directory + CLevelDBWrapper odbw(ph, (1 << 10), false, false, true); + + // Check that the key/val we wrote with unobfuscated wrapper exists and + // is readable. + uint256 res2; + BOOST_CHECK(odbw.Read(key, res2)); + BOOST_CHECK_EQUAL(res2.ToString(), in.ToString()); + + BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data + BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string + + uint256 in2 = GetRandHash(); + uint256 res3; + + // Check that we can write successfully + BOOST_CHECK(odbw.Write(key, in2)); + BOOST_CHECK(odbw.Read(key, res3)); + BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); +} + +// Ensure that we start obfuscating during a reindex. +BOOST_AUTO_TEST_CASE(existing_data_reindex) +{ + // We're going to share this path between two wrappers + path ph = temp_directory_path() / unique_path(); + create_directories(ph); + + // Set up a non-obfuscated wrapper to write some initial data. + CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw->Write(key, in)); + BOOST_CHECK(dbw->Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + + // Call the destructor to free leveldb LOCK + delete dbw; + + // Simulate a -reindex by wiping the existing data store + CLevelDBWrapper odbw(ph, (1 << 10), false, true, true); + + // Check that the key/val we wrote with unobfuscated wrapper doesn't exist + uint256 res2; + BOOST_CHECK(!odbw.Read(key, res2)); + BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey())); + + uint256 in2 = GetRandHash(); + uint256 res3; + + // Check that we can write successfully + BOOST_CHECK(odbw.Write(key, in2)); + BOOST_CHECK(odbw.Read(key, res3)); + BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp new file mode 100644 index 0000000000..0ed8f363d7 --- /dev/null +++ b/src/test/streams_tests.cpp @@ -0,0 +1,67 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "streams.h" +#include "support/allocators/zeroafterfree.h" +#include "test/test_bitcoin.h" + +#include <boost/assign/std/vector.hpp> // for 'operator+=()' +#include <boost/assert.hpp> +#include <boost/test/unit_test.hpp> + +using namespace std; +using namespace boost::assign; // bring 'operator+=()' into scope + +BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(streams_serializedata_xor) +{ + std::vector<char> in; + std::vector<char> expected_xor; + std::vector<unsigned char> key; + CDataStream ds(in, 0, 0); + + // Degenerate case + + key += '\x00','\x00'; + ds.Xor(key); + BOOST_CHECK_EQUAL( + std::string(expected_xor.begin(), expected_xor.end()), + std::string(ds.begin(), ds.end())); + + in += '\x0f','\xf0'; + expected_xor += '\xf0','\x0f'; + + // Single character key + + ds.clear(); + ds.insert(ds.begin(), in.begin(), in.end()); + key.clear(); + + key += '\xff'; + ds.Xor(key); + BOOST_CHECK_EQUAL( + std::string(expected_xor.begin(), expected_xor.end()), + std::string(ds.begin(), ds.end())); + + // Multi character key + + in.clear(); + expected_xor.clear(); + in += '\xf0','\x0f'; + expected_xor += '\x0f','\x00'; + + ds.clear(); + ds.insert(ds.begin(), in.begin(), in.end()); + + key.clear(); + key += '\xff','\x0f'; + + ds.Xor(key); + BOOST_CHECK_EQUAL( + std::string(expected_xor.begin(), expected_xor.end()), + std::string(ds.begin(), ds.end())); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.cpp b/src/txdb.cpp index 21ecd65238..9738dea03d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -29,18 +29,8 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -void static BatchWriteCoins(CLevelDBBatch &batch, const uint256 &hash, const CCoins &coins) { - if (coins.IsPruned()) - batch.Erase(make_pair(DB_COINS, hash)); - else - batch.Write(make_pair(DB_COINS, hash), coins); -} - -void static BatchWriteHashBestChain(CLevelDBBatch &batch, const uint256 &hash) { - batch.Write(DB_BEST_BLOCK, hash); -} - -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) { +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) +{ } bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { @@ -59,12 +49,15 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CLevelDBBatch batch; + CLevelDBBatch batch(db.GetObfuscateKey()); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - BatchWriteCoins(batch, it->first, it->second.coins); + if (it->second.coins.IsPruned()) + batch.Erase(make_pair(DB_COINS, it->first)); + else + batch.Write(make_pair(DB_COINS, it->first), it->second.coins); changed++; } count++; @@ -72,7 +65,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { mapCoins.erase(itOld); } if (!hashBlock.IsNull()) - BatchWriteHashBestChain(batch, hashBlock); + batch.Write(DB_BEST_BLOCK, hashBlock); LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return db.WriteBatch(batch); @@ -158,7 +151,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) { - CLevelDBBatch batch; + CLevelDBBatch batch(GetObfuscateKey()); for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -174,7 +167,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) { - CLevelDBBatch batch; + CLevelDBBatch batch(GetObfuscateKey()); for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); diff --git a/src/util.cpp b/src/util.cpp index f50d25e17a..8192a7c71c 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -280,10 +280,13 @@ int LogPrintStr(const std::string &str) { int ret = 0; // Returns total number of characters written static bool fStartedNewLine = true; + + string strTimestamped = LogTimestampStr(str, &fStartedNewLine); + if (fPrintToConsole) { // print to console - ret = fwrite(str.data(), 1, str.size(), stdout); + ret = fwrite(strTimestamped.data(), 1, strTimestamped.size(), stdout); fflush(stdout); } else if (fPrintToDebugLog) @@ -291,8 +294,6 @@ int LogPrintStr(const std::string &str) boost::call_once(&DebugPrintInit, debugPrintInitFlag); boost::mutex::scoped_lock scoped_lock(*mutexDebugLog); - string strTimestamped = LogTimestampStr(str, &fStartedNewLine); - // buffer if we haven't opened the log yet if (fileout == NULL) { assert(vMsgsBeforeOpenLog); |