diff options
87 files changed, 1394 insertions, 994 deletions
diff --git a/.gitignore b/.gitignore index 9f1b2a4695..bafc5919c1 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,5 @@ qa/pull-tester/run-bitcoind-for-test.sh qa/pull-tester/build-tests.sh !src/leveldb*/Makefile + +/doc/doxygen/ diff --git a/configure.ac b/configure.ac index 90aa112421..b603d1766d 100644 --- a/configure.ac +++ b/configure.ac @@ -11,8 +11,21 @@ AC_CONFIG_SRCDIR([src/main.cpp]) AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) -LT_INIT([disable-shared]) + AC_CANONICAL_HOST + +dnl By default, libtool for mingw refuses to link static libs into a dll for +dnl fear of mixing pic/non-pic objects, and import/export complications. Since +dnl we have those under control, re-enable that functionality. + +case $host in + *mingw*) + lt_cv_deplibs_check_method="pass_all" + ;; +esac + +LT_INIT([disable-shared]) + AH_TOP([#ifndef BITCOIN_CONFIG_H]) AH_TOP([#define BITCOIN_CONFIG_H]) AH_BOTTOM([#endif //BITCOIN_CONFIG_H]) @@ -462,7 +475,7 @@ dnl Check for libminiupnpc (optional) if test x$use_upnp != xno; then AC_CHECK_HEADERS( [miniupnpc/miniwget.h miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], - [AC_CHECK_LIB([miniupnpc], [main],, [have_miniupnpc=no])], + [AC_CHECK_LIB([miniupnpc], [main],[MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], [have_miniupnpc=no] ) fi @@ -632,9 +645,9 @@ else AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),) AC_CHECK_LIB([ssl], [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing)) - BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],,BITCOIN_QT_FAIL(libprotobuf not found))) + BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], BITCOIN_QT_FAIL(libprotobuf not found))) if test x$use_qr != xno; then - BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],, [have_qrencode=no])]) + BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) BITCOIN_QT_CHECK([AC_CHECK_HEADER([qrencode.h],, have_qrencode=no)]) fi fi @@ -808,6 +821,7 @@ AC_SUBST(BUILD_TEST) AC_SUBST(BUILD_QT) AC_SUBST(BUILD_TEST_QT) AC_SUBST(MINIUPNPC_CPPFLAGS) +AC_SUBST(MINIUPNPC_LIBS) AC_CONFIG_FILES([Makefile src/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py]) AC_CONFIG_FILES([qa/pull-tester/run-bitcoind-for-test.sh],[chmod +x qa/pull-tester/run-bitcoind-for-test.sh]) AC_CONFIG_FILES([qa/pull-tester/build-tests.sh],[chmod +x qa/pull-tester/build-tests.sh]) diff --git a/qa/pull-tester/build-tests.sh.in b/qa/pull-tester/build-tests.sh.in index 1ef47d77f4..f5c5f0bf70 100755 --- a/qa/pull-tester/build-tests.sh.in +++ b/qa/pull-tester/build-tests.sh.in @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2013 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Copyright (c) 2013-2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Param1: The prefix to mingw staging diff --git a/qa/pull-tester/pull-tester.py b/qa/pull-tester/pull-tester.py index 61ce813d80..66688dd4b5 100755 --- a/qa/pull-tester/pull-tester.py +++ b/qa/pull-tester/pull-tester.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright (c) 2013 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Copyright (c) 2013-2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # import json diff --git a/qa/pull-tester/pull-tester.sh b/qa/pull-tester/pull-tester.sh index d3356f736a..3fe4a05c76 100755 --- a/qa/pull-tester/pull-tester.sh +++ b/qa/pull-tester/pull-tester.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2013 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Copyright (c) 2013-2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Helper script for pull-tester. diff --git a/qa/pull-tester/run-bitcoind-for-test.sh.in b/qa/pull-tester/run-bitcoind-for-test.sh.in index 15363d09a6..42d8ad52e8 100755 --- a/qa/pull-tester/run-bitcoind-for-test.sh.in +++ b/qa/pull-tester/run-bitcoind-for-test.sh.in @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2013 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Copyright (c) 2013-2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # DATADIR="@abs_top_builddir@/.bitcoin" diff --git a/qa/rpc-tests/conflictedbalance.sh b/qa/rpc-tests/conflictedbalance.sh index 849ad31fb8..71761321ed 100755 --- a/qa/rpc-tests/conflictedbalance.sh +++ b/qa/rpc-tests/conflictedbalance.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Test marking of spent outputs diff --git a/qa/rpc-tests/forknotify.py b/qa/rpc-tests/forknotify.py index 23bfb74175..a4c2dc944e 100755 --- a/qa/rpc-tests/forknotify.py +++ b/qa/rpc-tests/forknotify.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/qa/rpc-tests/getblocktemplate.py b/qa/rpc-tests/getblocktemplate.py index 5ae5d09601..500662bf87 100755 --- a/qa/rpc-tests/getblocktemplate.py +++ b/qa/rpc-tests/getblocktemplate.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Exercise the listtransactions API diff --git a/qa/rpc-tests/getchaintips.py b/qa/rpc-tests/getchaintips.py index 842fcad2b2..dda3450eb6 100755 --- a/qa/rpc-tests/getchaintips.py +++ b/qa/rpc-tests/getchaintips.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Exercise the getchaintips API. We introduce a network split, work diff --git a/qa/rpc-tests/keypool.py b/qa/rpc-tests/keypool.py index 86ad20de52..354ee0822c 100755 --- a/qa/rpc-tests/keypool.py +++ b/qa/rpc-tests/keypool.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Exercise the wallet keypool, and interaction with wallet encryption/locking diff --git a/qa/rpc-tests/listtransactions.py b/qa/rpc-tests/listtransactions.py index 6102052a6f..530b00db8f 100755 --- a/qa/rpc-tests/listtransactions.py +++ b/qa/rpc-tests/listtransactions.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Exercise the listtransactions API diff --git a/qa/rpc-tests/netutil.py b/qa/rpc-tests/netutil.py index 9bea2e355e..b740ee0f98 100644 --- a/qa/rpc-tests/netutil.py +++ b/qa/rpc-tests/netutil.py @@ -1,3 +1,8 @@ +#!/usr/bin/env python +# Copyright (c) 2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + # Linux network utilities import sys import socket diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index 7170255242..9fc661fe80 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Exercise the listreceivedbyaddress API diff --git a/qa/rpc-tests/rpcbind_test.py b/qa/rpc-tests/rpcbind_test.py index a823404e00..5e476e2505 100755 --- a/qa/rpc-tests/rpcbind_test.py +++ b/qa/rpc-tests/rpcbind_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Test for -rpcbind, as well as -rpcallowip and -rpcconnect diff --git a/qa/rpc-tests/send.sh b/qa/rpc-tests/send.sh index 37367865c1..2d54cc6ded 100755 --- a/qa/rpc-tests/send.sh +++ b/qa/rpc-tests/send.sh @@ -1,6 +1,6 @@ #!/bin/bash # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. TIMEOUT=10 SIGNAL=HUP diff --git a/qa/rpc-tests/smartfees.py b/qa/rpc-tests/smartfees.py index 065bdb01c3..57a72dd96b 100755 --- a/qa/rpc-tests/smartfees.py +++ b/qa/rpc-tests/smartfees.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +# Copyright (c) 2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Test fee estimation code diff --git a/qa/rpc-tests/test_framework.py b/qa/rpc-tests/test_framework.py index 8c8453054d..c3396a5a83 100755 --- a/qa/rpc-tests/test_framework.py +++ b/qa/rpc-tests/test_framework.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Base class for RPC testing diff --git a/qa/rpc-tests/txnmall.sh b/qa/rpc-tests/txnmall.sh index bd36967a63..035e7ee15f 100755 --- a/qa/rpc-tests/txnmall.sh +++ b/qa/rpc-tests/txnmall.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Test proper accounting with malleable transactions diff --git a/qa/rpc-tests/util.py b/qa/rpc-tests/util.py index 276b5c635f..6d0b21c927 100644 --- a/qa/rpc-tests/util.py +++ b/qa/rpc-tests/util.py @@ -1,5 +1,5 @@ # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Helpful routines for regression testing diff --git a/qa/rpc-tests/util.sh b/qa/rpc-tests/util.sh index b726ef627f..c2b7004308 100644 --- a/qa/rpc-tests/util.sh +++ b/qa/rpc-tests/util.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Functions used by more than one test diff --git a/qa/rpc-tests/wallet.sh b/qa/rpc-tests/wallet.sh index 98532fa858..bb5f34f590 100755 --- a/qa/rpc-tests/wallet.sh +++ b/qa/rpc-tests/wallet.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Copyright (c) 2013-2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Test block generation and basic wallet sending diff --git a/qa/rpc-tests/walletbackup.sh b/qa/rpc-tests/walletbackup.sh index b88d5920f7..ee11418bef 100755 --- a/qa/rpc-tests/walletbackup.sh +++ b/qa/rpc-tests/walletbackup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Copyright (c) 2014 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Test wallet backup / dump / restore functionality diff --git a/qa/rpc-tests/zapwallettxes.sh b/qa/rpc-tests/zapwallettxes.sh index bc52a7dacd..e6d490cccc 100755 --- a/qa/rpc-tests/zapwallettxes.sh +++ b/qa/rpc-tests/zapwallettxes.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Copyright (c) 2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. # Test -zapwallettxes=<mode> diff --git a/src/Makefile.am b/src/Makefile.am index 155adfef7d..dbefa71fc6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -36,13 +36,15 @@ LIBBITCOIN_CRYPTO=crypto/libbitcoin_crypto.a LIBBITCOIN_UNIVALUE=univalue/libbitcoin_univalue.a LIBBITCOINQT=qt/libbitcoinqt.a +# Make is not made aware of per-object dependencies to avoid limiting building parallelization +# But to build the less dependent modules first, we manually select their order here: noinst_LIBRARIES = \ - libbitcoin_server.a \ - libbitcoin_common.a \ - libbitcoin_cli.a \ + crypto/libbitcoin_crypto.a \ libbitcoin_util.a \ + libbitcoin_common.a \ univalue/libbitcoin_univalue.a \ - crypto/libbitcoin_crypto.a + libbitcoin_server.a \ + libbitcoin_cli.a if ENABLE_WALLET BITCOIN_INCLUDES += $(BDB_CPPFLAGS) noinst_LIBRARIES += libbitcoin_wallet.a @@ -107,6 +109,7 @@ BITCOIN_CORE_H = \ script/sign.h \ script/standard.h \ serialize.h \ + streams.h \ sync.h \ threadsafety.h \ timedata.h \ @@ -281,7 +284,7 @@ if TARGET_WINDOWS bitcoind_SOURCES += bitcoind-res.rc endif -bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) +bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) bitcoind_CPPFLAGS = $(BITCOIN_INCLUDES) bitcoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) @@ -293,7 +296,8 @@ bitcoin_cli_LDADD = \ $(LIBBITCOIN_CRYPTO) \ $(BOOST_LIBS) \ $(SSL_LIBS) \ - $(CRYPTO_LIBS) + $(CRYPTO_LIBS) \ + $(MINIUPNPC_LIBS) bitcoin_cli_SOURCES = \ bitcoin-cli.cpp @@ -317,7 +321,9 @@ endif bitcoin_tx_LDADD += $(BOOST_LIBS) \ $(SSL_LIBS) \ - $(CRYPTO_LIBS) + $(CRYPTO_LIBS) \ + $(MINIUPNPC_LIBS) + bitcoin_tx_SOURCES = bitcoin-tx.cpp bitcoin_tx_CPPFLAGS = $(BITCOIN_INCLUDES) # diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 8fb4af81ac..ac6d60df03 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -360,7 +360,7 @@ if ENABLE_WALLET qt_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET) endif qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_UNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ - $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) + $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) if USE_LIBSECP256K1 qt_bitcoin_qt_LDADD += secp256k1/libsecp256k1.la endif diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 23375bef82..064b531b93 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -32,7 +32,7 @@ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_UNIVALUE) $(LIBLEVELDB) \ $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ - $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) + $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) if USE_LIBSECP256K1 qt_test_test_bitcoin_qt_LDADD += secp256k1/libsecp256k1.la endif diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b20e226c3d..340eb9f1a7 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -89,7 +89,7 @@ if USE_LIBSECP256K1 test_test_bitcoin_LDADD += secp256k1/libsecp256k1.la endif -test_test_bitcoin_LDADD += $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) +test_test_bitcoin_LDADD += $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/addrman.cpp b/src/addrman.cpp index 7b674a66e7..1982db52ae 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,11 +1,12 @@ // Copyright (c) 2012 Pieter Wuille -// Distributed under the MIT/X11 software license, see the accompanying +// Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "addrman.h" #include "hash.h" #include "serialize.h" +#include "streams.h" using namespace std; @@ -39,7 +40,7 @@ int CAddrInfo::GetNewBucket(const std::vector<unsigned char>& nKey, const CNetAd bool CAddrInfo::IsTerrible(int64_t nNow) const { - if (nLastTry && nLastTry >= nNow - 60) // never remove things tried the last minute + if (nLastTry && nLastTry >= nNow - 60) // never remove things tried in the last minute return false; if (nTime > nNow + 10 * 60) // came in a flying DeLorean @@ -131,7 +132,7 @@ int CAddrMan::SelectTried(int nKBucket) { std::vector<int>& vTried = vvTried[nKBucket]; - // random shuffle the first few elements (using the entire list) + // randomly shuffle the first few elements (using the entire list) // find the least recently tried among them int64_t nOldest = -1; int nOldestPos = -1; @@ -211,7 +212,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId, int nOrigin) assert(info.nRefCount == 0); - // what tried bucket to move the entry to + // which tried bucket to move the entry to int nKBucket = info.GetTriedBucket(nKey); std::vector<int>& vTried = vvTried[nKBucket]; diff --git a/src/addrman.h b/src/addrman.h index 5fd698f18a..914086fc76 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Distributed under the MIT/X11 software license, see the accompanying +// Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef _BITCOIN_ADDRMAN @@ -17,29 +17,31 @@ #include <stdint.h> #include <vector> -/** Extended statistics about a CAddress */ +/** + * Extended statistics about a CAddress + */ class CAddrInfo : public CAddress { private: - // where knowledge about this address first came from + //! where knowledge about this address first came from CNetAddr source; - // last successful connection by us + //! last successful connection by us int64_t nLastSuccess; - // last try whatsoever by us: + //! last try whatsoever by us: // int64_t CAddress::nLastTry - // connection attempts since last successful attempt + //! connection attempts since last successful attempt int nAttempts; - // reference count in new sets (memory only) + //! reference count in new sets (memory only) int nRefCount; - // in tried set? (memory only) + //! in tried set? (memory only) bool fInTried; - // position in vRandom + //! position in vRandom int nRandomPos; friend class CAddrMan; @@ -76,200 +78,205 @@ public: Init(); } - // Calculate in which "tried" bucket this entry belongs + //! Calculate in which "tried" bucket this entry belongs int GetTriedBucket(const std::vector<unsigned char> &nKey) const; - // Calculate in which "new" bucket this entry belongs, given a certain source + //! Calculate in which "new" bucket this entry belongs, given a certain source int GetNewBucket(const std::vector<unsigned char> &nKey, const CNetAddr& src) const; - // Calculate in which "new" bucket this entry belongs, using its default source + //! Calculate in which "new" bucket this entry belongs, using its default source int GetNewBucket(const std::vector<unsigned char> &nKey) const { return GetNewBucket(nKey, source); } - // Determine whether the statistics about this entry are bad enough so that it can just be deleted + //! Determine whether the statistics about this entry are bad enough so that it can just be deleted bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; - // Calculate the relative chance this entry should be given when selecting nodes to connect to + //! Calculate the relative chance this entry should be given when selecting nodes to connect to double GetChance(int64_t nNow = GetAdjustedTime()) const; }; -// Stochastic address manager -// -// Design goals: -// * Only keep a limited number of addresses around, so that addr.dat and memory requirements do not grow without bound. -// * Keep the address tables in-memory, and asynchronously dump the entire to able in addr.dat. -// * Make sure no (localized) attacker can fill the entire table with his nodes/addresses. -// -// To that end: -// * Addresses are organized into buckets. -// * Address that have not yet been tried go into 256 "new" buckets. -// * Based on the address range (/16 for IPv4) of source of the information, 32 buckets are selected at random -// * The actual bucket is chosen from one of these, based on the range the address itself is located. -// * One single address can occur in up to 4 different buckets, to increase selection chances for addresses that -// are seen frequently. The chance for increasing this multiplicity decreases exponentially. -// * When adding a new address to a full bucket, a randomly chosen entry (with a bias favoring less recently seen -// ones) is removed from it first. -// * Addresses of nodes that are known to be accessible go into 64 "tried" buckets. -// * Each address range selects at random 4 of these buckets. -// * The actual bucket is chosen from one of these, based on the full address. -// * When adding a new good address to a full bucket, a randomly chosen entry (with a bias favoring less recently -// tried ones) is evicted from it, back to the "new" buckets. -// * Bucket selection is based on cryptographic hashing, using a randomly-generated 256-bit key, which should not -// be observable by adversaries. -// * Several indexes are kept for high performance. Defining DEBUG_ADDRMAN will introduce frequent (and expensive) -// consistency checks for the entire data structure. - -// total number of buckets for tried addresses +/** Stochastic address manager + * + * Design goals: + * * Keep the address tables in-memory, and asynchronously dump the entire to able in peers.dat. + * * Make sure no (localized) attacker can fill the entire table with his nodes/addresses. + * + * To that end: + * * Addresses are organized into buckets. + * * Address that have not yet been tried go into 256 "new" buckets. + * * Based on the address range (/16 for IPv4) of source of the information, 32 buckets are selected at random + * * The actual bucket is chosen from one of these, based on the range the address itself is located. + * * One single address can occur in up to 4 different buckets, to increase selection chances for addresses that + * are seen frequently. The chance for increasing this multiplicity decreases exponentially. + * * When adding a new address to a full bucket, a randomly chosen entry (with a bias favoring less recently seen + * ones) is removed from it first. + * * Addresses of nodes that are known to be accessible go into 64 "tried" buckets. + * * Each address range selects at random 4 of these buckets. + * * The actual bucket is chosen from one of these, based on the full address. + * * When adding a new good address to a full bucket, a randomly chosen entry (with a bias favoring less recently + * tried ones) is evicted from it, back to the "new" buckets. + * * Bucket selection is based on cryptographic hashing, using a randomly-generated 256-bit key, which should not + * be observable by adversaries. + * * Several indexes are kept for high performance. Defining DEBUG_ADDRMAN will introduce frequent (and expensive) + * consistency checks for the entire data structure. + */ + +//! total number of buckets for tried addresses #define ADDRMAN_TRIED_BUCKET_COUNT 64 -// maximum allowed number of entries in buckets for tried addresses +//! maximum allowed number of entries in buckets for tried addresses #define ADDRMAN_TRIED_BUCKET_SIZE 64 -// total number of buckets for new addresses +//! total number of buckets for new addresses #define ADDRMAN_NEW_BUCKET_COUNT 256 -// maximum allowed number of entries in buckets for new addresses +//! maximum allowed number of entries in buckets for new addresses #define ADDRMAN_NEW_BUCKET_SIZE 64 -// over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread +//! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread #define ADDRMAN_TRIED_BUCKETS_PER_GROUP 4 -// over how many buckets entries with new addresses originating from a single group are spread +//! over how many buckets entries with new addresses originating from a single group are spread #define ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP 32 -// in how many buckets for entries with new addresses a single address may occur +//! in how many buckets for entries with new addresses a single address may occur #define ADDRMAN_NEW_BUCKETS_PER_ADDRESS 4 -// how many entries in a bucket with tried addresses are inspected, when selecting one to replace +//! how many entries in a bucket with tried addresses are inspected, when selecting one to replace #define ADDRMAN_TRIED_ENTRIES_INSPECT_ON_EVICT 4 -// how old addresses can maximally be +//! how old addresses can maximally be #define ADDRMAN_HORIZON_DAYS 30 -// after how many failed attempts we give up on a new node +//! after how many failed attempts we give up on a new node #define ADDRMAN_RETRIES 3 -// how many successive failures are allowed ... +//! how many successive failures are allowed ... #define ADDRMAN_MAX_FAILURES 10 -// ... in at least this many days +//! ... in at least this many days #define ADDRMAN_MIN_FAIL_DAYS 7 -// the maximum percentage of nodes to return in a getaddr call +//! the maximum percentage of nodes to return in a getaddr call #define ADDRMAN_GETADDR_MAX_PCT 23 -// the maximum number of nodes to return in a getaddr call +//! the maximum number of nodes to return in a getaddr call #define ADDRMAN_GETADDR_MAX 2500 -/** Stochastical (IP) address manager */ +/** + * Stochastical (IP) address manager + */ class CAddrMan { private: - // critical section to protect the inner data structures + //! critical section to protect the inner data structures mutable CCriticalSection cs; - // secret key to randomize bucket select with + //! secret key to randomize bucket select with std::vector<unsigned char> nKey; - // last used nId + //! last used nId int nIdCount; - // table with information about all nIds + //! table with information about all nIds std::map<int, CAddrInfo> mapInfo; - // find an nId based on its network address + //! find an nId based on its network address std::map<CNetAddr, int> mapAddr; - // randomly-ordered vector of all nIds + //! randomly-ordered vector of all nIds std::vector<int> vRandom; // number of "tried" entries int nTried; - // list of "tried" buckets + //! list of "tried" buckets std::vector<std::vector<int> > vvTried; - // number of (unique) "new" entries + //! number of (unique) "new" entries int nNew; - // list of "new" buckets + //! list of "new" buckets std::vector<std::set<int> > vvNew; protected: - // Find an entry. + //! Find an entry. CAddrInfo* Find(const CNetAddr& addr, int *pnId = NULL); - // find an entry, creating it if necessary. - // nTime and nServices of found node is updated, if necessary. + //! find an entry, creating it if necessary. + //! nTime and nServices of the found node are updated, if necessary. CAddrInfo* Create(const CAddress &addr, const CNetAddr &addrSource, int *pnId = NULL); - // Swap two elements in vRandom. + //! Swap two elements in vRandom. void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2); - // Return position in given bucket to replace. + //! Return position in given bucket to replace. int SelectTried(int nKBucket); - // Remove an element from a "new" bucket. - // This is the only place where actual deletes occur. - // They are never deleted while in the "tried" table, only possibly evicted back to the "new" table. + //! Remove an element from a "new" bucket. + //! This is the only place where actual deletions occur. + //! Elements are never deleted while in the "tried" table, only possibly evicted back to the "new" table. int ShrinkNew(int nUBucket); - // Move an entry from the "new" table(s) to the "tried" table - // @pre vvUnkown[nOrigin].count(nId) != 0 + //! Move an entry from the "new" table(s) to the "tried" table + //! @pre vvUnkown[nOrigin].count(nId) != 0 void MakeTried(CAddrInfo& info, int nId, int nOrigin); - // Mark an entry "good", possibly moving it from "new" to "tried". + //! Mark an entry "good", possibly moving it from "new" to "tried". void Good_(const CService &addr, int64_t nTime); - // Add an entry to the "new" table. + //! Add an entry to the "new" table. bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty); - // Mark an entry as attempted to connect. + //! Mark an entry as attempted to connect. void Attempt_(const CService &addr, int64_t nTime); - // Select an address to connect to. - // nUnkBias determines how much to favor new addresses over tried ones (min=0, max=100) + //! Select an address to connect to. + //! nUnkBias determines how much to favor new addresses over tried ones (min=0, max=100) CAddress Select_(int nUnkBias); #ifdef DEBUG_ADDRMAN - // Perform consistency check. Returns an error code or zero. + //! Perform consistency check. Returns an error code or zero. int Check_(); #endif - // Select several addresses at once. + //! Select several addresses at once. void GetAddr_(std::vector<CAddress> &vAddr); - // Mark an entry as currently-connected-to. + //! Mark an entry as currently-connected-to. void Connected_(const CService &addr, int64_t nTime); public: - // serialized format: - // * version byte (currently 0) - // * nKey - // * nNew - // * nTried - // * number of "new" buckets - // * all nNew addrinfos in vvNew - // * all nTried addrinfos in vvTried - // * for each bucket: - // * number of elements - // * for each element: index - // - // Notice that vvTried, mapAddr and vVector are never encoded explicitly; - // they are instead reconstructed from the other information. - // - // vvNew is serialized, but only used if ADDRMAN_UNKOWN_BUCKET_COUNT didn't change, - // otherwise it is reconstructed as well. - // - // This format is more complex, but significantly smaller (at most 1.5 MiB), and supports - // changes to the ADDRMAN_ parameters without breaking the on-disk structure. - // - // We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has - // very little in common. + /** + * serialized format: + * * version byte (currently 0) + * * nKey + * * nNew + * * nTried + * * number of "new" buckets + * * all nNew addrinfos in vvNew + * * all nTried addrinfos in vvTried + * * for each bucket: + * * number of elements + * * for each element: index + * + * Notice that vvTried, mapAddr and vVector are never encoded explicitly; + * they are instead reconstructed from the other information. + * + * vvNew is serialized, but only used if ADDRMAN_UNKOWN_BUCKET_COUNT didn't change, + * otherwise it is reconstructed as well. + * + * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports + * changes to the ADDRMAN_ parameters without breaking the on-disk structure. + * + * We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has + * very little in common. + * + */ template<typename Stream> void Serialize(Stream &s, int nType, int nVersionDummy) const { @@ -394,13 +401,13 @@ public: nNew = 0; } - // Return the number of (unique) addresses in all tables. + //! Return the number of (unique) addresses in all tables. int size() { return vRandom.size(); } - // Consistency check + //! Consistency check void Check() { #ifdef DEBUG_ADDRMAN @@ -413,7 +420,7 @@ public: #endif } - // Add a single address. + //! Add a single address. bool Add(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty = 0) { bool fRet = false; @@ -428,7 +435,7 @@ public: return fRet; } - // Add multiple addresses. + //! Add multiple addresses. bool Add(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty = 0) { int nAdd = 0; @@ -444,7 +451,7 @@ public: return nAdd > 0; } - // Mark an entry as accessible. + //! Mark an entry as accessible. void Good(const CService &addr, int64_t nTime = GetAdjustedTime()) { { @@ -455,7 +462,7 @@ public: } } - // Mark an entry as connection attempted to. + //! Mark an entry as connection attempted to. void Attempt(const CService &addr, int64_t nTime = GetAdjustedTime()) { { @@ -466,8 +473,10 @@ public: } } - // Choose an address to connect to. - // nUnkBias determines how much "new" entries are favored over "tried" ones (0-100). + /** + * Choose an address to connect to. + * nUnkBias determines how much "new" entries are favored over "tried" ones (0-100). + */ CAddress Select(int nUnkBias = 50) { CAddress addrRet; @@ -480,7 +489,7 @@ public: return addrRet; } - // Return a bunch of addresses, selected at random. + //! Return a bunch of addresses, selected at random. std::vector<CAddress> GetAddr() { Check(); @@ -493,7 +502,7 @@ public: return vAddr; } - // Mark an entry as currently-connected-to. + //! Mark an entry as currently-connected-to. void Connected(const CService &addr, int64_t nTime = GetAdjustedTime()) { { diff --git a/src/allocators.h b/src/allocators.h index 6b69e7ae69..78a3b76d0c 100644 --- a/src/allocators.h +++ b/src/allocators.h @@ -9,6 +9,7 @@ #include <map> #include <string> #include <string.h> +#include <vector> #include <boost/thread/mutex.hpp> #include <boost/thread/once.hpp> @@ -261,4 +262,7 @@ struct zero_after_free_allocator : public std::allocator<T> { // This is exactly like std::string, but with a custom allocator. typedef std::basic_string<char, std::char_traits<char>, secure_allocator<char> > SecureString; +// Byte-vector that clears its contents before deletion. +typedef std::vector<char, zero_after_free_allocator<char> > CSerializeData; + #endif // BITCOIN_ALLOCATORS_H diff --git a/src/bloom.cpp b/src/bloom.cpp index cef74a3a54..cac71fdbbf 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -7,6 +7,7 @@ #include "core.h" #include "script/script.h" #include "script/standard.h" +#include "streams.h" #include <math.h> #include <stdlib.h> diff --git a/src/core_read.cpp b/src/core_read.cpp index 8b85a03c54..dcbcf4b4f7 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -7,6 +7,7 @@ #include "core.h" #include "script/script.h" #include "serialize.h" +#include "streams.h" #include "univalue/univalue.h" #include "util.h" #include "utilstrencodings.h" diff --git a/src/core_write.cpp b/src/core_write.cpp index e42e0b62a9..b2b29fb367 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -9,6 +9,7 @@ #include "script/script.h" #include "script/standard.h" #include "serialize.h" +#include "streams.h" #include "univalue/univalue.h" #include "util.h" #include "utilmoneystr.h" @@ -7,6 +7,7 @@ #define BITCOIN_DB_H #include "serialize.h" +#include "streams.h" #include "sync.h" #include "version.h" diff --git a/src/leveldb/CONTRIBUTING.md b/src/leveldb/CONTRIBUTING.md new file mode 100644 index 0000000000..cd600ff46b --- /dev/null +++ b/src/leveldb/CONTRIBUTING.md @@ -0,0 +1,36 @@ +# Contributing + +We'd love to accept your code patches! However, before we can take them, we +have to jump a couple of legal hurdles. + +## Contributor License Agreements + +Please fill out either the individual or corporate Contributor License +Agreement as appropriate. + +* If you are an individual writing original source code and you're sure you +own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual). +* If you work for a company that wants to allow you to contribute your work, +then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). + +Follow either of the two links above to access the appropriate CLA and +instructions for how to sign and return it. + +## Submitting a Patch + +1. Sign the contributors license agreement above. +2. Decide which code you want to submit. A submission should be a set of changes +that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues). +Please don't mix more than one logical change per submission, because it makes +the history hard to follow. If you want to make a change +(e.g. add a sample or feature) that doesn't have a corresponding issue in the +issue tracker, please create one. +3. **Submitting**: When you are ready to submit, send us a Pull Request. Be +sure to include the issue number you fixed and the name you used to sign +the CLA. + +## Writing Code ## + +If your contribution contains code, please make sure that it follows +[the style guide](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml). +Otherwise we will have to ask you to make changes, and that's no fun for anyone. diff --git a/src/leveldb/Makefile b/src/leveldb/Makefile index f8903b69e4..2bd2cadcdd 100644 --- a/src/leveldb/Makefile +++ b/src/leveldb/Makefile @@ -6,9 +6,12 @@ # Uncomment exactly one of the lines labelled (A), (B), and (C) below # to switch between compilation modes. -OPT ?= -O2 -DNDEBUG # (A) Production use (optimized mode) -# OPT ?= -g2 # (B) Debug mode, w/ full line-level debugging symbols -# OPT ?= -O2 -g2 -DNDEBUG # (C) Profiling mode: opt, but w/debugging symbols +# (A) Production use (optimized mode) +OPT ?= -O2 -DNDEBUG +# (B) Debug mode, w/ full line-level debugging symbols +# OPT ?= -g2 +# (C) Profiling mode: opt, but w/debugging symbols +# OPT ?= -O2 -g2 -DNDEBUG #----------------------------------------------- # detect what platform we're building on @@ -29,6 +32,11 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o) TESTUTIL = ./util/testutil.o TESTHARNESS = ./util/testharness.o $(TESTUTIL) +# Note: iOS should probably be using libtool, not ar. +ifeq ($(PLATFORM), IOS) +AR=xcrun ar +endif + TESTS = \ arena_test \ autocompact_test \ @@ -43,6 +51,7 @@ TESTS = \ env_test \ filename_test \ filter_block_test \ + hash_test \ issue178_test \ issue200_test \ log_test \ @@ -72,7 +81,7 @@ SHARED = $(SHARED1) else # Update db.h if you change these. SHARED_MAJOR = 1 -SHARED_MINOR = 17 +SHARED_MINOR = 18 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) @@ -152,6 +161,9 @@ filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) @@ -194,17 +206,17 @@ IOSARCH=-arch armv6 -arch armv7 -arch armv7s -arch arm64 .cc.o: mkdir -p ios-x86/$(dir $@) - $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ + xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ mkdir -p ios-arm/$(dir $@) xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ - lipo ios-x86/$@ ios-arm/$@ -create -output $@ + xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@ .c.o: mkdir -p ios-x86/$(dir $@) - $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ + xcrun -sdk iphonesimulator $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ mkdir -p ios-arm/$(dir $@) xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ - lipo ios-x86/$@ ios-arm/$@ -create -output $@ + xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@ else .cc.o: diff --git a/src/leveldb/README.md b/src/leveldb/README.md new file mode 100644 index 0000000000..480affb5ca --- /dev/null +++ b/src/leveldb/README.md @@ -0,0 +1,138 @@ +**LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.** + +Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) + +# Features + * Keys and values are arbitrary byte arrays. + * Data is stored sorted by key. + * Callers can provide a custom comparison function to override the sort order. + * The basic operations are `Put(key,value)`, `Get(key)`, `Delete(key)`. + * Multiple changes can be made in one atomic batch. + * Users can create a transient snapshot to get a consistent view of data. + * Forward and backward iteration is supported over the data. + * Data is automatically compressed using the [Snappy compression library](http://code.google.com/p/snappy). + * External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions. + * [Detailed documentation](http://htmlpreview.github.io/?https://github.com/google/leveldb/blob/master/doc/index.html) about how to use the library is included with the source code. + + +# Limitations + * This is not a SQL database. It does not have a relational data model, it does not support SQL queries, and it has no support for indexes. + * Only a single process (possibly multi-threaded) can access a particular database at a time. + * There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library. + +# Performance + +Here is a performance report (with explanations) from the run of the +included db_bench program. The results are somewhat noisy, but should +be enough to get a ballpark performance estimate. + +## Setup + +We use a database with a million entries. Each entry has a 16 byte +key, and a 100 byte value. Values used by the benchmark compress to +about half their original size. + + LevelDB: version 1.1 + Date: Sun May 1 12:11:26 2011 + CPU: 4 x Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz + CPUCache: 4096 KB + Keys: 16 bytes each + Values: 100 bytes each (50 bytes after compression) + Entries: 1000000 + Raw Size: 110.6 MB (estimated) + File Size: 62.9 MB (estimated) + +## Write performance + +The "fill" benchmarks create a brand new database, in either +sequential, or random order. The "fillsync" benchmark flushes data +from the operating system to the disk after every operation; the other +write operations leave the data sitting in the operating system buffer +cache for a while. The "overwrite" benchmark does random writes that +update existing keys in the database. + + fillseq : 1.765 micros/op; 62.7 MB/s + fillsync : 268.409 micros/op; 0.4 MB/s (10000 ops) + fillrandom : 2.460 micros/op; 45.0 MB/s + overwrite : 2.380 micros/op; 46.5 MB/s + +Each "op" above corresponds to a write of a single key/value pair. +I.e., a random write benchmark goes at approximately 400,000 writes per second. + +Each "fillsync" operation costs much less (0.3 millisecond) +than a disk seek (typically 10 milliseconds). We suspect that this is +because the hard disk itself is buffering the update in its memory and +responding before the data has been written to the platter. This may +or may not be safe based on whether or not the hard disk has enough +power to save its memory in the event of a power failure. + +## Read performance + +We list the performance of reading sequentially in both the forward +and reverse direction, and also the performance of a random lookup. +Note that the database created by the benchmark is quite small. +Therefore the report characterizes the performance of leveldb when the +working set fits in memory. The cost of reading a piece of data that +is not present in the operating system buffer cache will be dominated +by the one or two disk seeks needed to fetch the data from disk. +Write performance will be mostly unaffected by whether or not the +working set fits in memory. + + readrandom : 16.677 micros/op; (approximately 60,000 reads per second) + readseq : 0.476 micros/op; 232.3 MB/s + readreverse : 0.724 micros/op; 152.9 MB/s + +LevelDB compacts its underlying storage data in the background to +improve read performance. The results listed above were done +immediately after a lot of random writes. The results after +compactions (which are usually triggered automatically) are better. + + readrandom : 11.602 micros/op; (approximately 85,000 reads per second) + readseq : 0.423 micros/op; 261.8 MB/s + readreverse : 0.663 micros/op; 166.9 MB/s + +Some of the high cost of reads comes from repeated decompression of blocks +read from disk. If we supply enough cache to the leveldb so it can hold the +uncompressed blocks in memory, the read performance improves again: + + readrandom : 9.775 micros/op; (approximately 100,000 reads per second before compaction) + readrandom : 5.215 micros/op; (approximately 190,000 reads per second after compaction) + +## Repository contents + +See doc/index.html for more explanation. See doc/impl.html for a brief overview of the implementation. + +The public interface is in include/*.h. Callers should not include or +rely on the details of any other header files in this package. Those +internal APIs may be changed without warning. + +Guide to header files: + +* **include/db.h**: Main interface to the DB: Start here + +* **include/options.h**: Control over the behavior of an entire database, +and also control over the behavior of individual reads and writes. + +* **include/comparator.h**: Abstraction for user-specified comparison function. +If you want just bytewise comparison of keys, you can use the default +comparator, but clients can write their own comparator implementations if they +want custom ordering (e.g. to handle different character encodings, etc.) + +* **include/iterator.h**: Interface for iterating over data. You can get +an iterator from a DB object. + +* **include/write_batch.h**: Interface for atomically applying multiple +updates to a database. + +* **include/slice.h**: A simple module for maintaining a pointer and a +length into some other byte array. + +* **include/status.h**: Status is returned from many of the public interfaces +and is used to report success and various kinds of errors. + +* **include/env.h**: +Abstraction of the OS environment. A posix implementation of this interface is +in util/env_posix.cc + +* **include/table.h, include/table_builder.h**: Lower-level modules that most +clients probably won't use directly diff --git a/src/leveldb/build_detect_platform b/src/leveldb/build_detect_platform index 85b1ce0224..a1101c1bda 100755 --- a/src/leveldb/build_detect_platform +++ b/src/leveldb/build_detect_platform @@ -20,7 +20,7 @@ # # The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following: # -# -DLEVELDB_CSTDATOMIC_PRESENT if <cstdatomic> is present +# -DLEVELDB_ATOMIC_PRESENT if <atomic> is present # -DLEVELDB_PLATFORM_POSIX for Posix-based platforms # -DSNAPPY if the Snappy library is present # @@ -72,6 +72,12 @@ if [ "$CXX" = "g++" ]; then fi case "$TARGET_OS" in + CYGWIN_*) + PLATFORM=OS_LINUX + COMMON_FLAGS="$MEMCMP_FLAG -lpthread -DOS_LINUX -DCYGWIN" + PLATFORM_LDFLAGS="-lpthread" + PORT_FILE=port/port_posix.cc + ;; Darwin) PLATFORM=OS_MACOSX COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX" @@ -185,13 +191,14 @@ if [ "$CROSS_COMPILE" = "true" ]; then else CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$" - # If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h. + # If -std=c++0x works, use <atomic> as fallback for when memory barriers + # are not available. $CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF - #include <cstdatomic> + #include <atomic> int main() {} EOF if [ "$?" = 0 ]; then - COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_CSTDATOMIC_PRESENT" + COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_ATOMIC_PRESENT" PLATFORM_CXXFLAGS="-std=c++0x" else COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX" diff --git a/src/leveldb/db/db_bench.cc b/src/leveldb/db/db_bench.cc index fc46d89693..705a170aae 100644 --- a/src/leveldb/db/db_bench.cc +++ b/src/leveldb/db/db_bench.cc @@ -431,7 +431,7 @@ class Benchmark { benchmarks = sep + 1; } - // Reset parameters that may be overriddden bwlow + // Reset parameters that may be overridden below num_ = FLAGS_num; reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); value_size_ = FLAGS_value_size; @@ -811,7 +811,6 @@ class Benchmark { void SeekRandom(ThreadState* thread) { ReadOptions options; - std::string value; int found = 0; for (int i = 0; i < reads_; i++) { Iterator* iter = db_->NewIterator(options); diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc index faf5e7d7ba..49b95953b4 100644 --- a/src/leveldb/db/db_impl.cc +++ b/src/leveldb/db/db_impl.cc @@ -392,7 +392,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, reporter.info_log = options_.info_log; reporter.fname = fname.c_str(); reporter.status = (options_.paranoid_checks ? &status : NULL); - // We intentially make log::Reader do checksumming even if + // We intentionally make log::Reader do checksumming even if // paranoid_checks==false so that corruptions cause entire commits // to be skipped instead of propagating bad information (like overly // large sequence numbers). @@ -1267,7 +1267,7 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) { break; } - // Append to *reuslt + // Append to *result if (result == first->batch) { // Switch to temporary batch instead of disturbing caller's batch result = tmp_batch_; diff --git a/src/leveldb/db/db_test.cc b/src/leveldb/db/db_test.cc index 280b01c14b..0fed9137d5 100644 --- a/src/leveldb/db/db_test.cc +++ b/src/leveldb/db/db_test.cc @@ -626,7 +626,7 @@ TEST(DBTest, GetEncountersEmptyLevel) { // * sstable B in level 2 // Then do enough Get() calls to arrange for an automatic compaction // of sstable A. A bug would cause the compaction to be marked as - // occuring at level 1 (instead of the correct level 0). + // occurring at level 1 (instead of the correct level 0). // Step 1: First place sstables in levels 0 and 2 int compaction_count = 0; diff --git a/src/leveldb/db/dbformat.h b/src/leveldb/db/dbformat.h index 5d8a032bd3..ea897b13c0 100644 --- a/src/leveldb/db/dbformat.h +++ b/src/leveldb/db/dbformat.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#ifndef STORAGE_LEVELDB_DB_FORMAT_H_ -#define STORAGE_LEVELDB_DB_FORMAT_H_ +#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_ +#define STORAGE_LEVELDB_DB_DBFORMAT_H_ #include <stdio.h> #include "leveldb/comparator.h" @@ -227,4 +227,4 @@ inline LookupKey::~LookupKey() { } // namespace leveldb -#endif // STORAGE_LEVELDB_DB_FORMAT_H_ +#endif // STORAGE_LEVELDB_DB_DBFORMAT_H_ diff --git a/src/leveldb/db/dumpfile.cc b/src/leveldb/db/dumpfile.cc new file mode 100644 index 0000000000..61c47c2ff9 --- /dev/null +++ b/src/leveldb/db/dumpfile.cc @@ -0,0 +1,225 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include <stdio.h> +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/version_edit.h" +#include "db/write_batch_internal.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" +#include "leveldb/status.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" + +namespace leveldb { + +namespace { + +bool GuessType(const std::string& fname, FileType* type) { + size_t pos = fname.rfind('/'); + std::string basename; + if (pos == std::string::npos) { + basename = fname; + } else { + basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); + } + uint64_t ignored; + return ParseFileName(basename, &ignored, type); +} + +// Notified when log reader encounters corruption. +class CorruptionReporter : public log::Reader::Reporter { + public: + WritableFile* dst_; + virtual void Corruption(size_t bytes, const Status& status) { + std::string r = "corruption: "; + AppendNumberTo(&r, bytes); + r += " bytes; "; + r += status.ToString(); + r.push_back('\n'); + dst_->Append(r); + } +}; + +// Print contents of a log file. (*func)() is called on every record. +Status PrintLogContents(Env* env, const std::string& fname, + void (*func)(uint64_t, Slice, WritableFile*), + WritableFile* dst) { + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + return s; + } + CorruptionReporter reporter; + reporter.dst_ = dst; + log::Reader reader(file, &reporter, true, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch)) { + (*func)(reader.LastRecordOffset(), record, dst); + } + delete file; + return Status::OK(); +} + +// Called on every item found in a WriteBatch. +class WriteBatchItemPrinter : public WriteBatch::Handler { + public: + WritableFile* dst_; + virtual void Put(const Slice& key, const Slice& value) { + std::string r = " put '"; + AppendEscapedStringTo(&r, key); + r += "' '"; + AppendEscapedStringTo(&r, value); + r += "'\n"; + dst_->Append(r); + } + virtual void Delete(const Slice& key) { + std::string r = " del '"; + AppendEscapedStringTo(&r, key); + r += "'\n"; + dst_->Append(r); + } +}; + + +// Called on every log record (each one of which is a WriteBatch) +// found in a kLogFile. +static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) { + std::string r = "--- offset "; + AppendNumberTo(&r, pos); + r += "; "; + if (record.size() < 12) { + r += "log record length "; + AppendNumberTo(&r, record.size()); + r += " is too small\n"; + dst->Append(r); + return; + } + WriteBatch batch; + WriteBatchInternal::SetContents(&batch, record); + r += "sequence "; + AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch)); + r.push_back('\n'); + dst->Append(r); + WriteBatchItemPrinter batch_item_printer; + batch_item_printer.dst_ = dst; + Status s = batch.Iterate(&batch_item_printer); + if (!s.ok()) { + dst->Append(" error: " + s.ToString() + "\n"); + } +} + +Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) { + return PrintLogContents(env, fname, WriteBatchPrinter, dst); +} + +// Called on every log record (each one of which is a WriteBatch) +// found in a kDescriptorFile. +static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) { + std::string r = "--- offset "; + AppendNumberTo(&r, pos); + r += "; "; + VersionEdit edit; + Status s = edit.DecodeFrom(record); + if (!s.ok()) { + r += s.ToString(); + r.push_back('\n'); + } else { + r += edit.DebugString(); + } + dst->Append(r); +} + +Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) { + return PrintLogContents(env, fname, VersionEditPrinter, dst); +} + +Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) { + uint64_t file_size; + RandomAccessFile* file = NULL; + Table* table = NULL; + Status s = env->GetFileSize(fname, &file_size); + if (s.ok()) { + s = env->NewRandomAccessFile(fname, &file); + } + if (s.ok()) { + // We use the default comparator, which may or may not match the + // comparator used in this database. However this should not cause + // problems since we only use Table operations that do not require + // any comparisons. In particular, we do not call Seek or Prev. + s = Table::Open(Options(), file, file_size, &table); + } + if (!s.ok()) { + delete table; + delete file; + return s; + } + + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = table->NewIterator(ro); + std::string r; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + r.clear(); + ParsedInternalKey key; + if (!ParseInternalKey(iter->key(), &key)) { + r = "badkey '"; + AppendEscapedStringTo(&r, iter->key()); + r += "' => '"; + AppendEscapedStringTo(&r, iter->value()); + r += "'\n"; + dst->Append(r); + } else { + r = "'"; + AppendEscapedStringTo(&r, key.user_key); + r += "' @ "; + AppendNumberTo(&r, key.sequence); + r += " : "; + if (key.type == kTypeDeletion) { + r += "del"; + } else if (key.type == kTypeValue) { + r += "val"; + } else { + AppendNumberTo(&r, key.type); + } + r += " => '"; + AppendEscapedStringTo(&r, iter->value()); + r += "'\n"; + dst->Append(r); + } + } + s = iter->status(); + if (!s.ok()) { + dst->Append("iterator error: " + s.ToString() + "\n"); + } + + delete iter; + delete table; + delete file; + return Status::OK(); +} + +} // namespace + +Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) { + FileType ftype; + if (!GuessType(fname, &ftype)) { + return Status::InvalidArgument(fname + ": unknown file type"); + } + switch (ftype) { + case kLogFile: return DumpLog(env, fname, dst); + case kDescriptorFile: return DumpDescriptor(env, fname, dst); + case kTableFile: return DumpTable(env, fname, dst); + default: + break; + } + return Status::InvalidArgument(fname + ": not a dump-able file type"); +} + +} // namespace leveldb diff --git a/src/leveldb/db/leveldb_main.cc b/src/leveldb/db/leveldb_main.cc index 995d76107a..9f4b7dd70c 100644 --- a/src/leveldb/db/leveldb_main.cc +++ b/src/leveldb/db/leveldb_main.cc @@ -3,212 +3,38 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include <stdio.h> -#include "db/dbformat.h" -#include "db/filename.h" -#include "db/log_reader.h" -#include "db/version_edit.h" -#include "db/write_batch_internal.h" +#include "leveldb/dumpfile.h" #include "leveldb/env.h" -#include "leveldb/iterator.h" -#include "leveldb/options.h" #include "leveldb/status.h" -#include "leveldb/table.h" -#include "leveldb/write_batch.h" -#include "util/logging.h" namespace leveldb { - namespace { -bool GuessType(const std::string& fname, FileType* type) { - size_t pos = fname.rfind('/'); - std::string basename; - if (pos == std::string::npos) { - basename = fname; - } else { - basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); - } - uint64_t ignored; - return ParseFileName(basename, &ignored, type); -} - -// Notified when log reader encounters corruption. -class CorruptionReporter : public log::Reader::Reporter { - public: - virtual void Corruption(size_t bytes, const Status& status) { - printf("corruption: %d bytes; %s\n", - static_cast<int>(bytes), - status.ToString().c_str()); - } -}; - -// Print contents of a log file. (*func)() is called on every record. -bool PrintLogContents(Env* env, const std::string& fname, - void (*func)(Slice)) { - SequentialFile* file; - Status s = env->NewSequentialFile(fname, &file); - if (!s.ok()) { - fprintf(stderr, "%s\n", s.ToString().c_str()); - return false; - } - CorruptionReporter reporter; - log::Reader reader(file, &reporter, true, 0); - Slice record; - std::string scratch; - while (reader.ReadRecord(&record, &scratch)) { - printf("--- offset %llu; ", - static_cast<unsigned long long>(reader.LastRecordOffset())); - (*func)(record); - } - delete file; - return true; -} - -// Called on every item found in a WriteBatch. -class WriteBatchItemPrinter : public WriteBatch::Handler { +class StdoutPrinter : public WritableFile { public: - uint64_t offset_; - uint64_t sequence_; - - virtual void Put(const Slice& key, const Slice& value) { - printf(" put '%s' '%s'\n", - EscapeString(key).c_str(), - EscapeString(value).c_str()); - } - virtual void Delete(const Slice& key) { - printf(" del '%s'\n", - EscapeString(key).c_str()); + virtual Status Append(const Slice& data) { + fwrite(data.data(), 1, data.size(), stdout); + return Status::OK(); } + virtual Status Close() { return Status::OK(); } + virtual Status Flush() { return Status::OK(); } + virtual Status Sync() { return Status::OK(); } }; - -// Called on every log record (each one of which is a WriteBatch) -// found in a kLogFile. -static void WriteBatchPrinter(Slice record) { - if (record.size() < 12) { - printf("log record length %d is too small\n", - static_cast<int>(record.size())); - return; - } - WriteBatch batch; - WriteBatchInternal::SetContents(&batch, record); - printf("sequence %llu\n", - static_cast<unsigned long long>(WriteBatchInternal::Sequence(&batch))); - WriteBatchItemPrinter batch_item_printer; - Status s = batch.Iterate(&batch_item_printer); - if (!s.ok()) { - printf(" error: %s\n", s.ToString().c_str()); - } -} - -bool DumpLog(Env* env, const std::string& fname) { - return PrintLogContents(env, fname, WriteBatchPrinter); -} - -// Called on every log record (each one of which is a WriteBatch) -// found in a kDescriptorFile. -static void VersionEditPrinter(Slice record) { - VersionEdit edit; - Status s = edit.DecodeFrom(record); - if (!s.ok()) { - printf("%s\n", s.ToString().c_str()); - return; - } - printf("%s", edit.DebugString().c_str()); -} - -bool DumpDescriptor(Env* env, const std::string& fname) { - return PrintLogContents(env, fname, VersionEditPrinter); -} - -bool DumpTable(Env* env, const std::string& fname) { - uint64_t file_size; - RandomAccessFile* file = NULL; - Table* table = NULL; - Status s = env->GetFileSize(fname, &file_size); - if (s.ok()) { - s = env->NewRandomAccessFile(fname, &file); - } - if (s.ok()) { - // We use the default comparator, which may or may not match the - // comparator used in this database. However this should not cause - // problems since we only use Table operations that do not require - // any comparisons. In particular, we do not call Seek or Prev. - s = Table::Open(Options(), file, file_size, &table); - } - if (!s.ok()) { - fprintf(stderr, "%s\n", s.ToString().c_str()); - delete table; - delete file; - return false; - } - - ReadOptions ro; - ro.fill_cache = false; - Iterator* iter = table->NewIterator(ro); - for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { - ParsedInternalKey key; - if (!ParseInternalKey(iter->key(), &key)) { - printf("badkey '%s' => '%s'\n", - EscapeString(iter->key()).c_str(), - EscapeString(iter->value()).c_str()); - } else { - char kbuf[20]; - const char* type; - if (key.type == kTypeDeletion) { - type = "del"; - } else if (key.type == kTypeValue) { - type = "val"; - } else { - snprintf(kbuf, sizeof(kbuf), "%d", static_cast<int>(key.type)); - type = kbuf; - } - printf("'%s' @ %8llu : %s => '%s'\n", - EscapeString(key.user_key).c_str(), - static_cast<unsigned long long>(key.sequence), - type, - EscapeString(iter->value()).c_str()); - } - } - s = iter->status(); - if (!s.ok()) { - printf("iterator error: %s\n", s.ToString().c_str()); - } - - delete iter; - delete table; - delete file; - return true; -} - -bool DumpFile(Env* env, const std::string& fname) { - FileType ftype; - if (!GuessType(fname, &ftype)) { - fprintf(stderr, "%s: unknown file type\n", fname.c_str()); - return false; - } - switch (ftype) { - case kLogFile: return DumpLog(env, fname); - case kDescriptorFile: return DumpDescriptor(env, fname); - case kTableFile: return DumpTable(env, fname); - - default: { - fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str()); - break; - } - } - return false; -} - bool HandleDumpCommand(Env* env, char** files, int num) { + StdoutPrinter printer; bool ok = true; for (int i = 0; i < num; i++) { - ok &= DumpFile(env, files[i]); + Status s = DumpFile(env, files[i], &printer); + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + ok = false; + } } return ok; } -} +} // namespace } // namespace leveldb static void Usage() { diff --git a/src/leveldb/db/log_format.h b/src/leveldb/db/log_format.h index 2690cb9789..a8c06efe18 100644 --- a/src/leveldb/db/log_format.h +++ b/src/leveldb/db/log_format.h @@ -26,8 +26,8 @@ static const int kMaxRecordType = kLastType; static const int kBlockSize = 32768; -// Header is checksum (4 bytes), type (1 byte), length (2 bytes). -static const int kHeaderSize = 4 + 1 + 2; +// Header is checksum (4 bytes), length (2 bytes), type (1 byte). +static const int kHeaderSize = 4 + 2 + 1; } // namespace log } // namespace leveldb diff --git a/src/leveldb/db/log_reader.cc b/src/leveldb/db/log_reader.cc index 4919216d04..e44b66c85b 100644 --- a/src/leveldb/db/log_reader.cc +++ b/src/leveldb/db/log_reader.cc @@ -167,14 +167,14 @@ uint64_t Reader::LastRecordOffset() { return last_record_offset_; } -void Reader::ReportCorruption(size_t bytes, const char* reason) { +void Reader::ReportCorruption(uint64_t bytes, const char* reason) { ReportDrop(bytes, Status::Corruption(reason)); } -void Reader::ReportDrop(size_t bytes, const Status& reason) { +void Reader::ReportDrop(uint64_t bytes, const Status& reason) { if (reporter_ != NULL && end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) { - reporter_->Corruption(bytes, reason); + reporter_->Corruption(static_cast<size_t>(bytes), reason); } } diff --git a/src/leveldb/db/log_reader.h b/src/leveldb/db/log_reader.h index 82d4bee68d..6aff791716 100644 --- a/src/leveldb/db/log_reader.h +++ b/src/leveldb/db/log_reader.h @@ -94,8 +94,8 @@ class Reader { // Reports dropped bytes to the reporter. // buffer_ must be updated to remove the dropped bytes prior to invocation. - void ReportCorruption(size_t bytes, const char* reason); - void ReportDrop(size_t bytes, const Status& reason); + void ReportCorruption(uint64_t bytes, const char* reason); + void ReportDrop(uint64_t bytes, const Status& reason); // No copying allowed Reader(const Reader&); diff --git a/src/leveldb/db/log_test.cc b/src/leveldb/db/log_test.cc index 91d3caafc3..dcf0562652 100644 --- a/src/leveldb/db/log_test.cc +++ b/src/leveldb/db/log_test.cc @@ -463,7 +463,7 @@ TEST(LogTest, ErrorJoinsRecords) { ASSERT_EQ("correct", Read()); ASSERT_EQ("EOF", Read()); - const int dropped = DroppedBytes(); + const size_t dropped = DroppedBytes(); ASSERT_LE(dropped, 2*kBlockSize + 100); ASSERT_GE(dropped, 2*kBlockSize); } diff --git a/src/leveldb/db/repair.cc b/src/leveldb/db/repair.cc index 7727fafc58..4cd4bb047f 100644 --- a/src/leveldb/db/repair.cc +++ b/src/leveldb/db/repair.cc @@ -186,7 +186,7 @@ class Repairer { reporter.env = env_; reporter.info_log = options_.info_log; reporter.lognum = log; - // We intentially make log::Reader do checksumming so that + // We intentionally make log::Reader do checksumming so that // corruptions cause entire commits to be skipped instead of // propagating bad information (like overly large sequence // numbers). diff --git a/src/leveldb/db/skiplist.h b/src/leveldb/db/skiplist.h index af85be6d01..ed8b092203 100644 --- a/src/leveldb/db/skiplist.h +++ b/src/leveldb/db/skiplist.h @@ -1,3 +1,6 @@ +#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_ +#define STORAGE_LEVELDB_DB_SKIPLIST_H_ + // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. @@ -377,3 +380,5 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const { } } // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_ diff --git a/src/leveldb/db/write_batch_internal.h b/src/leveldb/db/write_batch_internal.h index 4423a7f318..310a3c8912 100644 --- a/src/leveldb/db/write_batch_internal.h +++ b/src/leveldb/db/write_batch_internal.h @@ -21,10 +21,10 @@ class WriteBatchInternal { // Set the count for the number of entries in the batch. static void SetCount(WriteBatch* batch, int n); - // Return the seqeunce number for the start of this batch. + // Return the sequence number for the start of this batch. static SequenceNumber Sequence(const WriteBatch* batch); - // Store the specified number as the seqeunce number for the start of + // Store the specified number as the sequence number for the start of // this batch. static void SetSequence(WriteBatch* batch, SequenceNumber seq); diff --git a/src/leveldb/doc/bench/db_bench_tree_db.cc b/src/leveldb/doc/bench/db_bench_tree_db.cc index ed86f031c2..4ca381f11f 100644 --- a/src/leveldb/doc/bench/db_bench_tree_db.cc +++ b/src/leveldb/doc/bench/db_bench_tree_db.cc @@ -338,7 +338,7 @@ class Benchmark { bool write_sync = false; if (name == Slice("fillseq")) { Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1); - + DBSynchronize(db_); } else if (name == Slice("fillrandom")) { Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1); DBSynchronize(db_); diff --git a/src/leveldb/doc/impl.html b/src/leveldb/doc/impl.html index 28817fe0da..6a468be095 100644 --- a/src/leveldb/doc/impl.html +++ b/src/leveldb/doc/impl.html @@ -111,7 +111,7 @@ A compaction merges the contents of the picked files to produce a sequence of level-(L+1) files. We switch to producing a new level-(L+1) file after the current output file has reached the target file size (2MB). We also switch to a new output file when the key -range of the current output file has grown enough to overlap more then +range of the current output file has grown enough to overlap more than ten level-(L+2) files. This last rule ensures that a later compaction of a level-(L+1) file will not pick up too much data from level-(L+2). @@ -151,7 +151,7 @@ compaction cost will be approximately 0.5 second. If we throttle the background writing to something small, say 10% of the full 100MB/s speed, a compaction may take up to 5 seconds. If the user is writing at 10MB/s, we might build up lots of level-0 files -(~50 to hold the 5*10MB). This may signficantly increase the cost of +(~50 to hold the 5*10MB). This may significantly increase the cost of reads due to the overhead of merging more files together on every read. diff --git a/src/leveldb/doc/log_format.txt b/src/leveldb/doc/log_format.txt index 5228f624de..4cca5ef6ea 100644 --- a/src/leveldb/doc/log_format.txt +++ b/src/leveldb/doc/log_format.txt @@ -11,7 +11,7 @@ Each block consists of a sequence of records: A record never starts within the last six bytes of a block (since it won't fit). Any leftover bytes here form the trailer, which must -consist entirely of zero bytes and must be skipped by readers. +consist entirely of zero bytes and must be skipped by readers. Aside: if exactly seven bytes are left in the current block, and a new non-zero length record is added, the writer must emit a FIRST record @@ -33,8 +33,8 @@ The FULL record contains the contents of an entire user record. FIRST, MIDDLE, LAST are types used for user records that have been split into multiple fragments (typically because of block boundaries). FIRST is the type of the first fragment of a user record, LAST is the -type of the last fragment of a user record, and MID is the type of all -interior fragments of a user record. +type of the last fragment of a user record, and MIDDLE is the type of +all interior fragments of a user record. Example: consider a sequence of user records: A: length 1000 diff --git a/src/leveldb/helpers/memenv/memenv.cc b/src/leveldb/helpers/memenv/memenv.cc index 5879de1214..43ef2e0729 100644 --- a/src/leveldb/helpers/memenv/memenv.cc +++ b/src/leveldb/helpers/memenv/memenv.cc @@ -55,14 +55,15 @@ class FileState { } const uint64_t available = size_ - offset; if (n > available) { - n = available; + n = static_cast<size_t>(available); } if (n == 0) { *result = Slice(); return Status::OK(); } - size_t block = offset / kBlockSize; + assert(offset / kBlockSize <= SIZE_MAX); + size_t block = static_cast<size_t>(offset / kBlockSize); size_t block_offset = offset % kBlockSize; if (n <= kBlockSize - block_offset) { @@ -167,7 +168,7 @@ class SequentialFileImpl : public SequentialFile { if (pos_ > file_->Size()) { return Status::IOError("pos_ > file_->Size()"); } - const size_t available = file_->Size() - pos_; + const uint64_t available = file_->Size() - pos_; if (n > available) { n = available; } @@ -177,7 +178,7 @@ class SequentialFileImpl : public SequentialFile { private: FileState* file_; - size_t pos_; + uint64_t pos_; }; class RandomAccessFileImpl : public RandomAccessFile { diff --git a/src/leveldb/include/leveldb/cache.h b/src/leveldb/include/leveldb/cache.h index 5e3b47637d..1a201e5e0a 100644 --- a/src/leveldb/include/leveldb/cache.h +++ b/src/leveldb/include/leveldb/cache.h @@ -96,4 +96,4 @@ class Cache { } // namespace leveldb -#endif // STORAGE_LEVELDB_UTIL_CACHE_H_ +#endif // STORAGE_LEVELDB_INCLUDE_CACHE_H_ diff --git a/src/leveldb/include/leveldb/db.h b/src/leveldb/include/leveldb/db.h index 40851b2aa8..4c169bf22e 100644 --- a/src/leveldb/include/leveldb/db.h +++ b/src/leveldb/include/leveldb/db.h @@ -14,7 +14,7 @@ namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 17; +static const int kMinorVersion = 18; struct Options; struct ReadOptions; diff --git a/src/leveldb/include/leveldb/dumpfile.h b/src/leveldb/include/leveldb/dumpfile.h new file mode 100644 index 0000000000..3f97fda16b --- /dev/null +++ b/src/leveldb/include/leveldb/dumpfile.h @@ -0,0 +1,25 @@ +// Copyright (c) 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ +#define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ + +#include <string> +#include "leveldb/env.h" +#include "leveldb/status.h" + +namespace leveldb { + +// Dump the contents of the file named by fname in text format to +// *dst. Makes a sequence of dst->Append() calls; each call is passed +// the newline-terminated text corresponding to a single item found +// in the file. +// +// Returns a non-OK result if fname does not name a leveldb storage +// file, or if the file cannot be read. +Status DumpFile(Env* env, const std::string& fname, WritableFile* dst); + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ diff --git a/src/leveldb/include/leveldb/env.h b/src/leveldb/include/leveldb/env.h index b2072d02c1..f709514da6 100644 --- a/src/leveldb/include/leveldb/env.h +++ b/src/leveldb/include/leveldb/env.h @@ -142,7 +142,7 @@ class Env { // useful for computing deltas of time. virtual uint64_t NowMicros() = 0; - // Sleep/delay the thread for the perscribed number of micro-seconds. + // Sleep/delay the thread for the prescribed number of micro-seconds. virtual void SleepForMicroseconds(int micros) = 0; private: diff --git a/src/leveldb/include/leveldb/iterator.h b/src/leveldb/include/leveldb/iterator.h index ad543eb46c..76aced04bd 100644 --- a/src/leveldb/include/leveldb/iterator.h +++ b/src/leveldb/include/leveldb/iterator.h @@ -61,7 +61,7 @@ class Iterator { // Return the value for the current entry. The underlying storage for // the returned slice is valid only until the next modification of // the iterator. - // REQUIRES: !AtEnd() && !AtStart() + // REQUIRES: Valid() virtual Slice value() const = 0; // If an error has occurred, return it. Else return an ok status. diff --git a/src/leveldb/include/leveldb/options.h b/src/leveldb/include/leveldb/options.h index fdda718d30..7c9b973454 100644 --- a/src/leveldb/include/leveldb/options.h +++ b/src/leveldb/include/leveldb/options.h @@ -153,7 +153,7 @@ struct ReadOptions { // If "snapshot" is non-NULL, read as of the supplied snapshot // (which must belong to the DB that is being read and which must - // not have been released). If "snapshot" is NULL, use an impliicit + // not have been released). If "snapshot" is NULL, use an implicit // snapshot of the state at the beginning of this read operation. // Default: NULL const Snapshot* snapshot; diff --git a/src/leveldb/port/atomic_pointer.h b/src/leveldb/port/atomic_pointer.h index a9866b2302..9bf091f757 100644 --- a/src/leveldb/port/atomic_pointer.h +++ b/src/leveldb/port/atomic_pointer.h @@ -5,14 +5,13 @@ // AtomicPointer provides storage for a lock-free pointer. // Platform-dependent implementation of AtomicPointer: // - If the platform provides a cheap barrier, we use it with raw pointers -// - If cstdatomic is present (on newer versions of gcc, it is), we use -// a cstdatomic-based AtomicPointer. However we prefer the memory +// - If <atomic> is present (on newer versions of gcc, it is), we use +// a <atomic>-based AtomicPointer. However we prefer the memory // barrier based version, because at least on a gcc 4.4 32-bit build -// on linux, we have encountered a buggy <cstdatomic> -// implementation. Also, some <cstdatomic> implementations are much -// slower than a memory-barrier based implementation (~16ns for -// <cstdatomic> based acquire-load vs. ~1ns for a barrier based -// acquire-load). +// on linux, we have encountered a buggy <atomic> implementation. +// Also, some <atomic> implementations are much slower than a memory-barrier +// based implementation (~16ns for <atomic> based acquire-load vs. ~1ns for +// a barrier based acquire-load). // This code is based on atomicops-internals-* in Google's perftools: // http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase @@ -20,8 +19,8 @@ #define PORT_ATOMIC_POINTER_H_ #include <stdint.h> -#ifdef LEVELDB_CSTDATOMIC_PRESENT -#include <cstdatomic> +#ifdef LEVELDB_ATOMIC_PRESENT +#include <atomic> #endif #ifdef OS_WIN #include <windows.h> @@ -126,7 +125,7 @@ class AtomicPointer { }; // AtomicPointer based on <cstdatomic> -#elif defined(LEVELDB_CSTDATOMIC_PRESENT) +#elif defined(LEVELDB_ATOMIC_PRESENT) class AtomicPointer { private: std::atomic<void*> rep_; @@ -207,7 +206,7 @@ class AtomicPointer { inline void NoBarrier_Store(void* v) { rep_ = v; } }; -// We have neither MemoryBarrier(), nor <cstdatomic> +// We have neither MemoryBarrier(), nor <atomic> #else #error Please implement AtomicPointer for this platform. diff --git a/src/leveldb/port/port_posix.h b/src/leveldb/port/port_posix.h index 21c845e211..ccca9939d3 100644 --- a/src/leveldb/port/port_posix.h +++ b/src/leveldb/port/port_posix.h @@ -21,14 +21,11 @@ #else #define PLATFORM_IS_LITTLE_ENDIAN false #endif -#elif defined(OS_FREEBSD) +#elif defined(OS_FREEBSD) || defined(OS_OPENBSD) ||\ + defined(OS_NETBSD) || defined(OS_DRAGONFLYBSD) #include <sys/types.h> #include <sys/endian.h> #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) -#elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\ - defined(OS_DRAGONFLYBSD) - #include <sys/types.h> - #include <sys/endian.h> #elif defined(OS_HPUX) #define PLATFORM_IS_LITTLE_ENDIAN false #elif defined(OS_ANDROID) @@ -55,7 +52,7 @@ #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ - defined(OS_ANDROID) || defined(OS_HPUX) + defined(OS_ANDROID) || defined(OS_HPUX) || defined(CYGWIN) // Use fread/fwrite/fflush on platforms without _unlocked variants #define fread_unlocked fread #define fwrite_unlocked fwrite diff --git a/src/leveldb/port/thread_annotations.h b/src/leveldb/port/thread_annotations.h index 6f9b6a7924..9470ef587c 100644 --- a/src/leveldb/port/thread_annotations.h +++ b/src/leveldb/port/thread_annotations.h @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H +#ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ +#define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ // Some environments provide custom macros to aid in static thread-safety // analysis. Provide empty definitions of such macros unless they are already @@ -56,4 +57,4 @@ #define NO_THREAD_SAFETY_ANALYSIS #endif -#endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H +#endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ diff --git a/src/leveldb/table/block.cc b/src/leveldb/table/block.cc index 79ea9d9ee5..43e402c9c0 100644 --- a/src/leveldb/table/block.cc +++ b/src/leveldb/table/block.cc @@ -46,7 +46,7 @@ Block::~Block() { // Helper routine: decode the next block entry starting at "p", // storing the number of shared key bytes, non_shared key bytes, // and the length of the value in "*shared", "*non_shared", and -// "*value_length", respectively. Will not derefence past "limit". +// "*value_length", respectively. Will not dereference past "limit". // // If any errors are detected, returns NULL. Otherwise, returns a // pointer to the key delta (just past the three decoded values). diff --git a/src/leveldb/table/block_builder.h b/src/leveldb/table/block_builder.h index 5b545bd1af..4fbcb33972 100644 --- a/src/leveldb/table/block_builder.h +++ b/src/leveldb/table/block_builder.h @@ -21,7 +21,7 @@ class BlockBuilder { // Reset the contents as if the BlockBuilder was just constructed. void Reset(); - // REQUIRES: Finish() has not been callled since the last call to Reset(). + // REQUIRES: Finish() has not been called since the last call to Reset(). // REQUIRES: key is larger than any previously added key void Add(const Slice& key, const Slice& value); diff --git a/src/leveldb/table/format.cc b/src/leveldb/table/format.cc index cda1decdf3..aa63144c9e 100644 --- a/src/leveldb/table/format.cc +++ b/src/leveldb/table/format.cc @@ -48,7 +48,7 @@ Status Footer::DecodeFrom(Slice* input) { const uint64_t magic = ((static_cast<uint64_t>(magic_hi) << 32) | (static_cast<uint64_t>(magic_lo))); if (magic != kTableMagicNumber) { - return Status::InvalidArgument("not an sstable (bad magic number)"); + return Status::Corruption("not an sstable (bad magic number)"); } Status result = metaindex_handle_.DecodeFrom(input); diff --git a/src/leveldb/table/table.cc b/src/leveldb/table/table.cc index 71c1756e5f..dff8a82590 100644 --- a/src/leveldb/table/table.cc +++ b/src/leveldb/table/table.cc @@ -41,7 +41,7 @@ Status Table::Open(const Options& options, Table** table) { *table = NULL; if (size < Footer::kEncodedLength) { - return Status::InvalidArgument("file is too short to be an sstable"); + return Status::Corruption("file is too short to be an sstable"); } char footer_space[Footer::kEncodedLength]; @@ -58,7 +58,11 @@ Status Table::Open(const Options& options, BlockContents contents; Block* index_block = NULL; if (s.ok()) { - s = ReadBlock(file, ReadOptions(), footer.index_handle(), &contents); + ReadOptions opt; + if (options.paranoid_checks) { + opt.verify_checksums = true; + } + s = ReadBlock(file, opt, footer.index_handle(), &contents); if (s.ok()) { index_block = new Block(contents); } @@ -92,6 +96,9 @@ void Table::ReadMeta(const Footer& footer) { // TODO(sanjay): Skip this if footer.metaindex_handle() size indicates // it is an empty block. ReadOptions opt; + if (rep_->options.paranoid_checks) { + opt.verify_checksums = true; + } BlockContents contents; if (!ReadBlock(rep_->file, opt, footer.metaindex_handle(), &contents).ok()) { // Do not propagate errors since meta info is not needed for operation @@ -120,6 +127,9 @@ void Table::ReadFilter(const Slice& filter_handle_value) { // We might want to unify with ReadBlock() if we start // requiring checksum verification in Table::Open. ReadOptions opt; + if (rep_->options.paranoid_checks) { + opt.verify_checksums = true; + } BlockContents block; if (!ReadBlock(rep_->file, opt, filter_handle, &block).ok()) { return; diff --git a/src/leveldb/util/bloom.cc b/src/leveldb/util/bloom.cc index d7941cd21f..a27a2ace28 100644 --- a/src/leveldb/util/bloom.cc +++ b/src/leveldb/util/bloom.cc @@ -29,7 +29,7 @@ class BloomFilterPolicy : public FilterPolicy { } virtual const char* Name() const { - return "leveldb.BuiltinBloomFilter"; + return "leveldb.BuiltinBloomFilter2"; } virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index 93eadb1a4f..ba2667864a 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -3,8 +3,6 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #if !defined(LEVELDB_PLATFORM_WINDOWS) -#include <deque> -#include <set> #include <dirent.h> #include <errno.h> #include <fcntl.h> @@ -18,9 +16,8 @@ #include <sys/types.h> #include <time.h> #include <unistd.h> -#if defined(LEVELDB_PLATFORM_ANDROID) -#include <sys/stat.h> -#endif +#include <deque> +#include <set> #include "leveldb/env.h" #include "leveldb/slice.h" #include "port/port.h" @@ -296,7 +293,8 @@ class PosixEnv : public Env { public: PosixEnv(); virtual ~PosixEnv() { - fprintf(stderr, "Destroying Env::Default()\n"); + char msg[] = "Destroying Env::Default()\n"; + fwrite(msg, 1, sizeof(msg), stderr); abort(); } diff --git a/src/leveldb/util/hash.cc b/src/leveldb/util/hash.cc index 07cf022060..ed439ce7a2 100644 --- a/src/leveldb/util/hash.cc +++ b/src/leveldb/util/hash.cc @@ -34,13 +34,13 @@ uint32_t Hash(const char* data, size_t n, uint32_t seed) { // Pick up remaining bytes switch (limit - data) { case 3: - h += data[2] << 16; + h += static_cast<unsigned char>(data[2]) << 16; FALLTHROUGH_INTENDED; case 2: - h += data[1] << 8; + h += static_cast<unsigned char>(data[1]) << 8; FALLTHROUGH_INTENDED; case 1: - h += data[0]; + h += static_cast<unsigned char>(data[0]); h *= m; h ^= (h >> r); break; diff --git a/src/leveldb/util/hash_test.cc b/src/leveldb/util/hash_test.cc new file mode 100644 index 0000000000..eaa1c92c23 --- /dev/null +++ b/src/leveldb/util/hash_test.cc @@ -0,0 +1,54 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "util/hash.h" +#include "util/testharness.h" + +namespace leveldb { + +class HASH { }; + +TEST(HASH, SignedUnsignedIssue) { + const unsigned char data1[1] = {0x62}; + const unsigned char data2[2] = {0xc3, 0x97}; + const unsigned char data3[3] = {0xe2, 0x99, 0xa5}; + const unsigned char data4[4] = {0xe1, 0x80, 0xb9, 0x32}; + const unsigned char data5[48] = { + 0x01, 0xc0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x14, + 0x00, 0x00, 0x00, 0x18, + 0x28, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }; + + ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34); + ASSERT_EQ( + Hash(reinterpret_cast<const char*>(data1), sizeof(data1), 0xbc9f1d34), + 0xef1345c4); + ASSERT_EQ( + Hash(reinterpret_cast<const char*>(data2), sizeof(data2), 0xbc9f1d34), + 0x5b663814); + ASSERT_EQ( + Hash(reinterpret_cast<const char*>(data3), sizeof(data3), 0xbc9f1d34), + 0x323c078f); + ASSERT_EQ( + Hash(reinterpret_cast<const char*>(data4), sizeof(data4), 0xbc9f1d34), + 0xed21633a); + ASSERT_EQ( + Hash(reinterpret_cast<const char*>(data5), sizeof(data5), 0x12345678), + 0xf333dabb); +} + +} // namespace leveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/src/leveldb/util/logging.cc b/src/leveldb/util/logging.cc index 22cf278512..ca6b324403 100644 --- a/src/leveldb/util/logging.cc +++ b/src/leveldb/util/logging.cc @@ -45,15 +45,6 @@ std::string EscapeString(const Slice& value) { return r; } -bool ConsumeChar(Slice* in, char c) { - if (!in->empty() && (*in)[0] == c) { - in->remove_prefix(1); - return true; - } else { - return false; - } -} - bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { uint64_t v = 0; int digits = 0; diff --git a/src/leveldb/util/logging.h b/src/leveldb/util/logging.h index b0c5da813e..1b450d2480 100644 --- a/src/leveldb/util/logging.h +++ b/src/leveldb/util/logging.h @@ -32,10 +32,6 @@ extern std::string NumberToString(uint64_t num); // Escapes any non-printable characters found in "value". extern std::string EscapeString(const Slice& value); -// If *in starts with "c", advances *in past the first character and -// returns true. Otherwise, returns false. -extern bool ConsumeChar(Slice* in, char c); - // Parse a human-readable number from "*in" into *value. On success, // advances "*in" past the consumed number and sets "*val" to the // numeric value. Otherwise, returns false and leaves *in in an diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index da5ba61c7b..d997d56e00 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -6,6 +6,7 @@ #define BITCOIN_LEVELDBWRAPPER_H #include "serialize.h" +#include "streams.h" #include "util.h" #include "version.h" @@ -14,6 +14,7 @@ #include "netbase.h" #include "protocol.h" #include "random.h" +#include "streams.h" #include "sync.h" #include "uint256.h" #include "utilstrencodings.h" diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index e52feeb46e..040a234177 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -21,7 +21,9 @@ class BitcoinAmountField: public QWidget { Q_OBJECT - Q_PROPERTY(CAmount value READ value WRITE setValue NOTIFY valueChanged USER true) + // ugly hack: for some unknown reason CAmount (instead of qint64) does not work here as expected + // discussion: https://github.com/bitcoin/bitcoin/pull/5117 + Q_PROPERTY(qint64 value READ value WRITE setValue NOTIFY valueChanged USER true) public: explicit BitcoinAmountField(QWidget *parent = 0); diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 0e5802922c..5deac8007c 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -7,6 +7,7 @@ #include "bitcoinunits.h" #include "guiutil.h" #include "optionsmodel.h" +#include "streams.h" #include <boost/foreach.hpp> diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 5beac0512a..78f5569895 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -225,7 +225,7 @@ Value getblockhash(const Array& params, bool fHelp) int nHeight = params[0].get_int(); if (nHeight < 0 || nHeight > chainActive.Height()) - throw runtime_error("Block number out of range."); + throw JSONRPCError(RPC_INVALID_PARAMETER, "Block height out of range"); CBlockIndex* pblockindex = chainActive[nHeight]; return pblockindex->GetBlockHash().GetHex(); @@ -319,6 +319,7 @@ Value gettxoutsetinfo(const Array& params, bool fHelp) Object ret; CCoinsStats stats; + pcoinsTip->Flush(); if (pcoinsTip->GetStats(stats)) { ret.push_back(Pair("height", (int64_t)stats.nHeight)); ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); diff --git a/src/serialize.h b/src/serialize.h index b9d5f95463..877ef8640a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -6,8 +6,6 @@ #ifndef BITCOIN_SERIALIZE_H #define BITCOIN_SERIALIZE_H -#include "allocators.h" - #include <algorithm> #include <assert.h> #include <ios> @@ -20,8 +18,6 @@ #include <utility> #include <vector> -class CAutoFile; -class CDataStream; class CScript; static const unsigned int MAX_SIZE = 0x02000000; @@ -761,8 +757,6 @@ inline void SerReadWrite(Stream& s, T& obj, int nType, int nVersion, CSerActionU -typedef std::vector<char, zero_after_free_allocator<char> > CSerializeData; - class CSizeComputer { protected: @@ -792,551 +786,4 @@ public: } }; -/** Double ended buffer combining vector and stream-like interfaces. - * - * >> and << read and write unformatted data using the above serialization templates. - * Fills with data in linear time; some stringstream implementations take N^2 time. - */ -class CDataStream -{ -protected: - typedef CSerializeData vector_type; - vector_type vch; - unsigned int nReadPos; -public: - int nType; - int nVersion; - - typedef vector_type::allocator_type allocator_type; - typedef vector_type::size_type size_type; - typedef vector_type::difference_type difference_type; - typedef vector_type::reference reference; - typedef vector_type::const_reference const_reference; - typedef vector_type::value_type value_type; - typedef vector_type::iterator iterator; - typedef vector_type::const_iterator const_iterator; - typedef vector_type::reverse_iterator reverse_iterator; - - explicit CDataStream(int nTypeIn, int nVersionIn) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) - { - Init(nTypeIn, nVersionIn); - } - -#if !defined(_MSC_VER) || _MSC_VER >= 1300 - CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) - { - Init(nTypeIn, nVersionIn); - } -#endif - - CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } - - void Init(int nTypeIn, int nVersionIn) - { - nReadPos = 0; - nType = nTypeIn; - nVersion = nVersionIn; - } - - CDataStream& operator+=(const CDataStream& b) - { - vch.insert(vch.end(), b.begin(), b.end()); - return *this; - } - - friend CDataStream operator+(const CDataStream& a, const CDataStream& b) - { - CDataStream ret = a; - ret += b; - return (ret); - } - - std::string str() const - { - return (std::string(begin(), end())); - } - - - // - // Vector subset - // - const_iterator begin() const { return vch.begin() + nReadPos; } - iterator begin() { return vch.begin() + nReadPos; } - const_iterator end() const { return vch.end(); } - iterator end() { return vch.end(); } - size_type size() const { return vch.size() - nReadPos; } - bool empty() const { return vch.size() == nReadPos; } - void resize(size_type n, value_type c=0) { vch.resize(n + nReadPos, c); } - void reserve(size_type n) { vch.reserve(n + nReadPos); } - const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } - reference operator[](size_type pos) { return vch[pos + nReadPos]; } - void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); } - - void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) - { - assert(last - first >= 0); - if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) - { - // special case for inserting at the front when there's room - nReadPos -= (last - first); - memcpy(&vch[nReadPos], &first[0], last - first); - } - else - vch.insert(it, first, last); - } - -#if !defined(_MSC_VER) || _MSC_VER >= 1300 - void insert(iterator it, const char* first, const char* last) - { - assert(last - first >= 0); - if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) - { - // special case for inserting at the front when there's room - nReadPos -= (last - first); - memcpy(&vch[nReadPos], &first[0], last - first); - } - else - vch.insert(it, first, last); - } -#endif - - iterator erase(iterator it) - { - if (it == vch.begin() + nReadPos) - { - // special case for erasing from the front - if (++nReadPos >= vch.size()) - { - // whenever we reach the end, we take the opportunity to clear the buffer - nReadPos = 0; - return vch.erase(vch.begin(), vch.end()); - } - return vch.begin() + nReadPos; - } - else - return vch.erase(it); - } - - iterator erase(iterator first, iterator last) - { - if (first == vch.begin() + nReadPos) - { - // special case for erasing from the front - if (last == vch.end()) - { - nReadPos = 0; - return vch.erase(vch.begin(), vch.end()); - } - else - { - nReadPos = (last - vch.begin()); - return last; - } - } - else - return vch.erase(first, last); - } - - inline void Compact() - { - vch.erase(vch.begin(), vch.begin() + nReadPos); - nReadPos = 0; - } - - bool Rewind(size_type n) - { - // Rewind by n characters if the buffer hasn't been compacted yet - if (n > nReadPos) - return false; - nReadPos -= n; - return true; - } - - - // - // Stream subset - // - bool eof() const { return size() == 0; } - CDataStream* rdbuf() { return this; } - int in_avail() { return size(); } - - void SetType(int n) { nType = n; } - int GetType() { return nType; } - void SetVersion(int n) { nVersion = n; } - int GetVersion() { return nVersion; } - void ReadVersion() { *this >> nVersion; } - void WriteVersion() { *this << nVersion; } - - CDataStream& read(char* pch, size_t nSize) - { - // Read from the beginning of the buffer - unsigned int nReadPosNext = nReadPos + nSize; - if (nReadPosNext >= vch.size()) - { - if (nReadPosNext > vch.size()) - { - throw std::ios_base::failure("CDataStream::read() : end of data"); - } - memcpy(pch, &vch[nReadPos], nSize); - nReadPos = 0; - vch.clear(); - return (*this); - } - memcpy(pch, &vch[nReadPos], nSize); - nReadPos = nReadPosNext; - return (*this); - } - - CDataStream& ignore(int nSize) - { - // Ignore from the beginning of the buffer - assert(nSize >= 0); - unsigned int nReadPosNext = nReadPos + nSize; - if (nReadPosNext >= vch.size()) - { - if (nReadPosNext > vch.size()) - throw std::ios_base::failure("CDataStream::ignore() : end of data"); - nReadPos = 0; - vch.clear(); - return (*this); - } - nReadPos = nReadPosNext; - return (*this); - } - - CDataStream& write(const char* pch, size_t nSize) - { - // Write to the end of the buffer - vch.insert(vch.end(), pch, pch + nSize); - return (*this); - } - - template<typename Stream> - void Serialize(Stream& s, int nType, int nVersion) const - { - // Special case: stream << stream concatenates like stream += stream - if (!vch.empty()) - s.write((char*)&vch[0], vch.size() * sizeof(vch[0])); - } - - template<typename T> - unsigned int GetSerializeSize(const T& obj) - { - // Tells the size of the object if serialized to this stream - return ::GetSerializeSize(obj, nType, nVersion); - } - - template<typename T> - CDataStream& operator<<(const T& obj) - { - // Serialize to this stream - ::Serialize(*this, obj, nType, nVersion); - return (*this); - } - - template<typename T> - CDataStream& operator>>(T& obj) - { - // Unserialize from this stream - ::Unserialize(*this, obj, nType, nVersion); - return (*this); - } - - void GetAndClear(CSerializeData &data) { - data.insert(data.end(), begin(), end()); - clear(); - } -}; - - - - - - - - - - -/** Non-refcounted RAII wrapper for FILE* - * - * Will automatically close the file when it goes out of scope if not null. - * If you're returning the file pointer, return file.release(). - * If you need to close the file early, use file.fclose() instead of fclose(file). - */ -class CAutoFile -{ -private: - // Disallow copies - CAutoFile(const CAutoFile&); - CAutoFile& operator=(const CAutoFile&); - - int nType; - int nVersion; - - FILE* file; - -public: - CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) - { - file = filenew; - nType = nTypeIn; - nVersion = nVersionIn; - } - - ~CAutoFile() - { - fclose(); - } - - void fclose() - { - if (file) { - ::fclose(file); - file = NULL; - } - } - - /** Get wrapped FILE* with transfer of ownership. - * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller - * of this function to clean up the returned FILE*. - */ - FILE* release() { FILE* ret = file; file = NULL; return ret; } - - /** Get wrapped FILE* without transfer of ownership. - * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the - * CAutoFile outlives use of the passed pointer. - */ - FILE* Get() const { return file; } - - /** Return true if the wrapped FILE* is NULL, false otherwise. - */ - bool IsNull() const { return (file == NULL); } - - // - // Stream subset - // - void SetType(int n) { nType = n; } - int GetType() { return nType; } - void SetVersion(int n) { nVersion = n; } - int GetVersion() { return nVersion; } - void ReadVersion() { *this >> nVersion; } - void WriteVersion() { *this << nVersion; } - - CAutoFile& read(char* pch, size_t nSize) - { - if (!file) - throw std::ios_base::failure("CAutoFile::read : file handle is NULL"); - if (fread(pch, 1, nSize, file) != nSize) - throw std::ios_base::failure(feof(file) ? "CAutoFile::read : end of file" : "CAutoFile::read : fread failed"); - return (*this); - } - - CAutoFile& write(const char* pch, size_t nSize) - { - if (!file) - throw std::ios_base::failure("CAutoFile::write : file handle is NULL"); - if (fwrite(pch, 1, nSize, file) != nSize) - throw std::ios_base::failure("CAutoFile::write : write failed"); - return (*this); - } - - template<typename T> - unsigned int GetSerializeSize(const T& obj) - { - // Tells the size of the object if serialized to this stream - return ::GetSerializeSize(obj, nType, nVersion); - } - - template<typename T> - CAutoFile& operator<<(const T& obj) - { - // Serialize to this stream - if (!file) - throw std::ios_base::failure("CAutoFile::operator<< : file handle is NULL"); - ::Serialize(*this, obj, nType, nVersion); - return (*this); - } - - template<typename T> - CAutoFile& operator>>(T& obj) - { - // Unserialize from this stream - if (!file) - throw std::ios_base::failure("CAutoFile::operator>> : file handle is NULL"); - ::Unserialize(*this, obj, nType, nVersion); - return (*this); - } -}; - -/** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to - * deserialize from. It guarantees the ability to rewind a given number of bytes. - * - * Will automatically close the file when it goes out of scope if not null. - * If you need to close the file early, use file.fclose() instead of fclose(file). - */ -class CBufferedFile -{ -private: - // Disallow copies - CBufferedFile(const CBufferedFile&); - CBufferedFile& operator=(const CBufferedFile&); - - int nType; - int nVersion; - - FILE *src; // source file - uint64_t nSrcPos; // how many bytes have been read from source - uint64_t nReadPos; // how many bytes have been read from this - uint64_t nReadLimit; // up to which position we're allowed to read - uint64_t nRewind; // how many bytes we guarantee to rewind - std::vector<char> vchBuf; // the buffer - -protected: - // read data from the source to fill the buffer - bool Fill() { - unsigned int pos = nSrcPos % vchBuf.size(); - unsigned int readNow = vchBuf.size() - pos; - unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind; - if (nAvail < readNow) - readNow = nAvail; - if (readNow == 0) - return false; - size_t read = fread((void*)&vchBuf[pos], 1, readNow, src); - if (read == 0) { - throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill : end of file" : "CBufferedFile::Fill : fread failed"); - } else { - nSrcPos += read; - return true; - } - } - -public: - CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : - nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0) - { - src = fileIn; - nType = nTypeIn; - nVersion = nVersionIn; - } - - ~CBufferedFile() - { - fclose(); - } - - void fclose() - { - if (src) { - ::fclose(src); - src = NULL; - } - } - - // check whether we're at the end of the source file - bool eof() const { - return nReadPos == nSrcPos && feof(src); - } - - // read a number of bytes - CBufferedFile& read(char *pch, size_t nSize) { - if (nSize + nReadPos > nReadLimit) - throw std::ios_base::failure("Read attempted past buffer limit"); - if (nSize + nRewind > vchBuf.size()) - throw std::ios_base::failure("Read larger than buffer size"); - while (nSize > 0) { - if (nReadPos == nSrcPos) - Fill(); - unsigned int pos = nReadPos % vchBuf.size(); - size_t nNow = nSize; - if (nNow + pos > vchBuf.size()) - nNow = vchBuf.size() - pos; - if (nNow + nReadPos > nSrcPos) - nNow = nSrcPos - nReadPos; - memcpy(pch, &vchBuf[pos], nNow); - nReadPos += nNow; - pch += nNow; - nSize -= nNow; - } - return (*this); - } - - // return the current reading position - uint64_t GetPos() { - return nReadPos; - } - - // rewind to a given reading position - bool SetPos(uint64_t nPos) { - nReadPos = nPos; - if (nReadPos + nRewind < nSrcPos) { - nReadPos = nSrcPos - nRewind; - return false; - } else if (nReadPos > nSrcPos) { - nReadPos = nSrcPos; - return false; - } else { - return true; - } - } - - bool Seek(uint64_t nPos) { - long nLongPos = nPos; - if (nPos != (uint64_t)nLongPos) - return false; - if (fseek(src, nLongPos, SEEK_SET)) - return false; - nLongPos = ftell(src); - nSrcPos = nLongPos; - nReadPos = nLongPos; - return true; - } - - // prevent reading beyond a certain position - // no argument removes the limit - bool SetLimit(uint64_t nPos = (uint64_t)(-1)) { - if (nPos < nReadPos) - return false; - nReadLimit = nPos; - return true; - } - - template<typename T> - CBufferedFile& operator>>(T& obj) { - // Unserialize from this stream - ::Unserialize(*this, obj, nType, nVersion); - return (*this); - } - - // search for a given byte in the stream, and remain positioned on it - void FindByte(char ch) { - while (true) { - if (nReadPos == nSrcPos) - Fill(); - if (vchBuf[nReadPos % vchBuf.size()] == ch) - break; - nReadPos++; - } - } -}; - #endif // BITCOIN_SERIALIZE_H diff --git a/src/streams.h b/src/streams.h new file mode 100644 index 0000000000..b07b11eb3d --- /dev/null +++ b/src/streams.h @@ -0,0 +1,571 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2013 The Bitcoin developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_STREAMS_H +#define BITCOIN_STREAMS_H + +#include "allocators.h" +#include "serialize.h" + +#include <algorithm> +#include <assert.h> +#include <ios> +#include <limits> +#include <map> +#include <set> +#include <stdint.h> +#include <string> +#include <string.h> +#include <utility> +#include <vector> + +/** Double ended buffer combining vector and stream-like interfaces. + * + * >> and << read and write unformatted data using the above serialization templates. + * Fills with data in linear time; some stringstream implementations take N^2 time. + */ +class CDataStream +{ +protected: + typedef CSerializeData vector_type; + vector_type vch; + unsigned int nReadPos; +public: + int nType; + int nVersion; + + typedef vector_type::allocator_type allocator_type; + typedef vector_type::size_type size_type; + typedef vector_type::difference_type difference_type; + typedef vector_type::reference reference; + typedef vector_type::const_reference const_reference; + typedef vector_type::value_type value_type; + typedef vector_type::iterator iterator; + typedef vector_type::const_iterator const_iterator; + typedef vector_type::reverse_iterator reverse_iterator; + + explicit CDataStream(int nTypeIn, int nVersionIn) + { + Init(nTypeIn, nVersionIn); + } + + CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) + { + Init(nTypeIn, nVersionIn); + } + +#if !defined(_MSC_VER) || _MSC_VER >= 1300 + CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) + { + Init(nTypeIn, nVersionIn); + } +#endif + + CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) + { + Init(nTypeIn, nVersionIn); + } + + CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) + { + Init(nTypeIn, nVersionIn); + } + + CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) + { + Init(nTypeIn, nVersionIn); + } + + void Init(int nTypeIn, int nVersionIn) + { + nReadPos = 0; + nType = nTypeIn; + nVersion = nVersionIn; + } + + CDataStream& operator+=(const CDataStream& b) + { + vch.insert(vch.end(), b.begin(), b.end()); + return *this; + } + + friend CDataStream operator+(const CDataStream& a, const CDataStream& b) + { + CDataStream ret = a; + ret += b; + return (ret); + } + + std::string str() const + { + return (std::string(begin(), end())); + } + + + // + // Vector subset + // + const_iterator begin() const { return vch.begin() + nReadPos; } + iterator begin() { return vch.begin() + nReadPos; } + const_iterator end() const { return vch.end(); } + iterator end() { return vch.end(); } + size_type size() const { return vch.size() - nReadPos; } + bool empty() const { return vch.size() == nReadPos; } + void resize(size_type n, value_type c=0) { vch.resize(n + nReadPos, c); } + void reserve(size_type n) { vch.reserve(n + nReadPos); } + const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } + reference operator[](size_type pos) { return vch[pos + nReadPos]; } + void clear() { vch.clear(); nReadPos = 0; } + iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); } + + void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) + { + assert(last - first >= 0); + if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) + { + // special case for inserting at the front when there's room + nReadPos -= (last - first); + memcpy(&vch[nReadPos], &first[0], last - first); + } + else + vch.insert(it, first, last); + } + +#if !defined(_MSC_VER) || _MSC_VER >= 1300 + void insert(iterator it, const char* first, const char* last) + { + assert(last - first >= 0); + if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) + { + // special case for inserting at the front when there's room + nReadPos -= (last - first); + memcpy(&vch[nReadPos], &first[0], last - first); + } + else + vch.insert(it, first, last); + } +#endif + + iterator erase(iterator it) + { + if (it == vch.begin() + nReadPos) + { + // special case for erasing from the front + if (++nReadPos >= vch.size()) + { + // whenever we reach the end, we take the opportunity to clear the buffer + nReadPos = 0; + return vch.erase(vch.begin(), vch.end()); + } + return vch.begin() + nReadPos; + } + else + return vch.erase(it); + } + + iterator erase(iterator first, iterator last) + { + if (first == vch.begin() + nReadPos) + { + // special case for erasing from the front + if (last == vch.end()) + { + nReadPos = 0; + return vch.erase(vch.begin(), vch.end()); + } + else + { + nReadPos = (last - vch.begin()); + return last; + } + } + else + return vch.erase(first, last); + } + + inline void Compact() + { + vch.erase(vch.begin(), vch.begin() + nReadPos); + nReadPos = 0; + } + + bool Rewind(size_type n) + { + // Rewind by n characters if the buffer hasn't been compacted yet + if (n > nReadPos) + return false; + nReadPos -= n; + return true; + } + + + // + // Stream subset + // + bool eof() const { return size() == 0; } + CDataStream* rdbuf() { return this; } + int in_avail() { return size(); } + + void SetType(int n) { nType = n; } + int GetType() { return nType; } + void SetVersion(int n) { nVersion = n; } + int GetVersion() { return nVersion; } + void ReadVersion() { *this >> nVersion; } + void WriteVersion() { *this << nVersion; } + + CDataStream& read(char* pch, size_t nSize) + { + // Read from the beginning of the buffer + unsigned int nReadPosNext = nReadPos + nSize; + if (nReadPosNext >= vch.size()) + { + if (nReadPosNext > vch.size()) + { + throw std::ios_base::failure("CDataStream::read() : end of data"); + } + memcpy(pch, &vch[nReadPos], nSize); + nReadPos = 0; + vch.clear(); + return (*this); + } + memcpy(pch, &vch[nReadPos], nSize); + nReadPos = nReadPosNext; + return (*this); + } + + CDataStream& ignore(int nSize) + { + // Ignore from the beginning of the buffer + assert(nSize >= 0); + unsigned int nReadPosNext = nReadPos + nSize; + if (nReadPosNext >= vch.size()) + { + if (nReadPosNext > vch.size()) + throw std::ios_base::failure("CDataStream::ignore() : end of data"); + nReadPos = 0; + vch.clear(); + return (*this); + } + nReadPos = nReadPosNext; + return (*this); + } + + CDataStream& write(const char* pch, size_t nSize) + { + // Write to the end of the buffer + vch.insert(vch.end(), pch, pch + nSize); + return (*this); + } + + template<typename Stream> + void Serialize(Stream& s, int nType, int nVersion) const + { + // Special case: stream << stream concatenates like stream += stream + if (!vch.empty()) + s.write((char*)&vch[0], vch.size() * sizeof(vch[0])); + } + + template<typename T> + unsigned int GetSerializeSize(const T& obj) + { + // Tells the size of the object if serialized to this stream + return ::GetSerializeSize(obj, nType, nVersion); + } + + template<typename T> + CDataStream& operator<<(const T& obj) + { + // Serialize to this stream + ::Serialize(*this, obj, nType, nVersion); + return (*this); + } + + template<typename T> + CDataStream& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj, nType, nVersion); + return (*this); + } + + void GetAndClear(CSerializeData &data) { + data.insert(data.end(), begin(), end()); + clear(); + } +}; + + + + + + + + + + +/** Non-refcounted RAII wrapper for FILE* + * + * Will automatically close the file when it goes out of scope if not null. + * If you're returning the file pointer, return file.release(). + * If you need to close the file early, use file.fclose() instead of fclose(file). + */ +class CAutoFile +{ +private: + // Disallow copies + CAutoFile(const CAutoFile&); + CAutoFile& operator=(const CAutoFile&); + + int nType; + int nVersion; + + FILE* file; + +public: + CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) + { + file = filenew; + nType = nTypeIn; + nVersion = nVersionIn; + } + + ~CAutoFile() + { + fclose(); + } + + void fclose() + { + if (file) { + ::fclose(file); + file = NULL; + } + } + + /** Get wrapped FILE* with transfer of ownership. + * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller + * of this function to clean up the returned FILE*. + */ + FILE* release() { FILE* ret = file; file = NULL; return ret; } + + /** Get wrapped FILE* without transfer of ownership. + * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the + * CAutoFile outlives use of the passed pointer. + */ + FILE* Get() const { return file; } + + /** Return true if the wrapped FILE* is NULL, false otherwise. + */ + bool IsNull() const { return (file == NULL); } + + // + // Stream subset + // + void SetType(int n) { nType = n; } + int GetType() { return nType; } + void SetVersion(int n) { nVersion = n; } + int GetVersion() { return nVersion; } + void ReadVersion() { *this >> nVersion; } + void WriteVersion() { *this << nVersion; } + + CAutoFile& read(char* pch, size_t nSize) + { + if (!file) + throw std::ios_base::failure("CAutoFile::read : file handle is NULL"); + if (fread(pch, 1, nSize, file) != nSize) + throw std::ios_base::failure(feof(file) ? "CAutoFile::read : end of file" : "CAutoFile::read : fread failed"); + return (*this); + } + + CAutoFile& write(const char* pch, size_t nSize) + { + if (!file) + throw std::ios_base::failure("CAutoFile::write : file handle is NULL"); + if (fwrite(pch, 1, nSize, file) != nSize) + throw std::ios_base::failure("CAutoFile::write : write failed"); + return (*this); + } + + template<typename T> + unsigned int GetSerializeSize(const T& obj) + { + // Tells the size of the object if serialized to this stream + return ::GetSerializeSize(obj, nType, nVersion); + } + + template<typename T> + CAutoFile& operator<<(const T& obj) + { + // Serialize to this stream + if (!file) + throw std::ios_base::failure("CAutoFile::operator<< : file handle is NULL"); + ::Serialize(*this, obj, nType, nVersion); + return (*this); + } + + template<typename T> + CAutoFile& operator>>(T& obj) + { + // Unserialize from this stream + if (!file) + throw std::ios_base::failure("CAutoFile::operator>> : file handle is NULL"); + ::Unserialize(*this, obj, nType, nVersion); + return (*this); + } +}; + +/** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to + * deserialize from. It guarantees the ability to rewind a given number of bytes. + * + * Will automatically close the file when it goes out of scope if not null. + * If you need to close the file early, use file.fclose() instead of fclose(file). + */ +class CBufferedFile +{ +private: + // Disallow copies + CBufferedFile(const CBufferedFile&); + CBufferedFile& operator=(const CBufferedFile&); + + int nType; + int nVersion; + + FILE *src; // source file + uint64_t nSrcPos; // how many bytes have been read from source + uint64_t nReadPos; // how many bytes have been read from this + uint64_t nReadLimit; // up to which position we're allowed to read + uint64_t nRewind; // how many bytes we guarantee to rewind + std::vector<char> vchBuf; // the buffer + +protected: + // read data from the source to fill the buffer + bool Fill() { + unsigned int pos = nSrcPos % vchBuf.size(); + unsigned int readNow = vchBuf.size() - pos; + unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind; + if (nAvail < readNow) + readNow = nAvail; + if (readNow == 0) + return false; + size_t read = fread((void*)&vchBuf[pos], 1, readNow, src); + if (read == 0) { + throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill : end of file" : "CBufferedFile::Fill : fread failed"); + } else { + nSrcPos += read; + return true; + } + } + +public: + CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : + nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0) + { + src = fileIn; + nType = nTypeIn; + nVersion = nVersionIn; + } + + ~CBufferedFile() + { + fclose(); + } + + void fclose() + { + if (src) { + ::fclose(src); + src = NULL; + } + } + + // check whether we're at the end of the source file + bool eof() const { + return nReadPos == nSrcPos && feof(src); + } + + // read a number of bytes + CBufferedFile& read(char *pch, size_t nSize) { + if (nSize + nReadPos > nReadLimit) + throw std::ios_base::failure("Read attempted past buffer limit"); + if (nSize + nRewind > vchBuf.size()) + throw std::ios_base::failure("Read larger than buffer size"); + while (nSize > 0) { + if (nReadPos == nSrcPos) + Fill(); + unsigned int pos = nReadPos % vchBuf.size(); + size_t nNow = nSize; + if (nNow + pos > vchBuf.size()) + nNow = vchBuf.size() - pos; + if (nNow + nReadPos > nSrcPos) + nNow = nSrcPos - nReadPos; + memcpy(pch, &vchBuf[pos], nNow); + nReadPos += nNow; + pch += nNow; + nSize -= nNow; + } + return (*this); + } + + // return the current reading position + uint64_t GetPos() { + return nReadPos; + } + + // rewind to a given reading position + bool SetPos(uint64_t nPos) { + nReadPos = nPos; + if (nReadPos + nRewind < nSrcPos) { + nReadPos = nSrcPos - nRewind; + return false; + } else if (nReadPos > nSrcPos) { + nReadPos = nSrcPos; + return false; + } else { + return true; + } + } + + bool Seek(uint64_t nPos) { + long nLongPos = nPos; + if (nPos != (uint64_t)nLongPos) + return false; + if (fseek(src, nLongPos, SEEK_SET)) + return false; + nLongPos = ftell(src); + nSrcPos = nLongPos; + nReadPos = nLongPos; + return true; + } + + // prevent reading beyond a certain position + // no argument removes the limit + bool SetLimit(uint64_t nPos = (uint64_t)(-1)) { + if (nPos < nReadPos) + return false; + nReadLimit = nPos; + return true; + } + + template<typename T> + CBufferedFile& operator>>(T& obj) { + // Unserialize from this stream + ::Unserialize(*this, obj, nType, nVersion); + return (*this); + } + + // search for a given byte in the stream, and remain positioned on it + void FindByte(char ch) { + while (true) { + if (nReadPos == nSrcPos) + Fill(); + if (vchBuf[nReadPos % vchBuf.size()] == ch) + break; + nReadPos++; + } + } +}; + +#endif // BITCOIN_STREAMS_H diff --git a/src/test/alert_tests.cpp b/src/test/alert_tests.cpp index 4af87cf8ef..28610f0d2e 100644 --- a/src/test/alert_tests.cpp +++ b/src/test/alert_tests.cpp @@ -10,6 +10,7 @@ #include "data/alertTests.raw.h" #include "serialize.h" +#include "streams.h" #include "util.h" #include "utilstrencodings.h" #include "version.h" diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 867a7df888..59e95f2fd1 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "serialize.h" +#include "streams.h" #include <stdint.h> diff --git a/src/txdb.cpp b/src/txdb.cpp index cb9f150011..8a73ce961c 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -104,7 +104,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - leveldb::Iterator *pcursor = const_cast<CLevelDBWrapper*>(&db)->NewIterator(); + boost::scoped_ptr<leveldb::Iterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator()); pcursor->SeekToFirst(); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 4522c63617..b0d6b4aefa 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -6,6 +6,7 @@ #include "txmempool.h" #include "core.h" +#include "streams.h" #include "util.h" #include "utilmoneystr.h" #include "version.h" diff --git a/src/txmempool.h b/src/txmempool.h index ad190eea9d..85cf5310ff 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -12,6 +12,8 @@ #include "core.h" #include "sync.h" +class CAutoFile; + inline bool AllowFree(double dPriority) { // Large (in bytes) low-priority (new, small-coin) transactions |