diff options
Diffstat (limited to 'src')
56 files changed, 2189 insertions, 1085 deletions
diff --git a/src/bitcoinrpc.cpp b/src/bitcoinrpc.cpp index 41850d8bb8..a1e39d7506 100644 --- a/src/bitcoinrpc.cpp +++ b/src/bitcoinrpc.cpp @@ -176,7 +176,7 @@ Value help(const Array& params, bool fHelp) Value stop(const Array& params, bool fHelp) { - // Accept the deprecated and ignored 'detach´ boolean argument + // Accept the deprecated and ignored 'detach' boolean argument if (fHelp || params.size() > 1) throw runtime_error( "stop\n" diff --git a/src/init.cpp b/src/init.cpp index 69ca4785c1..7285e7c0d5 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -790,7 +790,7 @@ bool AppInit2() nCoinCacheSize = nTotalCache / 300; // coins in memory require around 300 bytes uiInterface.InitMessage(_("Loading block index...")); - printf("Loading block index...\n"); + nStart = GetTimeMillis(); pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex); @@ -852,7 +852,7 @@ bool AppInit2() // ********************************************************* Step 8: load wallet uiInterface.InitMessage(_("Loading wallet...")); - printf("Loading wallet...\n"); + nStart = GetTimeMillis(); bool fFirstRun = true; pwalletMain = new CWallet("wallet.dat"); @@ -950,7 +950,7 @@ bool AppInit2() // ********************************************************* Step 10: load peers uiInterface.InitMessage(_("Loading addresses...")); - printf("Loading addresses...\n"); + nStart = GetTimeMillis(); { @@ -985,7 +985,6 @@ bool AppInit2() // ********************************************************* Step 12: finished uiInterface.InitMessage(_("Done loading")); - printf("Done loading\n"); if (!strErrors.str().empty()) return InitError(strErrors.str()); diff --git a/src/leveldb/.gitignore b/src/leveldb/.gitignore index f030430565..55ba072e0e 100644 --- a/src/leveldb/.gitignore +++ b/src/leveldb/.gitignore @@ -6,3 +6,7 @@ build_config.mk *.so.* *_test db_bench +Release +Debug +Benchmark +vs2010.* diff --git a/src/leveldb/Makefile b/src/leveldb/Makefile index 14e494f3ce..42c4952fec 100755..100644 --- a/src/leveldb/Makefile +++ b/src/leveldb/Makefile @@ -2,9 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. See the AUTHORS file for names of contributors. -# Inherit some settings from environment variables, if available -INSTALL_PATH ?= $(CURDIR) - #----------------------------------------------- # Uncomment exactly one of the lines labelled (A), (B), and (C) below # to switch between compilation modes. @@ -15,14 +12,16 @@ OPT ?= -O2 -DNDEBUG # (A) Production use (optimized mode) #----------------------------------------------- # detect what platform we're building on -$(shell ./build_detect_platform build_config.mk) +$(shell CC=$(CC) CXX=$(CXX) TARGET_OS=$(TARGET_OS) \ + ./build_detect_platform build_config.mk ./) # this file is generated by the previous line to set build flags and sources include build_config.mk -xCFLAGS = -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) $(CFLAGS) -xCXXFLAGS = -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) $(CXXFLAGS) +CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) +CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -xLDFLAGS = $(PLATFORM_LDFLAGS) $(LDFLAGS) +LDFLAGS += $(PLATFORM_LDFLAGS) +LIBS += $(PLATFORM_LIBS) LIBOBJECTS = $(SOURCES:.cc=.o) MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o) @@ -51,7 +50,7 @@ TESTS = \ version_set_test \ write_batch_test -PROGRAMS = db_bench $(TESTS) +PROGRAMS = db_bench leveldbutil $(TESTS) BENCHMARKS = db_bench_sqlite3 db_bench_tree_db LIBRARY = libleveldb.a @@ -70,7 +69,7 @@ SHARED = $(SHARED1) else # Update db.h if you change these. SHARED_MAJOR = 1 -SHARED_MINOR = 5 +SHARED_MINOR = 9 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) @@ -82,7 +81,7 @@ $(SHARED2): $(SHARED3) endif $(SHARED3): - $(CXX) $(xLDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(xCXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) $(PLATFORM_EXTRALIBS) -o $(SHARED3) + $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3) $(LIBS) endif # PLATFORM_SHARED_EXT @@ -100,74 +99,77 @@ $(LIBRARY): $(LIBOBJECTS) $(AR) -rs $@ $(LIBOBJECTS) db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS) db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) -lsqlite3 + $(CXX) $(LDFLAGS) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS) db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) -lkyotocabinet + $(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) + +leveldbutil: db/leveldb_main.o $(LIBOBJECTS) + $(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS) arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(MEMENVLIBRARY) : $(MEMENVOBJECTS) rm -f $@ $(AR) -rs $@ $(MEMENVOBJECTS) memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) - $(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(xLDFLAGS) $(PLATFORM_EXTRALIBS) + $(CXX) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LIBS) ifeq ($(PLATFORM), IOS) # For iOS, create universal object files to be used on both the simulator and @@ -179,22 +181,22 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBu .cc.o: mkdir -p ios-x86/$(dir $@) - $(SIMULATORROOT)/usr/bin/$(CXX) $(xCXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ + $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ mkdir -p ios-arm/$(dir $@) - $(DEVICEROOT)/usr/bin/$(CXX) $(xCXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ + $(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ lipo ios-x86/$@ ios-arm/$@ -create -output $@ .c.o: mkdir -p ios-x86/$(dir $@) - $(SIMULATORROOT)/usr/bin/$(CC) $(xCFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ + $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ mkdir -p ios-arm/$(dir $@) - $(DEVICEROOT)/usr/bin/$(CC) $(xCFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ + $(DEVICEROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ lipo ios-x86/$@ ios-arm/$@ -create -output $@ else .cc.o: - $(CXX) $(xCXXFLAGS) -c $< -o $@ + $(CXX) $(CXXFLAGS) -c $< -o $@ .c.o: - $(CC) $(xCFLAGS) -c $< -o $@ + $(CC) $(CFLAGS) -c $< -o $@ endif diff --git a/src/leveldb/NEWS b/src/leveldb/NEWS new file mode 100644 index 0000000000..3fd99242d7 --- /dev/null +++ b/src/leveldb/NEWS @@ -0,0 +1,17 @@ +Release 1.2 2011-05-16 +---------------------- + +Fixes for larger databases (tested up to one billion 100-byte entries, +i.e., ~100GB). + +(1) Place hard limit on number of level-0 files. This fixes errors +of the form "too many open files". + +(2) Fixed memtable management. Before the fix, a heavy write burst +could cause unbounded memory usage. + +A fix for a logging bug where the reader would incorrectly complain +about corruption. + +Allow public access to WriteBatch contents so that users can easily +wrap a DB. diff --git a/src/leveldb/README b/src/leveldb/README index 2bf787ef23..3618adeeed 100644 --- a/src/leveldb/README +++ b/src/leveldb/README @@ -1,11 +1,3 @@ -LevelDB is a third party library used for the transaction database. -It is imported into the Bitcoin codebase due to being relatively new -and not widely packaged. - - - ---------------------------------------------------------------------- - leveldb: A key-value store Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) diff --git a/src/leveldb/TODO b/src/leveldb/TODO index 9130b6a9fa..e603c07137 100644 --- a/src/leveldb/TODO +++ b/src/leveldb/TODO @@ -7,6 +7,7 @@ db within [start_key..end_key]? For Chrome, deletion of obsolete object stores, etc. can be done in the background anyway, so probably not that important. +- There have been requests for MultiGet. After a range is completely deleted, what gets rid of the corresponding files if we do no future changes to that range. Make diff --git a/src/leveldb/WINDOWS.md b/src/leveldb/WINDOWS.md new file mode 100644 index 0000000000..5b76c2448f --- /dev/null +++ b/src/leveldb/WINDOWS.md @@ -0,0 +1,39 @@ +# Building LevelDB On Windows + +## Prereqs + +Install the [Windows Software Development Kit version 7.1](http://www.microsoft.com/downloads/dlx/en-us/listdetailsview.aspx?FamilyID=6b6c21d2-2006-4afa-9702-529fa782d63b). + +Download and extract the [Snappy source distribution](http://snappy.googlecode.com/files/snappy-1.0.5.tar.gz) + +1. Open the "Windows SDK 7.1 Command Prompt" : + Start Menu -> "Microsoft Windows SDK v7.1" > "Windows SDK 7.1 Command Prompt" +2. Change the directory to the leveldb project + +## Building the Static lib + +* 32 bit Version + + setenv /x86 + msbuild.exe /p:Configuration=Release /p:Platform=Win32 /p:Snappy=..\snappy-1.0.5 + +* 64 bit Version + + setenv /x64 + msbuild.exe /p:Configuration=Release /p:Platform=x64 /p:Snappy=..\snappy-1.0.5 + + +## Building and Running the Benchmark app + +* 32 bit Version + + setenv /x86 + msbuild.exe /p:Configuration=Benchmark /p:Platform=Win32 /p:Snappy=..\snappy-1.0.5 + Benchmark\leveldb.exe + +* 64 bit Version + + setenv /x64 + msbuild.exe /p:Configuration=Benchmark /p:Platform=x64 /p:Snappy=..\snappy-1.0.5 + x64\Benchmark\leveldb.exe + diff --git a/src/leveldb/build_detect_platform b/src/leveldb/build_detect_platform index 566d1f83a4..609cb51224 100755 --- a/src/leveldb/build_detect_platform +++ b/src/leveldb/build_detect_platform @@ -7,8 +7,11 @@ # CC C Compiler path # CXX C++ Compiler path # PLATFORM_LDFLAGS Linker flags +# PLATFORM_LIBS Libraries flags # PLATFORM_SHARED_EXT Extension for shared libraries # PLATFORM_SHARED_LDFLAGS Flags for building shared library +# This flag is embedded just before the name +# of the shared library without intervening spaces # PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library # PLATFORM_CCFLAGS C compiler flags # PLATFORM_CXXFLAGS C++ compiler flags. Will contain: @@ -23,8 +26,9 @@ # OUTPUT=$1 -if test -z "$OUTPUT"; then - echo "usage: $0 <output-filename>" >&2 +PREFIX=$2 +if test -z "$OUTPUT" || test -z "$PREFIX"; then + echo "usage: $0 <output-filename> <directory_prefix>" >&2 exit 1 fi @@ -50,85 +54,87 @@ CROSS_COMPILE= PLATFORM_CCFLAGS= PLATFORM_CXXFLAGS= PLATFORM_LDFLAGS= -PLATFORM_EXTRALIBS= -PLATFORM_SOURCES= +PLATFORM_LIBS= PLATFORM_SHARED_EXT="so" PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl," PLATFORM_SHARED_CFLAGS="-fPIC" PLATFORM_SHARED_VERSIONED=true -# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp +MEMCMP_FLAG= +if [ "$CXX" = "g++" ]; then + # Use libc's memcmp instead of GCC's memcmp. This results in ~40% + # performance improvement on readrandom under gcc 4.4.3 on Linux/x86. + MEMCMP_FLAG="-fno-builtin-memcmp" +fi + case "$TARGET_OS" in Darwin) PLATFORM=OS_MACOSX - COMMON_FLAGS="-fno-builtin-memcmp -DOS_MACOSX" + COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX" PLATFORM_SHARED_EXT=dylib - PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " - PLATFORM_SOURCES="port/port_posix.cc util/env_posix.cc" + [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd` + PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/" + PORT_FILE=port/port_posix.cc ;; Linux) PLATFORM=OS_LINUX - COMMON_FLAGS="-fno-builtin-memcmp -pthread -DOS_LINUX" + COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX" PLATFORM_LDFLAGS="-pthread" - PLATFORM_SOURCES="port/port_posix.cc util/env_posix.cc" + PORT_FILE=port/port_posix.cc ;; SunOS) PLATFORM=OS_SOLARIS - COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS" - PLATFORM_LDFLAGS="-lpthread -lrt" - PLATFORM_SOURCES="port/port_posix.cc util/env_posix.cc" + COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS" + PLATFORM_LIBS="-lpthread -lrt" + PORT_FILE=port/port_posix.cc ;; FreeBSD) PLATFORM=OS_FREEBSD - COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD" - PLATFORM_LDFLAGS="-lpthread" - PLATFORM_SOURCES="port/port_posix.cc util/env_posix.cc" + COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD" + PLATFORM_LIBS="-lpthread" + PORT_FILE=port/port_posix.cc ;; NetBSD) PLATFORM=OS_NETBSD - COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD" - PLATFORM_LDFLAGS="-lpthread -lgcc_s" - PLATFORM_SOURCES="port/port_posix.cc util/env_posix.cc" + COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD" + PLATFORM_LIBS="-lpthread -lgcc_s" + PORT_FILE=port/port_posix.cc ;; OpenBSD) PLATFORM=OS_OPENBSD - COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD" + COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD" PLATFORM_LDFLAGS="-pthread" - PLATFORM_SOURCES="port/port_posix.cc util/env_posix.cc" + PORT_FILE=port/port_posix.cc ;; DragonFly) PLATFORM=OS_DRAGONFLYBSD - COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD" - PLATFORM_LDFLAGS="-lpthread" - PLATFORM_SOURCES="port/port_posix.cc util/env_posix.cc" + COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD" + PLATFORM_LIBS="-lpthread" + PORT_FILE=port/port_posix.cc ;; OS_ANDROID_CROSSCOMPILE) PLATFORM=OS_ANDROID - COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX" + COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX" PLATFORM_LDFLAGS="" # All pthread features are in the Android C library - PLATFORM_SOURCES="port/port_posix.cc util/env_posix.cc" + PORT_FILE=port/port_posix.cc CROSS_COMPILE=true ;; - OS_WINDOWS_CROSSCOMPILE) - PLATFORM=OS_WINDOWS - COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_WINDOWS -DLEVELDB_PLATFORM_WINDOWS -DBOOST_THREAD_USE_LIB" - PLATFORM_CXXFLAGS="" - PLATFORM_LDFLAGS="" - PLATFORM_SHARED_CFLAGS="" - PLATFORM_SOURCES="port/port_win.cc util/env_boost.cc util/win_logger.cc" - PLATFORM_EXTRALIBS="-lboost_system-mt-s -lboost_filesystem-mt-s -lboost_thread_win32-mt-s" - CROSS_COMPILE=true + HP-UX) + PLATFORM=OS_HPUX + COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX" + PLATFORM_LDFLAGS="-pthread" + PORT_FILE=port/port_posix.cc + # man ld: +h internal_name + PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl," ;; - NATIVE_WINDOWS) + OS_WINDOWS_CROSSCOMPILE | NATIVE_WINDOWS) PLATFORM=OS_WINDOWS - COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_WINDOWS -DLEVELDB_PLATFORM_WINDOWS -DBOOST_THREAD_USE_LIB" - PLATFORM_CXXFLAGS="" - PLATFORM_LDFLAGS="" - PLATFORM_SHARED_CFLAGS="" - PLATFORM_SOURCES="port/port_win.cc util/env_boost.cc util/win_logger.cc" - PLATFORM_EXTRALIBS="-lboost_system-mgw45-mt-s-1_50 -lboost_filesystem-mgw45-mt-s-1_50 -lboost_thread-mgw45-mt-s-1_50 -lboost_chrono-mgw45-mt-s-1_50" + COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_WINDOWS -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1" + PLATFORM_SOURCES="util/env_win.cc" + PLATFORM_LIBS="-lshlwapi" + PORT_FILE=port/port_win.cc CROSS_COMPILE=true - ;; + ;; *) echo "Unknown platform!" >&2 exit 1 @@ -138,16 +144,19 @@ esac # except for the test and benchmark files. By default, find will output a list # of all files matching either rule, so we need to append -print to make the # prune take effect. -DIRS="util db table" +DIRS="$PREFIX/db $PREFIX/util $PREFIX/table" + set -f # temporarily disable globbing so that our patterns aren't expanded PRUNE_TEST="-name *test*.cc -prune" PRUNE_BENCH="-name *_bench.cc -prune" -PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -not -name 'env_*.cc' -not -name '*_logger.cc' -print | sort | tr "\n" " "` +PRUNE_TOOL="-name leveldb_main.cc -prune" +PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "` + set +f # re-enable globbing # The sources consist of the portable files, plus the platform-specific port # file. -echo "SOURCES=$PORTABLE_FILES $PLATFORM_SOURCES" >> $OUTPUT +echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT if [ "$CROSS_COMPILE" = "true" ]; then @@ -155,7 +164,7 @@ if [ "$CROSS_COMPILE" = "true" ]; then true else # If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h. - $CXX $CFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF + $CXX $CXXFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF #include <cstdatomic> int main() {} EOF @@ -167,11 +176,11 @@ EOF fi # Test whether tcmalloc is available - $CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF + $CXX $CXXFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF int main() {} EOF if [ "$?" = 0 ]; then - PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc" + PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc" fi fi @@ -182,9 +191,9 @@ echo "CC=$CC" >> $OUTPUT echo "CXX=$CXX" >> $OUTPUT echo "PLATFORM=$PLATFORM" >> $OUTPUT echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT +echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT -echo "PLATFORM_EXTRALIBS=$PLATFORM_EXTRALIBS" >> $OUTPUT echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT diff --git a/src/leveldb/db/c.cc b/src/leveldb/db/c.cc index 2dde400e77..08ff0ad90a 100644 --- a/src/leveldb/db/c.cc +++ b/src/leveldb/db/c.cc @@ -24,6 +24,8 @@ using leveldb::Env; using leveldb::FileLock; using leveldb::FilterPolicy; using leveldb::Iterator; +using leveldb::kMajorVersion; +using leveldb::kMinorVersion; using leveldb::Logger; using leveldb::NewBloomFilterPolicy; using leveldb::NewLRUCache; @@ -578,4 +580,16 @@ void leveldb_env_destroy(leveldb_env_t* env) { delete env; } +void leveldb_free(void* ptr) { + free(ptr); +} + +int leveldb_major_version() { + return kMajorVersion; +} + +int leveldb_minor_version() { + return kMinorVersion; +} + } // end extern "C" diff --git a/src/leveldb/db/c_test.c b/src/leveldb/db/c_test.c index 9792447157..7cd5ee0207 100644 --- a/src/leveldb/db/c_test.c +++ b/src/leveldb/db/c_test.c @@ -165,6 +165,9 @@ int main(int argc, char** argv) { char* err = NULL; int run = -1; + CheckCondition(leveldb_major_version() >= 1); + CheckCondition(leveldb_minor_version() >= 1); + snprintf(dbname, sizeof(dbname), "%s/leveldb_c_test-%d", GetTempDir(), @@ -204,6 +207,12 @@ int main(int argc, char** argv) { CheckCondition(err != NULL); Free(&err); + StartPhase("leveldb_free"); + db = leveldb_open(options, dbname, &err); + CheckCondition(err != NULL); + leveldb_free(err); + err = NULL; + StartPhase("open"); leveldb_options_set_create_if_missing(options, 1); db = leveldb_open(options, dbname, &err); diff --git a/src/leveldb/db/db_bench.cc b/src/leveldb/db/db_bench.cc index 21d3e25f31..7abdf87587 100644 --- a/src/leveldb/db/db_bench.cc +++ b/src/leveldb/db/db_bench.cc @@ -693,6 +693,7 @@ class Benchmark { options.create_if_missing = !FLAGS_use_existing_db; options.block_cache = cache_; options.write_buffer_size = FLAGS_write_buffer_size; + options.max_open_files = FLAGS_open_files; options.filter_policy = filter_policy_; Status s = DB::Open(options, FLAGS_db, &db_); if (!s.ok()) { diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc index 90c1c811d8..c9de169f29 100644 --- a/src/leveldb/db/db_impl.cc +++ b/src/leveldb/db/db_impl.cc @@ -609,7 +609,11 @@ void DBImpl::BackgroundCall() { assert(bg_compaction_scheduled_); if (!shutting_down_.Acquire_Load()) { Status s = BackgroundCompaction(); - if (!s.ok()) { + if (s.ok()) { + // Success + } else if (shutting_down_.Acquire_Load()) { + // Error most likely due to shutdown; do not wait + } else { // Wait a little bit before retrying background compaction in // case this is an environmental problem and we do not want to // chew up resources for failed compactions for the duration of diff --git a/src/leveldb/db/db_impl.h b/src/leveldb/db/db_impl.h index 8d2bb3405f..bd29dd8055 100644 --- a/src/leveldb/db/db_impl.h +++ b/src/leveldb/db/db_impl.h @@ -13,6 +13,7 @@ #include "leveldb/db.h" #include "leveldb/env.h" #include "port/port.h" +#include "port/thread_annotations.h" namespace leveldb { @@ -71,7 +72,7 @@ class DBImpl : public DB { // Recover the descriptor from persistent storage. May do a significant // amount of work to recover recently logged updates. Any changes to // be made to the descriptor are added to *edit. - Status Recover(VersionEdit* edit); + Status Recover(VersionEdit* edit) EXCLUSIVE_LOCKS_REQUIRED(mutex_); void MaybeIgnoreError(Status* s) const; @@ -80,27 +81,34 @@ class DBImpl : public DB { // Compact the in-memory write buffer to disk. Switches to a new // log-file/memtable and writes a new descriptor iff successful. - Status CompactMemTable(); + Status CompactMemTable() + EXCLUSIVE_LOCKS_REQUIRED(mutex_); Status RecoverLogFile(uint64_t log_number, VersionEdit* edit, - SequenceNumber* max_sequence); + SequenceNumber* max_sequence) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); - Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base); + Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); - Status MakeRoomForWrite(bool force /* compact even if there is room? */); + Status MakeRoomForWrite(bool force /* compact even if there is room? */) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); WriteBatch* BuildBatchGroup(Writer** last_writer); - void MaybeScheduleCompaction(); + void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); static void BGWork(void* db); void BackgroundCall(); - Status BackgroundCompaction(); - void CleanupCompaction(CompactionState* compact); - Status DoCompactionWork(CompactionState* compact); + Status BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void CleanupCompaction(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Status DoCompactionWork(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); Status OpenCompactionOutputFile(CompactionState* compact); Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input); - Status InstallCompactionResults(CompactionState* compact); + Status InstallCompactionResults(CompactionState* compact) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Constant after construction Env* const env_; diff --git a/src/leveldb/db/db_test.cc b/src/leveldb/db/db_test.cc index 3744d0e117..684ea3bdbc 100644 --- a/src/leveldb/db/db_test.cc +++ b/src/leveldb/db/db_test.cc @@ -59,6 +59,12 @@ class SpecialEnv : public EnvWrapper { // Simulate non-writable file system while this pointer is non-NULL port::AtomicPointer non_writable_; + // Force sync of manifest files to fail while this pointer is non-NULL + port::AtomicPointer manifest_sync_error_; + + // Force write to manifest files to fail while this pointer is non-NULL + port::AtomicPointer manifest_write_error_; + bool count_random_reads_; AtomicCounter random_read_counter_; @@ -69,6 +75,8 @@ class SpecialEnv : public EnvWrapper { no_space_.Release_Store(NULL); non_writable_.Release_Store(NULL); count_random_reads_ = false; + manifest_sync_error_.Release_Store(NULL); + manifest_write_error_.Release_Store(NULL); } Status NewWritableFile(const std::string& f, WritableFile** r) { @@ -100,6 +108,30 @@ class SpecialEnv : public EnvWrapper { return base_->Sync(); } }; + class ManifestFile : public WritableFile { + private: + SpecialEnv* env_; + WritableFile* base_; + public: + ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { } + ~ManifestFile() { delete base_; } + Status Append(const Slice& data) { + if (env_->manifest_write_error_.Acquire_Load() != NULL) { + return Status::IOError("simulated writer error"); + } else { + return base_->Append(data); + } + } + Status Close() { return base_->Close(); } + Status Flush() { return base_->Flush(); } + Status Sync() { + if (env_->manifest_sync_error_.Acquire_Load() != NULL) { + return Status::IOError("simulated sync error"); + } else { + return base_->Sync(); + } + } + }; if (non_writable_.Acquire_Load() != NULL) { return Status::IOError("simulated write error"); @@ -109,6 +141,8 @@ class SpecialEnv : public EnvWrapper { if (s.ok()) { if (strstr(f.c_str(), ".sst") != NULL) { *r = new SSTableFile(this, *r); + } else if (strstr(f.c_str(), "MANIFEST") != NULL) { + *r = new ManifestFile(this, *r); } } return s; @@ -1442,6 +1476,12 @@ TEST(DBTest, DBOpen_Options) { db = NULL; } +TEST(DBTest, Locking) { + DB* db2 = NULL; + Status s = DB::Open(CurrentOptions(), dbname_, &db2); + ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db"; +} + // Check that number of files does not grow when we are out of space TEST(DBTest, NoSpace) { Options options = CurrentOptions(); @@ -1486,6 +1526,47 @@ TEST(DBTest, NonWritableFileSystem) { env_->non_writable_.Release_Store(NULL); } +TEST(DBTest, ManifestWriteError) { + // Test for the following problem: + // (a) Compaction produces file F + // (b) Log record containing F is written to MANIFEST file, but Sync() fails + // (c) GC deletes F + // (d) After reopening DB, reads fail since deleted F is named in log record + + // We iterate twice. In the second iteration, everything is the + // same except the log record never makes it to the MANIFEST file. + for (int iter = 0; iter < 2; iter++) { + port::AtomicPointer* error_type = (iter == 0) + ? &env_->manifest_sync_error_ + : &env_->manifest_write_error_; + + // Insert foo=>bar mapping + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + DestroyAndReopen(&options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Memtable compaction (will succeed) + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level + + // Merging compaction (will fail) + error_type->Release_Store(env_); + dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail + ASSERT_EQ("bar", Get("foo")); + + // Recovery: should not lose data + error_type->Release_Store(NULL); + Reopen(&options); + ASSERT_EQ("bar", Get("foo")); + } +} + TEST(DBTest, FilesDeletedAfterCompaction) { ASSERT_OK(Put("foo", "v2")); Compact("a", "z"); diff --git a/src/leveldb/db/leveldb_main.cc b/src/leveldb/db/leveldb_main.cc new file mode 100644 index 0000000000..995d76107a --- /dev/null +++ b/src/leveldb/db/leveldb_main.cc @@ -0,0 +1,238 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include <stdio.h> +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/version_edit.h" +#include "db/write_batch_internal.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" +#include "leveldb/status.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" + +namespace leveldb { + +namespace { + +bool GuessType(const std::string& fname, FileType* type) { + size_t pos = fname.rfind('/'); + std::string basename; + if (pos == std::string::npos) { + basename = fname; + } else { + basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); + } + uint64_t ignored; + return ParseFileName(basename, &ignored, type); +} + +// Notified when log reader encounters corruption. +class CorruptionReporter : public log::Reader::Reporter { + public: + virtual void Corruption(size_t bytes, const Status& status) { + printf("corruption: %d bytes; %s\n", + static_cast<int>(bytes), + status.ToString().c_str()); + } +}; + +// Print contents of a log file. (*func)() is called on every record. +bool PrintLogContents(Env* env, const std::string& fname, + void (*func)(Slice)) { + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + return false; + } + CorruptionReporter reporter; + log::Reader reader(file, &reporter, true, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch)) { + printf("--- offset %llu; ", + static_cast<unsigned long long>(reader.LastRecordOffset())); + (*func)(record); + } + delete file; + return true; +} + +// Called on every item found in a WriteBatch. +class WriteBatchItemPrinter : public WriteBatch::Handler { + public: + uint64_t offset_; + uint64_t sequence_; + + virtual void Put(const Slice& key, const Slice& value) { + printf(" put '%s' '%s'\n", + EscapeString(key).c_str(), + EscapeString(value).c_str()); + } + virtual void Delete(const Slice& key) { + printf(" del '%s'\n", + EscapeString(key).c_str()); + } +}; + + +// Called on every log record (each one of which is a WriteBatch) +// found in a kLogFile. +static void WriteBatchPrinter(Slice record) { + if (record.size() < 12) { + printf("log record length %d is too small\n", + static_cast<int>(record.size())); + return; + } + WriteBatch batch; + WriteBatchInternal::SetContents(&batch, record); + printf("sequence %llu\n", + static_cast<unsigned long long>(WriteBatchInternal::Sequence(&batch))); + WriteBatchItemPrinter batch_item_printer; + Status s = batch.Iterate(&batch_item_printer); + if (!s.ok()) { + printf(" error: %s\n", s.ToString().c_str()); + } +} + +bool DumpLog(Env* env, const std::string& fname) { + return PrintLogContents(env, fname, WriteBatchPrinter); +} + +// Called on every log record (each one of which is a WriteBatch) +// found in a kDescriptorFile. +static void VersionEditPrinter(Slice record) { + VersionEdit edit; + Status s = edit.DecodeFrom(record); + if (!s.ok()) { + printf("%s\n", s.ToString().c_str()); + return; + } + printf("%s", edit.DebugString().c_str()); +} + +bool DumpDescriptor(Env* env, const std::string& fname) { + return PrintLogContents(env, fname, VersionEditPrinter); +} + +bool DumpTable(Env* env, const std::string& fname) { + uint64_t file_size; + RandomAccessFile* file = NULL; + Table* table = NULL; + Status s = env->GetFileSize(fname, &file_size); + if (s.ok()) { + s = env->NewRandomAccessFile(fname, &file); + } + if (s.ok()) { + // We use the default comparator, which may or may not match the + // comparator used in this database. However this should not cause + // problems since we only use Table operations that do not require + // any comparisons. In particular, we do not call Seek or Prev. + s = Table::Open(Options(), file, file_size, &table); + } + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + delete table; + delete file; + return false; + } + + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = table->NewIterator(ro); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey key; + if (!ParseInternalKey(iter->key(), &key)) { + printf("badkey '%s' => '%s'\n", + EscapeString(iter->key()).c_str(), + EscapeString(iter->value()).c_str()); + } else { + char kbuf[20]; + const char* type; + if (key.type == kTypeDeletion) { + type = "del"; + } else if (key.type == kTypeValue) { + type = "val"; + } else { + snprintf(kbuf, sizeof(kbuf), "%d", static_cast<int>(key.type)); + type = kbuf; + } + printf("'%s' @ %8llu : %s => '%s'\n", + EscapeString(key.user_key).c_str(), + static_cast<unsigned long long>(key.sequence), + type, + EscapeString(iter->value()).c_str()); + } + } + s = iter->status(); + if (!s.ok()) { + printf("iterator error: %s\n", s.ToString().c_str()); + } + + delete iter; + delete table; + delete file; + return true; +} + +bool DumpFile(Env* env, const std::string& fname) { + FileType ftype; + if (!GuessType(fname, &ftype)) { + fprintf(stderr, "%s: unknown file type\n", fname.c_str()); + return false; + } + switch (ftype) { + case kLogFile: return DumpLog(env, fname); + case kDescriptorFile: return DumpDescriptor(env, fname); + case kTableFile: return DumpTable(env, fname); + + default: { + fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str()); + break; + } + } + return false; +} + +bool HandleDumpCommand(Env* env, char** files, int num) { + bool ok = true; + for (int i = 0; i < num; i++) { + ok &= DumpFile(env, files[i]); + } + return ok; +} + +} +} // namespace leveldb + +static void Usage() { + fprintf( + stderr, + "Usage: leveldbutil command...\n" + " dump files... -- dump contents of specified files\n" + ); +} + +int main(int argc, char** argv) { + leveldb::Env* env = leveldb::Env::Default(); + bool ok = true; + if (argc < 2) { + Usage(); + ok = false; + } else { + std::string command = argv[1]; + if (command == "dump") { + ok = leveldb::HandleDumpCommand(env, argv+2, argc-2); + } else { + Usage(); + ok = false; + } + } + return (ok ? 0 : 1); +} diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index cf976b437e..7d0a5de2b9 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -786,12 +786,23 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { if (s.ok()) { s = descriptor_file_->Sync(); } + if (!s.ok()) { + Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str()); + if (ManifestContains(record)) { + Log(options_->info_log, + "MANIFEST contains log record despite error; advancing to new " + "version to prevent mismatch between in-memory and logged state"); + s = Status::OK(); + } + } } // If we just created a new descriptor file, install it by writing a // new CURRENT file that points to it. if (s.ok() && !new_manifest_file.empty()) { s = SetCurrentFile(env_, dbname_, manifest_file_number_); + // No need to double-check MANIFEST in case of error since it + // will be discarded below. } mu->Lock(); @@ -865,7 +876,7 @@ Status VersionSet::Recover() { if (edit.has_comparator_ && edit.comparator_ != icmp_.user_comparator()->Name()) { s = Status::InvalidArgument( - edit.comparator_ + "does not match existing comparator ", + edit.comparator_ + " does not match existing comparator ", icmp_.user_comparator()->Name()); } } @@ -1025,6 +1036,31 @@ const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const { return scratch->buffer; } +// Return true iff the manifest contains the specified record. +bool VersionSet::ManifestContains(const std::string& record) const { + std::string fname = DescriptorFileName(dbname_, manifest_file_number_); + Log(options_->info_log, "ManifestContains: checking %s\n", fname.c_str()); + SequentialFile* file = NULL; + Status s = env_->NewSequentialFile(fname, &file); + if (!s.ok()) { + Log(options_->info_log, "ManifestContains: %s\n", s.ToString().c_str()); + return false; + } + log::Reader reader(file, NULL, true/*checksum*/, 0); + Slice r; + std::string scratch; + bool result = false; + while (reader.ReadRecord(&r, &scratch)) { + if (r == Slice(record)) { + result = true; + break; + } + } + delete file; + Log(options_->info_log, "ManifestContains: result = %d\n", result ? 1 : 0); + return result; +} + uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { uint64_t result = 0; for (int level = 0; level < config::kNumLevels; level++) { diff --git a/src/leveldb/db/version_set.h b/src/leveldb/db/version_set.h index 61c4c99a08..9d084fdb7d 100644 --- a/src/leveldb/db/version_set.h +++ b/src/leveldb/db/version_set.h @@ -21,6 +21,7 @@ #include "db/dbformat.h" #include "db/version_edit.h" #include "port/port.h" +#include "port/thread_annotations.h" namespace leveldb { @@ -159,7 +160,8 @@ class VersionSet { // current version. Will release *mu while actually writing to the file. // REQUIRES: *mu is held on entry. // REQUIRES: no other thread concurrently calls LogAndApply() - Status LogAndApply(VersionEdit* edit, port::Mutex* mu); + Status LogAndApply(VersionEdit* edit, port::Mutex* mu) + EXCLUSIVE_LOCKS_REQUIRED(mu); // Recover the last saved descriptor from persistent storage. Status Recover(); @@ -275,6 +277,8 @@ class VersionSet { void AppendVersion(Version* v); + bool ManifestContains(const std::string& record) const; + Env* const env_; const std::string dbname_; const Options* const options_; diff --git a/src/leveldb/doc/bench/db_bench_sqlite3.cc b/src/leveldb/doc/bench/db_bench_sqlite3.cc index 256793a9db..e63aaa8dcc 100644 --- a/src/leveldb/doc/bench/db_bench_sqlite3.cc +++ b/src/leveldb/doc/bench/db_bench_sqlite3.cc @@ -618,7 +618,7 @@ class Benchmark { ErrorCheck(status); // Execute read statement - while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW); + while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {} StepErrorCheck(status); // Reset SQLite statement for another use diff --git a/src/leveldb/doc/index.html b/src/leveldb/doc/index.html index 521d2baf41..3ed0ed9d9e 100644 --- a/src/leveldb/doc/index.html +++ b/src/leveldb/doc/index.html @@ -408,7 +408,7 @@ The optional <code>FilterPolicy</code> mechanism can be used to reduce the number of disk reads substantially. <pre> leveldb::Options options; - options.filter_policy = NewBloomFilter(10); + options.filter_policy = NewBloomFilterPolicy(10); leveldb::DB* db; leveldb::DB::Open(options, "/tmp/testdb", &db); ... use the database ... @@ -420,7 +420,7 @@ The preceding code associates a based filtering policy with the database. Bloom filter based filtering relies on keeping some number of bits of data in memory per key (in this case 10 bits per key since that is the argument we passed -to NewBloomFilter). This filter will reduce the number of unnecessary +to NewBloomFilterPolicy). This filter will reduce the number of unnecessary disk reads needed for <code>Get()</code> calls by a factor of approximately a 100. Increasing the bits per key will lead to a larger reduction at the cost of more memory usage. We recommend that @@ -430,7 +430,7 @@ lot of random reads set a filter policy. If you are using a custom comparator, you should ensure that the filter policy you are using is compatible with your comparator. For example, consider a comparator that ignores trailing spaces when comparing keys. -<code>NewBloomFilter</code> must not be used with such a comparator. +<code>NewBloomFilterPolicy</code> must not be used with such a comparator. Instead, the application should provide a custom filter policy that also ignores trailing spaces. For example: <pre> @@ -438,7 +438,7 @@ also ignores trailing spaces. For example: private: FilterPolicy* builtin_policy_; public: - CustomFilterPolicy() : builtin_policy_(NewBloomFilter(10)) { } + CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) { } ~CustomFilterPolicy() { delete builtin_policy_; } const char* Name() const { return "IgnoreTrailingSpacesFilter"; } diff --git a/src/leveldb/doc/log_format.txt b/src/leveldb/doc/log_format.txt index 3a0414b65a..5228f624de 100644 --- a/src/leveldb/doc/log_format.txt +++ b/src/leveldb/doc/log_format.txt @@ -4,8 +4,8 @@ exception is that the tail of the file may contain a partial block. Each block consists of a sequence of records: block := record* trailer? record := - checksum: uint32 // crc32c of type and data[] - length: uint16 + checksum: uint32 // crc32c of type and data[] ; little-endian + length: uint16 // little-endian type: uint8 // One of FULL, FIRST, MIDDLE, LAST data: uint8[length] diff --git a/src/leveldb/doc/table_format.txt b/src/leveldb/doc/table_format.txt index d0f3065ed0..ca8f9b4460 100644 --- a/src/leveldb/doc/table_format.txt +++ b/src/leveldb/doc/table_format.txt @@ -18,6 +18,8 @@ The file contains internal pointers. Each such pointer is called a BlockHandle and contains the following information: offset: varint64 size: varint64 +See https://developers.google.com/protocol-buffers/docs/encoding#varints +for an explanation of varint64 format. (1) The sequence of key/value pairs in the file are stored in sorted order and partitioned into a sequence of data blocks. These blocks @@ -41,11 +43,11 @@ BlockHandle for the data block. (6) At the very end of the file is a fixed length footer that contains the BlockHandle of the metaindex and index blocks as well as a magic number. - metaindex_handle: char[p]; // Block handle for metaindex - index_handle: char[q]; // Block handle for index - padding: char[40-p-q]; // 0 bytes to make fixed length - // (40==2*BlockHandle::kMaxEncodedLength) - magic: fixed64; // == 0xdb4775248b80fb57 + metaindex_handle: char[p]; // Block handle for metaindex + index_handle: char[q]; // Block handle for index + padding: char[40-p-q]; // zeroed bytes to make fixed length + // (40==2*BlockHandle::kMaxEncodedLength) + magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) "filter" Meta Block ------------------- diff --git a/src/leveldb/helpers/memenv/memenv.cc b/src/leveldb/helpers/memenv/memenv.cc index 2082083b3f..5879de1214 100644 --- a/src/leveldb/helpers/memenv/memenv.cc +++ b/src/leveldb/helpers/memenv/memenv.cc @@ -221,6 +221,11 @@ class WritableFileImpl : public WritableFile { FileState* file_; }; +class NoOpLogger : public Logger { + public: + virtual void Logv(const char* format, va_list ap) { } +}; + class InMemoryEnv : public EnvWrapper { public: explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { } @@ -358,6 +363,11 @@ class InMemoryEnv : public EnvWrapper { return Status::OK(); } + virtual Status NewLogger(const std::string& fname, Logger** result) { + *result = new NoOpLogger; + return Status::OK(); + } + private: // Map from filenames to FileState objects, representing a simple file system. typedef std::map<std::string, FileState*> FileSystem; diff --git a/src/leveldb/include/leveldb/c.h b/src/leveldb/include/leveldb/c.h index 70e3cc6528..1fa58866c3 100644 --- a/src/leveldb/include/leveldb/c.h +++ b/src/leveldb/include/leveldb/c.h @@ -28,6 +28,7 @@ be true on entry: *errptr == NULL *errptr points to a malloc()ed null-terminated error message + (On Windows, *errptr must have been malloc()-ed by this library.) On success, a leveldb routine leaves *errptr unchanged. On failure, leveldb frees the old value of *errptr and set *errptr to a malloc()ed error message. @@ -268,6 +269,21 @@ extern void leveldb_cache_destroy(leveldb_cache_t* cache); extern leveldb_env_t* leveldb_create_default_env(); extern void leveldb_env_destroy(leveldb_env_t*); +/* Utility */ + +/* Calls free(ptr). + REQUIRES: ptr was malloc()-ed and returned by one of the routines + in this file. Note that in certain cases (typically on Windows), you + may need to call this routine instead of free(ptr) to dispose of + malloc()-ed memory returned by this library. */ +extern void leveldb_free(void* ptr); + +/* Return the major version number for this release. */ +extern int leveldb_major_version(); + +/* Return the minor version number for this release. */ +extern int leveldb_minor_version(); + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/src/leveldb/include/leveldb/db.h b/src/leveldb/include/leveldb/db.h index ed56b87c38..29d3674479 100644 --- a/src/leveldb/include/leveldb/db.h +++ b/src/leveldb/include/leveldb/db.h @@ -14,7 +14,7 @@ namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 5; +static const int kMinorVersion = 9; struct Options; struct ReadOptions; diff --git a/src/leveldb/include/leveldb/env.h b/src/leveldb/include/leveldb/env.h index 2720667185..fa32289f58 100644 --- a/src/leveldb/include/leveldb/env.h +++ b/src/leveldb/include/leveldb/env.h @@ -175,6 +175,11 @@ class SequentialFile { // // REQUIRES: External synchronization virtual Status Skip(uint64_t n) = 0; + + private: + // No copying allowed + SequentialFile(const SequentialFile&); + void operator=(const SequentialFile&); }; // A file abstraction for randomly reading the contents of a file. @@ -194,6 +199,11 @@ class RandomAccessFile { // Safe for concurrent use by multiple threads. virtual Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const = 0; + + private: + // No copying allowed + RandomAccessFile(const RandomAccessFile&); + void operator=(const RandomAccessFile&); }; // A file abstraction for sequential writing. The implementation diff --git a/src/leveldb/port/atomic_pointer.h b/src/leveldb/port/atomic_pointer.h index c58bffbf1b..e17bf435ea 100644 --- a/src/leveldb/port/atomic_pointer.h +++ b/src/leveldb/port/atomic_pointer.h @@ -36,6 +36,8 @@ #define ARCH_CPU_X86_FAMILY 1 #elif defined(__ARMEL__) #define ARCH_CPU_ARM_FAMILY 1 +#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__) +#define ARCH_CPU_PPC_FAMILY 1 #endif namespace leveldb { @@ -91,6 +93,15 @@ inline void MemoryBarrier() { } #define LEVELDB_HAVE_MEMORY_BARRIER +// PPC +#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__) +inline void MemoryBarrier() { + // TODO for some powerpc expert: is there a cheaper suitable variant? + // Perhaps by having separate barriers for acquire and release ops. + asm volatile("sync" : : : "memory"); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + #endif // AtomicPointer built using platform-specific MemoryBarrier() @@ -136,6 +147,66 @@ class AtomicPointer { } }; +// Atomic pointer based on sparc memory barriers +#elif defined(__sparcv9) && defined(__GNUC__) +class AtomicPointer { + private: + void* rep_; + public: + AtomicPointer() { } + explicit AtomicPointer(void* v) : rep_(v) { } + inline void* Acquire_Load() const { + void* val; + __asm__ __volatile__ ( + "ldx [%[rep_]], %[val] \n\t" + "membar #LoadLoad|#LoadStore \n\t" + : [val] "=r" (val) + : [rep_] "r" (&rep_) + : "memory"); + return val; + } + inline void Release_Store(void* v) { + __asm__ __volatile__ ( + "membar #LoadStore|#StoreStore \n\t" + "stx %[v], [%[rep_]] \n\t" + : + : [rep_] "r" (&rep_), [v] "r" (v) + : "memory"); + } + inline void* NoBarrier_Load() const { return rep_; } + inline void NoBarrier_Store(void* v) { rep_ = v; } +}; + +// Atomic pointer based on ia64 acq/rel +#elif defined(__ia64) && defined(__GNUC__) +class AtomicPointer { + private: + void* rep_; + public: + AtomicPointer() { } + explicit AtomicPointer(void* v) : rep_(v) { } + inline void* Acquire_Load() const { + void* val ; + __asm__ __volatile__ ( + "ld8.acq %[val] = [%[rep_]] \n\t" + : [val] "=r" (val) + : [rep_] "r" (&rep_) + : "memory" + ); + return val; + } + inline void Release_Store(void* v) { + __asm__ __volatile__ ( + "st8.rel [%[rep_]] = %[v] \n\t" + : + : [rep_] "r" (&rep_), [v] "r" (v) + : "memory" + ); + } + inline void* NoBarrier_Load() const { return rep_; } + inline void NoBarrier_Store(void* v) { rep_ = v; } +}; + // We have neither MemoryBarrier(), nor <cstdatomic> #else #error Please implement AtomicPointer for this platform. @@ -145,6 +216,7 @@ class AtomicPointer { #undef LEVELDB_HAVE_MEMORY_BARRIER #undef ARCH_CPU_X86_FAMILY #undef ARCH_CPU_ARM_FAMILY +#undef ARCH_CPU_PPC_FAMILY } // namespace port } // namespace leveldb diff --git a/src/leveldb/port/port_posix.h b/src/leveldb/port/port_posix.h index 6ca352e2ca..f2b89bffb9 100644 --- a/src/leveldb/port/port_posix.h +++ b/src/leveldb/port/port_posix.h @@ -26,11 +26,17 @@ #include <sys/endian.h> #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\ - defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID) + defined(OS_DRAGONFLYBSD) #include <sys/types.h> #include <sys/endian.h> #elif defined(OS_HPUX) #define PLATFORM_IS_LITTLE_ENDIAN false +#elif defined(OS_ANDROID) + // Due to a bug in the NDK x86 <sys/endian.h> definition, + // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android. + // See http://code.google.com/p/android/issues/detail?id=39824 + #include <endian.h> + #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #else #include <endian.h> #endif diff --git a/src/leveldb/port/port_win.cc b/src/leveldb/port/port_win.cc index 786cd6018a..99c1d8e346 100644 --- a/src/leveldb/port/port_win.cc +++ b/src/leveldb/port/port_win.cc @@ -37,102 +37,96 @@ namespace leveldb { namespace port { Mutex::Mutex() : - mutex_(::CreateMutex(NULL, FALSE, NULL)) { - assert(mutex_); + cs_(NULL) { + assert(!cs_); + cs_ = static_cast<void *>(new CRITICAL_SECTION()); + ::InitializeCriticalSection(static_cast<CRITICAL_SECTION *>(cs_)); + assert(cs_); } Mutex::~Mutex() { - assert(mutex_); - ::CloseHandle(mutex_); + assert(cs_); + ::DeleteCriticalSection(static_cast<CRITICAL_SECTION *>(cs_)); + delete static_cast<CRITICAL_SECTION *>(cs_); + cs_ = NULL; + assert(!cs_); } void Mutex::Lock() { - assert(mutex_); - ::WaitForSingleObject(mutex_, INFINITE); + assert(cs_); + ::EnterCriticalSection(static_cast<CRITICAL_SECTION *>(cs_)); } void Mutex::Unlock() { - assert(mutex_); - ::ReleaseMutex(mutex_); + assert(cs_); + ::LeaveCriticalSection(static_cast<CRITICAL_SECTION *>(cs_)); } void Mutex::AssertHeld() { - assert(mutex_); + assert(cs_); assert(1); } CondVar::CondVar(Mutex* mu) : waiting_(0), mu_(mu), - sema_(::CreateSemaphore(NULL, 0, 0x7fffffff, NULL)), - event_(::CreateEvent(NULL, FALSE, FALSE, NULL)), - broadcasted_(false){ + sem1_(::CreateSemaphore(NULL, 0, 10000, NULL)), + sem2_(::CreateSemaphore(NULL, 0, 10000, NULL)) { assert(mu_); } CondVar::~CondVar() { - ::CloseHandle(sema_); - ::CloseHandle(event_); + ::CloseHandle(sem1_); + ::CloseHandle(sem2_); } void CondVar::Wait() { + mu_->AssertHeld(); + wait_mtx_.Lock(); ++waiting_; - assert(waiting_ > 0); wait_mtx_.Unlock(); - ::SignalObjectAndWait(mu_->mutex_, sema_, INFINITE, FALSE); - - wait_mtx_.Lock(); - bool last = broadcasted_ && (--waiting_ == 0); - assert(waiting_ >= 0); - wait_mtx_.Unlock(); + mu_->Unlock(); - // we leave this function with the mutex held - if (last) - { - ::SignalObjectAndWait(event_, mu_->mutex_, INFINITE, FALSE); - } - else - { - ::WaitForSingleObject(mu_->mutex_, INFINITE); - } + // initiate handshake + ::WaitForSingleObject(sem1_, INFINITE); + ::ReleaseSemaphore(sem2_, 1, NULL); + mu_->Lock(); } void CondVar::Signal() { wait_mtx_.Lock(); - bool waiters = waiting_ > 0; - wait_mtx_.Unlock(); + if (waiting_ > 0) { + --waiting_; - if (waiters) - { - ::ReleaseSemaphore(sema_, 1, 0); + // finalize handshake + ::ReleaseSemaphore(sem1_, 1, NULL); + ::WaitForSingleObject(sem2_, INFINITE); } + wait_mtx_.Unlock(); } void CondVar::SignalAll() { wait_mtx_.Lock(); - - broadcasted_ = (waiting_ > 0); - - if (broadcasted_) - { - // release all - ::ReleaseSemaphore(sema_, waiting_, 0); - wait_mtx_.Unlock(); - ::WaitForSingleObject(event_, INFINITE); - broadcasted_ = false; - } - else - { - wait_mtx_.Unlock(); + for(long i = 0; i < waiting_; ++i) { + ::ReleaseSemaphore(sem1_, 1, NULL); + while(waiting_ > 0) { + --waiting_; + ::WaitForSingleObject(sem2_, INFINITE); + } } + wait_mtx_.Unlock(); } AtomicPointer::AtomicPointer(void* v) { Release_Store(v); } +void InitOnce(OnceType* once, void (*initializer)()) { + once->InitOnce(initializer); +} + void* AtomicPointer::Acquire_Load() const { void * p = NULL; InterlockedExchangePointer(&p, rep_); @@ -151,32 +145,5 @@ void AtomicPointer::NoBarrier_Store(void* v) { rep_ = v; } -enum InitializationState -{ - Uninitialized = 0, - Running = 1, - Initialized = 2 -}; - -void InitOnce(OnceType* once, void (*initializer)()) { - - assert(Uninitialized == LEVELDB_ONCE_INIT); - - InitializationState state = static_cast<InitializationState>(InterlockedCompareExchange(once, Running, Uninitialized)); - - if (state == Uninitialized) { - initializer(); - *once = Initialized; - } - - if (state == Running) { - while(*once != Initialized) { - Sleep(0); // yield - } - } - - assert(*once == Initialized); -} - } } diff --git a/src/leveldb/port/port_win.h b/src/leveldb/port/port_win.h index 893919998c..45bf2f0ea7 100644 --- a/src/leveldb/port/port_win.h +++ b/src/leveldb/port/port_win.h @@ -37,14 +37,11 @@ #define fread_unlocked _fread_nolock #endif - -#ifdef SNAPPY -#include <snappy/snappy.h> -#endif - #include <string> - #include <stdint.h> +#ifdef SNAPPY +#include <snappy.h> +#endif namespace leveldb { namespace port { @@ -67,9 +64,8 @@ class Mutex { friend class CondVar; // critical sections are more efficient than mutexes // but they are not recursive and can only be used to synchronize threads within the same process - // additionnaly they cannot be used with SignalObjectAndWait that we use for CondVar // we use opaque void * to avoid including windows.h in port_win.h - void * mutex_; + void * cs_; // No copying Mutex(const Mutex&); @@ -79,7 +75,7 @@ class Mutex { // the Win32 API offers a dependable condition variable mechanism, but only starting with // Windows 2008 and Vista // no matter what we will implement our own condition variable with a semaphore -// implementation as described in a paper written by Douglas C. Schmidt and Irfan Pyarali +// implementation as described in a paper written by Andrew D. Birrell in 2003 class CondVar { public: explicit CondVar(Mutex* mu); @@ -93,12 +89,34 @@ class CondVar { Mutex wait_mtx_; long waiting_; - void * sema_; - void * event_; + void * sem1_; + void * sem2_; + + +}; - bool broadcasted_; +class OnceType { +public: +// OnceType() : init_(false) {} + OnceType(const OnceType &once) : init_(once.init_) {} + OnceType(bool f) : init_(f) {} + void InitOnce(void (*initializer)()) { + mutex_.Lock(); + if (!init_) { + init_ = true; + initializer(); + } + mutex_.Unlock(); + } + +private: + bool init_; + Mutex mutex_; }; +#define LEVELDB_ONCE_INIT false +extern void InitOnce(port::OnceType*, void (*initializer)()); + // Storage for a lock-free pointer class AtomicPointer { private: @@ -115,11 +133,6 @@ class AtomicPointer { void NoBarrier_Store(void* v); }; -typedef volatile long OnceType; -#define LEVELDB_ONCE_INIT (0) - -extern void InitOnce(OnceType* once, void (*initializer)()); - inline bool Snappy_Compress(const char* input, size_t length, ::std::string* output) { #ifdef SNAPPY diff --git a/src/leveldb/port/thread_annotations.h b/src/leveldb/port/thread_annotations.h new file mode 100644 index 0000000000..6f9b6a7924 --- /dev/null +++ b/src/leveldb/port/thread_annotations.h @@ -0,0 +1,59 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H + +// Some environments provide custom macros to aid in static thread-safety +// analysis. Provide empty definitions of such macros unless they are already +// defined. + +#ifndef EXCLUSIVE_LOCKS_REQUIRED +#define EXCLUSIVE_LOCKS_REQUIRED(...) +#endif + +#ifndef SHARED_LOCKS_REQUIRED +#define SHARED_LOCKS_REQUIRED(...) +#endif + +#ifndef LOCKS_EXCLUDED +#define LOCKS_EXCLUDED(...) +#endif + +#ifndef LOCK_RETURNED +#define LOCK_RETURNED(x) +#endif + +#ifndef LOCKABLE +#define LOCKABLE +#endif + +#ifndef SCOPED_LOCKABLE +#define SCOPED_LOCKABLE +#endif + +#ifndef EXCLUSIVE_LOCK_FUNCTION +#define EXCLUSIVE_LOCK_FUNCTION(...) +#endif + +#ifndef SHARED_LOCK_FUNCTION +#define SHARED_LOCK_FUNCTION(...) +#endif + +#ifndef EXCLUSIVE_TRYLOCK_FUNCTION +#define EXCLUSIVE_TRYLOCK_FUNCTION(...) +#endif + +#ifndef SHARED_TRYLOCK_FUNCTION +#define SHARED_TRYLOCK_FUNCTION(...) +#endif + +#ifndef UNLOCK_FUNCTION +#define UNLOCK_FUNCTION(...) +#endif + +#ifndef NO_THREAD_SAFETY_ANALYSIS +#define NO_THREAD_SAFETY_ANALYSIS +#endif + +#endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H diff --git a/src/leveldb/table/block.cc b/src/leveldb/table/block.cc index 199d453773..ab83c1112c 100644 --- a/src/leveldb/table/block.cc +++ b/src/leveldb/table/block.cc @@ -162,8 +162,8 @@ class Block::Iter : public Iterator { } virtual void Seek(const Slice& target) { - // Binary search in restart array to find the first restart point - // with a key >= target + // Binary search in restart array to find the last restart point + // with a key < target uint32_t left = 0; uint32_t right = num_restarts_ - 1; while (left < right) { diff --git a/src/leveldb/util/bloom_test.cc b/src/leveldb/util/bloom_test.cc index 4a6ea1b7c8..0bf8e8d6eb 100644 --- a/src/leveldb/util/bloom_test.cc +++ b/src/leveldb/util/bloom_test.cc @@ -4,6 +4,7 @@ #include "leveldb/filter_policy.h" +#include "util/coding.h" #include "util/logging.h" #include "util/testharness.h" #include "util/testutil.h" @@ -13,8 +14,8 @@ namespace leveldb { static const int kVerbose = 1; static Slice Key(int i, char* buffer) { - memcpy(buffer, &i, sizeof(i)); - return Slice(buffer, sizeof(i)); + EncodeFixed32(buffer, i); + return Slice(buffer, sizeof(uint32_t)); } class BloomTest { diff --git a/src/leveldb/util/coding.cc b/src/leveldb/util/coding.cc index dbd7a6545c..21e3186d5d 100644 --- a/src/leveldb/util/coding.cc +++ b/src/leveldb/util/coding.cc @@ -7,29 +7,29 @@ namespace leveldb { void EncodeFixed32(char* buf, uint32_t value) { -#if __BYTE_ORDER == __LITTLE_ENDIAN - memcpy(buf, &value, sizeof(value)); -#else - buf[0] = value & 0xff; - buf[1] = (value >> 8) & 0xff; - buf[2] = (value >> 16) & 0xff; - buf[3] = (value >> 24) & 0xff; -#endif + if (port::kLittleEndian) { + memcpy(buf, &value, sizeof(value)); + } else { + buf[0] = value & 0xff; + buf[1] = (value >> 8) & 0xff; + buf[2] = (value >> 16) & 0xff; + buf[3] = (value >> 24) & 0xff; + } } void EncodeFixed64(char* buf, uint64_t value) { -#if __BYTE_ORDER == __LITTLE_ENDIAN - memcpy(buf, &value, sizeof(value)); -#else - buf[0] = value & 0xff; - buf[1] = (value >> 8) & 0xff; - buf[2] = (value >> 16) & 0xff; - buf[3] = (value >> 24) & 0xff; - buf[4] = (value >> 32) & 0xff; - buf[5] = (value >> 40) & 0xff; - buf[6] = (value >> 48) & 0xff; - buf[7] = (value >> 56) & 0xff; -#endif + if (port::kLittleEndian) { + memcpy(buf, &value, sizeof(value)); + } else { + buf[0] = value & 0xff; + buf[1] = (value >> 8) & 0xff; + buf[2] = (value >> 16) & 0xff; + buf[3] = (value >> 24) & 0xff; + buf[4] = (value >> 32) & 0xff; + buf[5] = (value >> 40) & 0xff; + buf[6] = (value >> 48) & 0xff; + buf[7] = (value >> 56) & 0xff; + } } void PutFixed32(std::string* dst, uint32_t value) { diff --git a/src/leveldb/util/env_boost.cc b/src/leveldb/util/env_boost.cc deleted file mode 100644 index 055c657438..0000000000 --- a/src/leveldb/util/env_boost.cc +++ /dev/null @@ -1,591 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include <deque> - -#ifdef LEVELDB_PLATFORM_WINDOWS -#include <windows.h> -#include <errno.h> -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <time.h> -#include <io.h> -#else -#include <dirent.h> -#include <errno.h> -#include <fcntl.h> -#include <pthread.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/mman.h> -#include <sys/stat.h> -#include <sys/time.h> -#include <sys/types.h> -#include <sys/param.h> -#include <time.h> -#include <unistd.h> -#endif -#if defined(LEVELDB_PLATFORM_ANDROID) -#include <sys/stat.h> -#endif -#include "leveldb/env.h" -#include "leveldb/slice.h" - -#ifdef LEVELDB_PLATFORM_WINDOWS -#include "util/win_logger.h" -#else -#include "util/posix_logger.h" -#endif -#include "port/port.h" -#include "util/logging.h" - -#ifdef __linux -#include <sys/sysinfo.h> -#include <linux/unistd.h> -#endif - -#include <fstream> - -// Boost includes - see WINDOWS file to see which modules to install -#include <boost/date_time/gregorian/gregorian.hpp> -#include <boost/date_time/posix_time/posix_time.hpp> -#include <boost/filesystem/convenience.hpp> -#include <boost/thread/once.hpp> -#include <boost/thread/thread.hpp> -#include <boost/bind.hpp> -#include <boost/scoped_ptr.hpp> -#include <boost/interprocess/sync/file_lock.hpp> -#include <boost/thread/condition_variable.hpp> - -namespace leveldb { -namespace { - -// returns the ID of the current process -static boost::uint32_t current_process_id(void) { -#ifdef _WIN32 - return static_cast<boost::uint32_t>(::GetCurrentProcessId()); -#else - return static_cast<boost::uint32_t>(::getpid()); -#endif -} - -// returns the ID of the current thread -static boost::uint32_t current_thread_id(void) { -#ifdef _WIN32 - return static_cast<boost::uint32_t>(::GetCurrentThreadId()); -#else -#ifdef __linux - return static_cast<boost::uint32_t>(::syscall(__NR_gettid)); -#else - // just return the pid - return current_process_id(); -#endif -#endif -} - -static char global_read_only_buf[0x8000]; - -class PosixSequentialFile: public SequentialFile { - private: - std::string filename_; - FILE* file_; - - public: - PosixSequentialFile(const std::string& fname, FILE* f) - : filename_(fname), file_(f) { } - virtual ~PosixSequentialFile() { fclose(file_); } - - virtual Status Read(size_t n, Slice* result, char* scratch) { - Status s; -#if defined(BSD) || defined(__MINGW32__) - // fread_unlocked doesn't exist on FreeBSD or MingW - size_t r = fread(scratch, 1, n, file_); -#else - size_t r = fread_unlocked(scratch, 1, n, file_); -#endif - *result = Slice(scratch, r); - if (r < n) { - if (feof(file_)) { - // We leave status as ok if we hit the end of the file - } else { - // A partial read with an error: return a non-ok status - s = Status::IOError(filename_, strerror(errno)); - } - } - return s; - } - - virtual Status Skip(uint64_t n) { - if (fseek(file_, n, SEEK_CUR)) { - return Status::IOError(filename_, strerror(errno)); - } - return Status::OK(); - } -}; - -class PosixRandomAccessFile: public RandomAccessFile { - private: - std::string filename_; - int fd_; - mutable boost::mutex mu_; - - public: - PosixRandomAccessFile(const std::string& fname, int fd) - : filename_(fname), fd_(fd) { } - virtual ~PosixRandomAccessFile() { close(fd_); } - - virtual Status Read(uint64_t offset, size_t n, Slice* result, - char* scratch) const { - Status s; -#ifdef LEVELDB_PLATFORM_WINDOWS - // no pread on Windows so we emulate it with a mutex - boost::unique_lock<boost::mutex> lock(mu_); - - if (::_lseeki64(fd_, offset, SEEK_SET) == -1L) { - return Status::IOError(filename_, strerror(errno)); - } - - int r = ::_read(fd_, scratch, n); - *result = Slice(scratch, (r < 0) ? 0 : r); - lock.unlock(); -#else - ssize_t r = pread(fd_, scratch, n, static_cast<off_t>(offset)); - *result = Slice(scratch, (r < 0) ? 0 : r); -#endif - if (r < 0) { - // An error: return a non-ok status - s = Status::IOError(filename_, strerror(errno)); - } - return s; - } -}; - -// We preallocate up to an extra megabyte and use memcpy to append new -// data to the file. This is safe since we either properly close the -// file before reading from it, or for log files, the reading code -// knows enough to skip zero suffixes. - -class BoostFile : public WritableFile { - -public: - explicit BoostFile(std::string path) : path_(path), written_(0) { - Open(); - } - - virtual ~BoostFile() { - Close(); - } - -private: - void Open() { - // we truncate the file as implemented in env_posix - file_.open(path_.generic_string().c_str(), - std::ios_base::trunc | std::ios_base::out | std::ios_base::binary); - written_ = 0; - } - -public: - virtual Status Append(const Slice& data) { - Status result; - file_.write(data.data(), data.size()); - if (!file_.good()) { - result = Status::IOError( - path_.generic_string() + " Append", "cannot write"); - } - return result; - } - - virtual Status Close() { - Status result; - - try { - if (file_.is_open()) { - Sync(); - file_.close(); - } - } catch (const std::exception & e) { - result = Status::IOError(path_.generic_string() + " close", e.what()); - } - - return result; - } - - virtual Status Flush() { - file_.flush(); - return Status::OK(); - } - - virtual Status Sync() { - Status result; - try { - Flush(); - } catch (const std::exception & e) { - result = Status::IOError(path_.string() + " sync", e.what()); - } - - return result; - } - -private: - boost::filesystem::path path_; - boost::uint64_t written_; - std::ofstream file_; -}; - - - -class BoostFileLock : public FileLock { - public: - boost::interprocess::file_lock fl_; -}; - -class PosixEnv : public Env { - public: - PosixEnv(); - virtual ~PosixEnv() { - fprintf(stderr, "Destroying Env::Default()\n"); - exit(1); - } - - virtual Status NewSequentialFile(const std::string& fname, - SequentialFile** result) { - FILE* f = fopen(fname.c_str(), "rb"); - if (f == NULL) { - *result = NULL; - return Status::IOError(fname, strerror(errno)); - } else { - *result = new PosixSequentialFile(fname, f); - return Status::OK(); - } - } - - virtual Status NewRandomAccessFile(const std::string& fname, - RandomAccessFile** result) { -#ifdef LEVELDB_PLATFORM_WINDOWS - int fd = _open(fname.c_str(), _O_RDONLY | _O_RANDOM | _O_BINARY); -#else - int fd = open(fname.c_str(), O_RDONLY); -#endif - if (fd < 0) { - *result = NULL; - return Status::IOError(fname, strerror(errno)); - } - *result = new PosixRandomAccessFile(fname, fd); - return Status::OK(); - } - - virtual Status NewWritableFile(const std::string& fname, - WritableFile** result) { - Status s; - try { - // will create a new empty file to write to - *result = new BoostFile(fname); - } - catch (const std::exception & e) { - s = Status::IOError(fname, e.what()); - } - - return s; - } - - virtual bool FileExists(const std::string& fname) { - return boost::filesystem::exists(fname); - } - - virtual Status GetChildren(const std::string& dir, - std::vector<std::string>* result) { - result->clear(); - - boost::system::error_code ec; - boost::filesystem::directory_iterator current(dir, ec); - if (ec != 0) { - return Status::IOError(dir, ec.message()); - } - - boost::filesystem::directory_iterator end; - - for(; current != end; ++current) { - result->push_back(current->path().filename().generic_string()); - } - - return Status::OK(); - } - - virtual Status DeleteFile(const std::string& fname) { - boost::system::error_code ec; - - boost::filesystem::remove(fname, ec); - - Status result; - - if (ec != 0) { - result = Status::IOError(fname, ec.message()); - } - - return result; - } - - virtual Status CreateDir(const std::string& name) { - Status result; - - if (boost::filesystem::exists(name) && - boost::filesystem::is_directory(name)) { - return result; - } - - boost::system::error_code ec; - - if (!boost::filesystem::create_directories(name, ec)) { - result = Status::IOError(name, ec.message()); - } - - return result; - }; - - virtual Status DeleteDir(const std::string& name) { - Status result; - - boost::system::error_code ec; - if (!boost::filesystem::remove_all(name, ec)) { - result = Status::IOError(name, ec.message()); - } - - return result; - }; - - virtual Status GetFileSize(const std::string& fname, uint64_t* size) { - boost::system::error_code ec; - - Status result; - - *size = static_cast<uint64_t>(boost::filesystem::file_size(fname, ec)); - if (ec != 0) { - *size = 0; - result = Status::IOError(fname, ec.message()); - } - - return result; - } - - virtual Status RenameFile(const std::string& src, const std::string& target) { - boost::system::error_code ec; - - boost::filesystem::rename(src, target, ec); - - Status result; - - if (ec != 0) { - result = Status::IOError(src, ec.message()); - } - - return result; - } - - virtual Status LockFile(const std::string& fname, FileLock** lock) { - *lock = NULL; - - Status result; - - try { - if (!boost::filesystem::exists(fname)) { - std::ofstream of(fname.c_str(), std::ios_base::trunc | std::ios_base::out); - } - - assert(boost::filesystem::exists(fname)); - - boost::interprocess::file_lock fl(fname.c_str()); - BoostFileLock * my_lock = new BoostFileLock(); - fl.swap(my_lock->fl_); - if (!my_lock->fl_.try_lock()) { - return Status::IOError("database already in use: could not acquire exclusive lock"); - } - *lock = my_lock; - } catch (const std::exception & e) { - result = Status::IOError("lock " + fname, e.what()); - } - - return result; - } - - virtual Status UnlockFile(FileLock* lock) { - - Status result; - - try { - BoostFileLock * my_lock = static_cast<BoostFileLock *>(lock); - my_lock->fl_.unlock(); - delete my_lock; - } catch (const std::exception & e) { - result = Status::IOError("unlock", e.what()); - } - - return result; - } - - virtual void Schedule(void (*function)(void*), void* arg); - - virtual void StartThread(void (*function)(void* arg), void* arg); - - virtual Status GetTestDirectory(std::string* result) { - boost::system::error_code ec; - boost::filesystem::path temp_dir = - boost::filesystem::temp_directory_path(ec); - if (ec != 0) { - temp_dir = "tmp"; - } - - temp_dir /= "leveldb_tests"; - temp_dir /= boost::lexical_cast<std::string>(current_process_id()); - - // Directory may already exist - CreateDir(temp_dir.generic_string()); - - *result = temp_dir.generic_string(); - - return Status::OK(); - } - -#ifndef LEVELDB_PLATFORM_WINDOWS - static uint64_t gettid() { - pthread_t tid = pthread_self(); - uint64_t thread_id = 0; - memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid))); - return thread_id; - } -#endif - - virtual Status NewLogger(const std::string& fname, Logger** result) { - FILE* f = fopen(fname.c_str(), "wt"); - if (f == NULL) { - *result = NULL; - return Status::IOError(fname, strerror(errno)); - } else { -#ifdef LEVELDB_PLATFORM_WINDOWS - *result = new WinLogger(f); -#else - *result = new PosixLogger(f, &PosixEnv::gettid); -#endif - return Status::OK(); - } - } - - virtual uint64_t NowMicros() { - return static_cast<uint64_t>( - boost::posix_time::microsec_clock::universal_time() - .time_of_day().total_microseconds()); - } - - virtual void SleepForMicroseconds(int micros) { - boost::this_thread::sleep(boost::posix_time::microseconds(micros)); - } - - private: - void PthreadCall(const char* label, int result) { - if (result != 0) { - fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); - exit(1); - } - } - - // BGThread() is the body of the background thread - void BGThread(); - - static void* BGThreadWrapper(void* arg) { - reinterpret_cast<PosixEnv*>(arg)->BGThread(); - return NULL; - } - - boost::mutex mu_; - boost::condition_variable bgsignal_; - boost::scoped_ptr<boost::thread> bgthread_; - - // Entry per Schedule() call - struct BGItem { void* arg; void (*function)(void*); }; - typedef std::deque<BGItem> BGQueue; - BGQueue queue_; -}; - -PosixEnv::PosixEnv() { } - -void PosixEnv::Schedule(void (*function)(void*), void* arg) { - boost::unique_lock<boost::mutex> lock(mu_); - - // Start background thread if necessary - if (!bgthread_) { - bgthread_.reset( - new boost::thread(boost::bind(&PosixEnv::BGThreadWrapper, this))); - } - - // Add to priority queue - queue_.push_back(BGItem()); - queue_.back().function = function; - queue_.back().arg = arg; - - lock.unlock(); - - bgsignal_.notify_one(); - -} - -void PosixEnv::BGThread() { - while (true) { - // Wait until there is an item that is ready to run - boost::unique_lock<boost::mutex> lock(mu_); - - while (queue_.empty()) { - bgsignal_.wait(lock); - } - - void (*function)(void*) = queue_.front().function; - void* arg = queue_.front().arg; - queue_.pop_front(); - - lock.unlock(); - (*function)(arg); - } -} - -namespace { -struct StartThreadState { - void (*user_function)(void*); - void* arg; -}; -} - -static void* StartThreadWrapper(void* arg) { - StartThreadState* state = reinterpret_cast<StartThreadState*>(arg); - state->user_function(state->arg); - delete state; - return NULL; -} - -void PosixEnv::StartThread(void (*function)(void* arg), void* arg) { - StartThreadState* state = new StartThreadState; - state->user_function = function; - state->arg = arg; - - boost::thread t(boost::bind(&StartThreadWrapper, state)); -} - -} - -static boost::once_flag once = BOOST_ONCE_INIT; -static Env* default_env; -static void InitDefaultEnv() { - ::memset(global_read_only_buf, 0, sizeof(global_read_only_buf)); - default_env = new PosixEnv; -} - -Env* Env::Default() { - boost::call_once(once, InitDefaultEnv); - - return default_env; -} - -} diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index cb1f6fc05a..db81f56d11 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -1,8 +1,10 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#if !defined(LEVELDB_PLATFORM_WINDOWS) #include <deque> +#include <set> #include <dirent.h> #include <errno.h> #include <fcntl.h> @@ -23,6 +25,7 @@ #include "leveldb/slice.h" #include "port/port.h" #include "util/logging.h" +#include "util/mutexlock.h" #include "util/posix_logger.h" namespace leveldb { @@ -90,18 +93,75 @@ class PosixRandomAccessFile: public RandomAccessFile { } }; +// Helper class to limit mmap file usage so that we do not end up +// running out virtual memory or running into kernel performance +// problems for very large databases. +class MmapLimiter { + public: + // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes. + MmapLimiter() { + SetAllowed(sizeof(void*) >= 8 ? 1000 : 0); + } + + // If another mmap slot is available, acquire it and return true. + // Else return false. + bool Acquire() { + if (GetAllowed() <= 0) { + return false; + } + MutexLock l(&mu_); + intptr_t x = GetAllowed(); + if (x <= 0) { + return false; + } else { + SetAllowed(x - 1); + return true; + } + } + + // Release a slot acquired by a previous call to Acquire() that returned true. + void Release() { + MutexLock l(&mu_); + SetAllowed(GetAllowed() + 1); + } + + private: + port::Mutex mu_; + port::AtomicPointer allowed_; + + intptr_t GetAllowed() const { + return reinterpret_cast<intptr_t>(allowed_.Acquire_Load()); + } + + // REQUIRES: mu_ must be held + void SetAllowed(intptr_t v) { + allowed_.Release_Store(reinterpret_cast<void*>(v)); + } + + MmapLimiter(const MmapLimiter&); + void operator=(const MmapLimiter&); +}; + // mmap() based random-access class PosixMmapReadableFile: public RandomAccessFile { private: std::string filename_; void* mmapped_region_; size_t length_; + MmapLimiter* limiter_; public: // base[0,length-1] contains the mmapped contents of the file. - PosixMmapReadableFile(const std::string& fname, void* base, size_t length) - : filename_(fname), mmapped_region_(base), length_(length) { } - virtual ~PosixMmapReadableFile() { munmap(mmapped_region_, length_); } + PosixMmapReadableFile(const std::string& fname, void* base, size_t length, + MmapLimiter* limiter) + : filename_(fname), mmapped_region_(base), length_(length), + limiter_(limiter) { + } + + virtual ~PosixMmapReadableFile() { + munmap(mmapped_region_, length_); + limiter_->Release(); + } virtual Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { @@ -300,6 +360,25 @@ static int LockOrUnlock(int fd, bool lock) { class PosixFileLock : public FileLock { public: int fd_; + std::string name_; +}; + +// Set of locked files. We keep a separate set instead of just +// relying on fcntrl(F_SETLK) since fcntl(F_SETLK) does not provide +// any protection against multiple uses from the same process. +class PosixLockTable { + private: + port::Mutex mu_; + std::set<std::string> locked_files_; + public: + bool Insert(const std::string& fname) { + MutexLock l(&mu_); + return locked_files_.insert(fname).second; + } + void Remove(const std::string& fname) { + MutexLock l(&mu_); + locked_files_.erase(fname); + } }; class PosixEnv : public Env { @@ -329,19 +408,21 @@ class PosixEnv : public Env { int fd = open(fname.c_str(), O_RDONLY); if (fd < 0) { s = IOError(fname, errno); - } else if (sizeof(void*) >= 8) { - // Use mmap when virtual address-space is plentiful. + } else if (mmap_limit_.Acquire()) { uint64_t size; s = GetFileSize(fname, &size); if (s.ok()) { void* base = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0); if (base != MAP_FAILED) { - *result = new PosixMmapReadableFile(fname, base, size); + *result = new PosixMmapReadableFile(fname, base, size, &mmap_limit_); } else { s = IOError(fname, errno); } } close(fd); + if (!s.ok()) { + mmap_limit_.Release(); + } } else { *result = new PosixRandomAccessFile(fname, fd); } @@ -430,12 +511,17 @@ class PosixEnv : public Env { int fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644); if (fd < 0) { result = IOError(fname, errno); + } else if (!locks_.Insert(fname)) { + close(fd); + result = Status::IOError("lock " + fname, "already held by process"); } else if (LockOrUnlock(fd, true) == -1) { result = IOError("lock " + fname, errno); close(fd); + locks_.Remove(fname); } else { PosixFileLock* my_lock = new PosixFileLock; my_lock->fd_ = fd; + my_lock->name_ = fname; *lock = my_lock; } return result; @@ -447,6 +533,7 @@ class PosixEnv : public Env { if (LockOrUnlock(my_lock->fd_, false) == -1) { result = IOError("unlock", errno); } + locks_.Remove(my_lock->name_); close(my_lock->fd_); delete my_lock; return result; @@ -523,6 +610,9 @@ class PosixEnv : public Env { struct BGItem { void* arg; void (*function)(void*); }; typedef std::deque<BGItem> BGQueue; BGQueue queue_; + + PosixLockTable locks_; + MmapLimiter mmap_limit_; }; PosixEnv::PosixEnv() : page_size_(getpagesize()), @@ -607,3 +697,5 @@ Env* Env::Default() { } } // namespace leveldb + +#endif diff --git a/src/leveldb/util/env_win.cc b/src/leveldb/util/env_win.cc new file mode 100644 index 0000000000..f1a7610624 --- /dev/null +++ b/src/leveldb/util/env_win.cc @@ -0,0 +1,1031 @@ +// This file contains source that originates from: +// http://code.google.com/p/leveldbwin/source/browse/trunk/win32_impl_src/env_win32.h +// http://code.google.com/p/leveldbwin/source/browse/trunk/win32_impl_src/port_win32.cc +// Those files dont' have any explict license headers but the +// project (http://code.google.com/p/leveldbwin/) lists the 'New BSD License' +// as the license. +#if defined(LEVELDB_PLATFORM_WINDOWS) +#include <map> + + +#include "leveldb/env.h" + +#include "port/port.h" +#include "leveldb/slice.h" +#include "util/logging.h" + +#include <shlwapi.h> +#include <process.h> +#include <cstring> +#include <stdio.h> +#include <errno.h> +#include <io.h> +#include <algorithm> + +#ifdef max +#undef max +#endif + +#ifndef va_copy +#define va_copy(d,s) ((d) = (s)) +#endif + +#if defined DeleteFile +#undef DeleteFile +#endif + +//Declarations +namespace leveldb +{ + +namespace Win32 +{ + +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) + +std::string GetCurrentDir(); +std::wstring GetCurrentDirW(); + +static const std::string CurrentDir = GetCurrentDir(); +static const std::wstring CurrentDirW = GetCurrentDirW(); + +std::string& ModifyPath(std::string& path); +std::wstring& ModifyPath(std::wstring& path); + +std::string GetLastErrSz(); +std::wstring GetLastErrSzW(); + +size_t GetPageSize(); + +typedef void (*ScheduleProc)(void*) ; + +struct WorkItemWrapper +{ + WorkItemWrapper(ScheduleProc proc_,void* content_); + ScheduleProc proc; + void* pContent; +}; + +DWORD WINAPI WorkItemWrapperProc(LPVOID pContent); + +class Win32SequentialFile : public SequentialFile +{ +public: + friend class Win32Env; + virtual ~Win32SequentialFile(); + virtual Status Read(size_t n, Slice* result, char* scratch); + virtual Status Skip(uint64_t n); + BOOL isEnable(); +private: + BOOL _Init(); + void _CleanUp(); + Win32SequentialFile(const std::string& fname); + std::string _filename; + ::HANDLE _hFile; + DISALLOW_COPY_AND_ASSIGN(Win32SequentialFile); +}; + +class Win32RandomAccessFile : public RandomAccessFile +{ +public: + friend class Win32Env; + virtual ~Win32RandomAccessFile(); + virtual Status Read(uint64_t offset, size_t n, Slice* result,char* scratch) const; + BOOL isEnable(); +private: + BOOL _Init(LPCWSTR path); + void _CleanUp(); + Win32RandomAccessFile(const std::string& fname); + HANDLE _hFile; + const std::string _filename; + DISALLOW_COPY_AND_ASSIGN(Win32RandomAccessFile); +}; + +class Win32MapFile : public WritableFile +{ +public: + Win32MapFile(const std::string& fname); + + ~Win32MapFile(); + virtual Status Append(const Slice& data); + virtual Status Close(); + virtual Status Flush(); + virtual Status Sync(); + BOOL isEnable(); +private: + std::string _filename; + HANDLE _hFile; + size_t _page_size; + size_t _map_size; // How much extra memory to map at a time + char* _base; // The mapped region + HANDLE _base_handle; + char* _limit; // Limit of the mapped region + char* _dst; // Where to write next (in range [base_,limit_]) + char* _last_sync; // Where have we synced up to + uint64_t _file_offset; // Offset of base_ in file + //LARGE_INTEGER file_offset_; + // Have we done an munmap of unsynced data? + bool _pending_sync; + + // Roundup x to a multiple of y + static size_t _Roundup(size_t x, size_t y); + size_t _TruncateToPageBoundary(size_t s); + bool _UnmapCurrentRegion(); + bool _MapNewRegion(); + DISALLOW_COPY_AND_ASSIGN(Win32MapFile); + BOOL _Init(LPCWSTR Path); +}; + +class Win32FileLock : public FileLock +{ +public: + friend class Win32Env; + virtual ~Win32FileLock(); + BOOL isEnable(); +private: + BOOL _Init(LPCWSTR path); + void _CleanUp(); + Win32FileLock(const std::string& fname); + HANDLE _hFile; + std::string _filename; + DISALLOW_COPY_AND_ASSIGN(Win32FileLock); +}; + +class Win32Logger : public Logger +{ +public: + friend class Win32Env; + virtual ~Win32Logger(); + virtual void Logv(const char* format, va_list ap); +private: + explicit Win32Logger(WritableFile* pFile); + WritableFile* _pFileProxy; + DISALLOW_COPY_AND_ASSIGN(Win32Logger); +}; + +class Win32Env : public Env +{ +public: + Win32Env(); + virtual ~Win32Env(); + virtual Status NewSequentialFile(const std::string& fname, + SequentialFile** result); + + virtual Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result); + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result); + + virtual bool FileExists(const std::string& fname); + + virtual Status GetChildren(const std::string& dir, + std::vector<std::string>* result); + + virtual Status DeleteFile(const std::string& fname); + + virtual Status CreateDir(const std::string& dirname); + + virtual Status DeleteDir(const std::string& dirname); + + virtual Status GetFileSize(const std::string& fname, uint64_t* file_size); + + virtual Status RenameFile(const std::string& src, + const std::string& target); + + virtual Status LockFile(const std::string& fname, FileLock** lock); + + virtual Status UnlockFile(FileLock* lock); + + virtual void Schedule( + void (*function)(void* arg), + void* arg); + + virtual void StartThread(void (*function)(void* arg), void* arg); + + virtual Status GetTestDirectory(std::string* path); + + //virtual void Logv(WritableFile* log, const char* format, va_list ap); + + virtual Status NewLogger(const std::string& fname, Logger** result); + + virtual uint64_t NowMicros(); + + virtual void SleepForMicroseconds(int micros); +}; + +void ToWidePath(const std::string& value, std::wstring& target) { + wchar_t buffer[MAX_PATH]; + MultiByteToWideChar(CP_ACP, 0, value.c_str(), -1, buffer, MAX_PATH); + target = buffer; +} + +void ToNarrowPath(const std::wstring& value, std::string& target) { + char buffer[MAX_PATH]; + WideCharToMultiByte(CP_ACP, 0, value.c_str(), -1, buffer, MAX_PATH, NULL, NULL); + target = buffer; +} + +std::string GetCurrentDir() +{ + CHAR path[MAX_PATH]; + ::GetModuleFileNameA(::GetModuleHandleA(NULL),path,MAX_PATH); + *strrchr(path,'\\') = 0; + return std::string(path); +} + +std::wstring GetCurrentDirW() +{ + WCHAR path[MAX_PATH]; + ::GetModuleFileNameW(::GetModuleHandleW(NULL),path,MAX_PATH); + *wcsrchr(path,L'\\') = 0; + return std::wstring(path); +} + +std::string& ModifyPath(std::string& path) +{ + if(path[0] == '/' || path[0] == '\\'){ + path = CurrentDir + path; + } + std::replace(path.begin(),path.end(),'/','\\'); + + return path; +} + +std::wstring& ModifyPath(std::wstring& path) +{ + if(path[0] == L'/' || path[0] == L'\\'){ + path = CurrentDirW + path; + } + std::replace(path.begin(),path.end(),L'/',L'\\'); + return path; +} + +std::string GetLastErrSz() +{ + LPWSTR lpMsgBuf; + FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + GetLastError(), + 0, // Default language + (LPWSTR) &lpMsgBuf, + 0, + NULL + ); + std::string Err; + ToNarrowPath(lpMsgBuf, Err); + LocalFree( lpMsgBuf ); + return Err; +} + +std::wstring GetLastErrSzW() +{ + LPVOID lpMsgBuf; + FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + GetLastError(), + 0, // Default language + (LPWSTR) &lpMsgBuf, + 0, + NULL + ); + std::wstring Err = (LPCWSTR)lpMsgBuf; + LocalFree(lpMsgBuf); + return Err; +} + +WorkItemWrapper::WorkItemWrapper( ScheduleProc proc_,void* content_ ) : + proc(proc_),pContent(content_) +{ + +} + +DWORD WINAPI WorkItemWrapperProc(LPVOID pContent) +{ + WorkItemWrapper* item = static_cast<WorkItemWrapper*>(pContent); + ScheduleProc TempProc = item->proc; + void* arg = item->pContent; + delete item; + TempProc(arg); + return 0; +} + +size_t GetPageSize() +{ + SYSTEM_INFO si; + GetSystemInfo(&si); + return std::max(si.dwPageSize,si.dwAllocationGranularity); +} + +const size_t g_PageSize = GetPageSize(); + + +Win32SequentialFile::Win32SequentialFile( const std::string& fname ) : + _filename(fname),_hFile(NULL) +{ + _Init(); +} + +Win32SequentialFile::~Win32SequentialFile() +{ + _CleanUp(); +} + +Status Win32SequentialFile::Read( size_t n, Slice* result, char* scratch ) +{ + Status sRet; + DWORD hasRead = 0; + if(_hFile && ReadFile(_hFile,scratch,n,&hasRead,NULL) ){ + *result = Slice(scratch,hasRead); + } else { + sRet = Status::IOError(_filename, Win32::GetLastErrSz() ); + } + return sRet; +} + +Status Win32SequentialFile::Skip( uint64_t n ) +{ + Status sRet; + LARGE_INTEGER Move,NowPointer; + Move.QuadPart = n; + if(!SetFilePointerEx(_hFile,Move,&NowPointer,FILE_CURRENT)){ + sRet = Status::IOError(_filename,Win32::GetLastErrSz()); + } + return sRet; +} + +BOOL Win32SequentialFile::isEnable() +{ + return _hFile ? TRUE : FALSE; +} + +BOOL Win32SequentialFile::_Init() +{ + std::wstring path; + ToWidePath(_filename, path); + _hFile = CreateFileW(path.c_str(), + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + return _hFile ? TRUE : FALSE; +} + +void Win32SequentialFile::_CleanUp() +{ + if(_hFile){ + CloseHandle(_hFile); + _hFile = NULL; + } +} + +Win32RandomAccessFile::Win32RandomAccessFile( const std::string& fname ) : + _filename(fname),_hFile(NULL) +{ + std::wstring path; + ToWidePath(fname, path); + _Init( path.c_str() ); +} + +Win32RandomAccessFile::~Win32RandomAccessFile() +{ + _CleanUp(); +} + +Status Win32RandomAccessFile::Read(uint64_t offset,size_t n,Slice* result,char* scratch) const +{ + Status sRet; + OVERLAPPED ol = {0}; + ZeroMemory(&ol,sizeof(ol)); + ol.Offset = (DWORD)offset; + ol.OffsetHigh = (DWORD)(offset >> 32); + DWORD hasRead = 0; + if(!ReadFile(_hFile,scratch,n,&hasRead,&ol)) + sRet = Status::IOError(_filename,Win32::GetLastErrSz()); + else + *result = Slice(scratch,hasRead); + return sRet; +} + +BOOL Win32RandomAccessFile::_Init( LPCWSTR path ) +{ + BOOL bRet = FALSE; + if(!_hFile) + _hFile = ::CreateFileW(path,GENERIC_READ,0,NULL,OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS,NULL); + if(!_hFile || _hFile == INVALID_HANDLE_VALUE ) + _hFile = NULL; + else + bRet = TRUE; + return bRet; +} + +BOOL Win32RandomAccessFile::isEnable() +{ + return _hFile ? TRUE : FALSE; +} + +void Win32RandomAccessFile::_CleanUp() +{ + if(_hFile){ + ::CloseHandle(_hFile); + _hFile = NULL; + } +} + +size_t Win32MapFile::_Roundup( size_t x, size_t y ) +{ + return ((x + y - 1) / y) * y; +} + +size_t Win32MapFile::_TruncateToPageBoundary( size_t s ) +{ + s -= (s & (_page_size - 1)); + assert((s % _page_size) == 0); + return s; +} + +bool Win32MapFile::_UnmapCurrentRegion() +{ + bool result = true; + if (_base != NULL) { + if (_last_sync < _limit) { + // Defer syncing this data until next Sync() call, if any + _pending_sync = true; + } + UnmapViewOfFile(_base); + CloseHandle(_base_handle); + _file_offset += _limit - _base; + _base = NULL; + _base_handle = NULL; + _limit = NULL; + _last_sync = NULL; + _dst = NULL; + // Increase the amount we map the next time, but capped at 1MB + if (_map_size < (1<<20)) { + _map_size *= 2; + } + } + return result; +} + +bool Win32MapFile::_MapNewRegion() +{ + assert(_base == NULL); + //LONG newSizeHigh = (LONG)((file_offset_ + map_size_) >> 32); + //LONG newSizeLow = (LONG)((file_offset_ + map_size_) & 0xFFFFFFFF); + DWORD off_hi = (DWORD)(_file_offset >> 32); + DWORD off_lo = (DWORD)(_file_offset & 0xFFFFFFFF); + LARGE_INTEGER newSize; + newSize.QuadPart = _file_offset + _map_size; + SetFilePointerEx(_hFile, newSize, NULL, FILE_BEGIN); + SetEndOfFile(_hFile); + + _base_handle = CreateFileMappingA( + _hFile, + NULL, + PAGE_READWRITE, + 0, + 0, + 0); + if (_base_handle != NULL) { + _base = (char*) MapViewOfFile(_base_handle, + FILE_MAP_ALL_ACCESS, + off_hi, + off_lo, + _map_size); + if (_base != NULL) { + _limit = _base + _map_size; + _dst = _base; + _last_sync = _base; + return true; + } + } + return false; +} + +Win32MapFile::Win32MapFile( const std::string& fname) : + _filename(fname), + _hFile(NULL), + _page_size(Win32::g_PageSize), + _map_size(_Roundup(65536, Win32::g_PageSize)), + _base(NULL), + _base_handle(NULL), + _limit(NULL), + _dst(NULL), + _last_sync(NULL), + _file_offset(0), + _pending_sync(false) +{ + std::wstring path; + ToWidePath(fname, path); + _Init(path.c_str()); + assert((Win32::g_PageSize & (Win32::g_PageSize - 1)) == 0); +} + +Status Win32MapFile::Append( const Slice& data ) +{ + const char* src = data.data(); + size_t left = data.size(); + Status s; + while (left > 0) { + assert(_base <= _dst); + assert(_dst <= _limit); + size_t avail = _limit - _dst; + if (avail == 0) { + if (!_UnmapCurrentRegion() || + !_MapNewRegion()) { + return Status::IOError("WinMmapFile.Append::UnmapCurrentRegion or MapNewRegion: ", Win32::GetLastErrSz()); + } + } + size_t n = (left <= avail) ? left : avail; + memcpy(_dst, src, n); + _dst += n; + src += n; + left -= n; + } + return s; +} + +Status Win32MapFile::Close() +{ + Status s; + size_t unused = _limit - _dst; + if (!_UnmapCurrentRegion()) { + s = Status::IOError("WinMmapFile.Close::UnmapCurrentRegion: ",Win32::GetLastErrSz()); + } else if (unused > 0) { + // Trim the extra space at the end of the file + LARGE_INTEGER newSize; + newSize.QuadPart = _file_offset - unused; + if (!SetFilePointerEx(_hFile, newSize, NULL, FILE_BEGIN)) { + s = Status::IOError("WinMmapFile.Close::SetFilePointer: ",Win32::GetLastErrSz()); + } else + SetEndOfFile(_hFile); + } + if (!CloseHandle(_hFile)) { + if (s.ok()) { + s = Status::IOError("WinMmapFile.Close::CloseHandle: ", Win32::GetLastErrSz()); + } + } + _hFile = INVALID_HANDLE_VALUE; + _base = NULL; + _base_handle = NULL; + _limit = NULL; + + return s; +} + +Status Win32MapFile::Sync() +{ + Status s; + if (_pending_sync) { + // Some unmapped data was not synced + _pending_sync = false; + if (!FlushFileBuffers(_hFile)) { + s = Status::IOError("WinMmapFile.Sync::FlushFileBuffers: ",Win32::GetLastErrSz()); + } + } + if (_dst > _last_sync) { + // Find the beginnings of the pages that contain the first and last + // bytes to be synced. + size_t p1 = _TruncateToPageBoundary(_last_sync - _base); + size_t p2 = _TruncateToPageBoundary(_dst - _base - 1); + _last_sync = _dst; + if (!FlushViewOfFile(_base + p1, p2 - p1 + _page_size)) { + s = Status::IOError("WinMmapFile.Sync::FlushViewOfFile: ",Win32::GetLastErrSz()); + } + } + return s; +} + +Status Win32MapFile::Flush() +{ + return Status::OK(); +} + +Win32MapFile::~Win32MapFile() +{ + if (_hFile != INVALID_HANDLE_VALUE) { + Win32MapFile::Close(); + } +} + +BOOL Win32MapFile::_Init( LPCWSTR Path ) +{ + DWORD Flag = PathFileExistsW(Path) ? OPEN_EXISTING : CREATE_ALWAYS; + _hFile = CreateFileW(Path, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_DELETE|FILE_SHARE_WRITE, + NULL, + Flag, + FILE_ATTRIBUTE_NORMAL, + NULL); + if(!_hFile || _hFile == INVALID_HANDLE_VALUE) + return FALSE; + else + return TRUE; +} + +BOOL Win32MapFile::isEnable() +{ + return _hFile ? TRUE : FALSE; +} + +Win32FileLock::Win32FileLock( const std::string& fname ) : + _hFile(NULL),_filename(fname) +{ + std::wstring path; + ToWidePath(fname, path); + _Init(path.c_str()); +} + +Win32FileLock::~Win32FileLock() +{ + _CleanUp(); +} + +BOOL Win32FileLock::_Init( LPCWSTR path ) +{ + BOOL bRet = FALSE; + if(!_hFile) + _hFile = ::CreateFileW(path,0,0,NULL,CREATE_ALWAYS,FILE_ATTRIBUTE_NORMAL,NULL); + if(!_hFile || _hFile == INVALID_HANDLE_VALUE ){ + _hFile = NULL; + } + else + bRet = TRUE; + return bRet; +} + +void Win32FileLock::_CleanUp() +{ + ::CloseHandle(_hFile); + _hFile = NULL; +} + +BOOL Win32FileLock::isEnable() +{ + return _hFile ? TRUE : FALSE; +} + +Win32Logger::Win32Logger(WritableFile* pFile) : _pFileProxy(pFile) +{ + assert(_pFileProxy); +} + +Win32Logger::~Win32Logger() +{ + if(_pFileProxy) + delete _pFileProxy; +} + +void Win32Logger::Logv( const char* format, va_list ap ) +{ + uint64_t thread_id = ::GetCurrentThreadId(); + + // We try twice: the first time with a fixed-size stack allocated buffer, + // and the second time with a much larger dynamically allocated buffer. + char buffer[500]; + for (int iter = 0; iter < 2; iter++) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } else { + bufsize = 30000; + base = new char[bufsize]; + } + char* p = base; + char* limit = base + bufsize; + + SYSTEMTIME st; + GetLocalTime(&st); + p += snprintf(p, limit - p, + "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", + int(st.wYear), + int(st.wMonth), + int(st.wDay), + int(st.wHour), + int(st.wMinute), + int(st.wMinute), + int(st.wMilliseconds), + static_cast<long long unsigned int>(thread_id)); + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + p += vsnprintf(p, limit - p, format, backup_ap); + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) { + continue; // Try again with larger buffer + } else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + DWORD hasWritten = 0; + if(_pFileProxy){ + _pFileProxy->Append(Slice(base, p - base)); + _pFileProxy->Flush(); + } + if (base != buffer) { + delete[] base; + } + break; + } +} + +bool Win32Env::FileExists(const std::string& fname) +{ + std::string path = fname; + std::wstring wpath; + ToWidePath(ModifyPath(path), wpath); + return ::PathFileExistsW(wpath.c_str()) ? true : false; +} + +Status Win32Env::GetChildren(const std::string& dir, std::vector<std::string>* result) +{ + Status sRet; + ::WIN32_FIND_DATAW wfd; + std::string path = dir; + ModifyPath(path); + path += "\\*.*"; + std::wstring wpath; + ToWidePath(path, wpath); + + ::HANDLE hFind = ::FindFirstFileW(wpath.c_str() ,&wfd); + if(hFind && hFind != INVALID_HANDLE_VALUE){ + BOOL hasNext = TRUE; + std::string child; + while(hasNext){ + ToNarrowPath(wfd.cFileName, child); + if(child != ".." && child != ".") { + result->push_back(child); + } + hasNext = ::FindNextFileW(hFind,&wfd); + } + ::FindClose(hFind); + } + else + sRet = Status::IOError(dir,"Could not get children."); + return sRet; +} + +void Win32Env::SleepForMicroseconds( int micros ) +{ + ::Sleep((micros + 999) /1000); +} + + +Status Win32Env::DeleteFile( const std::string& fname ) +{ + Status sRet; + std::string path = fname; + std::wstring wpath; + ToWidePath(ModifyPath(path), wpath); + + if(!::DeleteFileW(wpath.c_str())) { + sRet = Status::IOError(path, "Could not delete file."); + } + return sRet; +} + +Status Win32Env::GetFileSize( const std::string& fname, uint64_t* file_size ) +{ + Status sRet; + std::string path = fname; + std::wstring wpath; + ToWidePath(ModifyPath(path), wpath); + + HANDLE file = ::CreateFileW(wpath.c_str(), + GENERIC_READ,FILE_SHARE_READ,NULL,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,NULL); + LARGE_INTEGER li; + if(::GetFileSizeEx(file,&li)){ + *file_size = (uint64_t)li.QuadPart; + }else + sRet = Status::IOError(path,"Could not get the file size."); + CloseHandle(file); + return sRet; +} + +Status Win32Env::RenameFile( const std::string& src, const std::string& target ) +{ + Status sRet; + std::string src_path = src; + std::wstring wsrc_path; + ToWidePath(ModifyPath(src_path), wsrc_path); + std::string target_path = target; + std::wstring wtarget_path; + ToWidePath(ModifyPath(target_path), wtarget_path); + + if(!MoveFileW(wsrc_path.c_str(), wtarget_path.c_str() ) ){ + DWORD err = GetLastError(); + if(err == 0x000000b7){ + if(!::DeleteFileW(wtarget_path.c_str() ) ) + sRet = Status::IOError(src, "Could not rename file."); + else if(!::MoveFileW(wsrc_path.c_str(), + wtarget_path.c_str() ) ) + sRet = Status::IOError(src, "Could not rename file."); + } + } + return sRet; +} + +Status Win32Env::LockFile( const std::string& fname, FileLock** lock ) +{ + Status sRet; + std::string path = fname; + ModifyPath(path); + Win32FileLock* _lock = new Win32FileLock(path); + if(!_lock->isEnable()){ + delete _lock; + *lock = NULL; + sRet = Status::IOError(path, "Could not lock file."); + } + else + *lock = _lock; + return sRet; +} + +Status Win32Env::UnlockFile( FileLock* lock ) +{ + Status sRet; + delete lock; + return sRet; +} + +void Win32Env::Schedule( void (*function)(void* arg), void* arg ) +{ + QueueUserWorkItem(Win32::WorkItemWrapperProc, + new Win32::WorkItemWrapper(function,arg), + WT_EXECUTEDEFAULT); +} + +void Win32Env::StartThread( void (*function)(void* arg), void* arg ) +{ + ::_beginthread(function,0,arg); +} + +Status Win32Env::GetTestDirectory( std::string* path ) +{ + Status sRet; + WCHAR TempPath[MAX_PATH]; + ::GetTempPathW(MAX_PATH,TempPath); + ToNarrowPath(TempPath, *path); + path->append("leveldb\\test\\"); + ModifyPath(*path); + return sRet; +} + +uint64_t Win32Env::NowMicros() +{ +#ifndef USE_VISTA_API +#define GetTickCount64 GetTickCount +#endif + return (uint64_t)(GetTickCount64()*1000); +} + +static Status CreateDirInner( const std::string& dirname ) +{ + Status sRet; + DWORD attr = ::GetFileAttributes(dirname.c_str()); + if (attr == INVALID_FILE_ATTRIBUTES) { // doesn't exist: + std::size_t slash = dirname.find_last_of("\\"); + if (slash != std::string::npos){ + sRet = CreateDirInner(dirname.substr(0, slash)); + if (!sRet.ok()) return sRet; + } + BOOL result = ::CreateDirectory(dirname.c_str(), NULL); + if (result == FALSE) { + sRet = Status::IOError(dirname, "Could not create directory."); + return sRet; + } + } + return sRet; +} + +Status Win32Env::CreateDir( const std::string& dirname ) +{ + std::string path = dirname; + if(path[path.length() - 1] != '\\'){ + path += '\\'; + } + ModifyPath(path); + + return CreateDirInner(path); +} + +Status Win32Env::DeleteDir( const std::string& dirname ) +{ + Status sRet; + std::wstring path; + ToWidePath(dirname, path); + ModifyPath(path); + if(!::RemoveDirectoryW( path.c_str() ) ){ + sRet = Status::IOError(dirname, "Could not delete directory."); + } + return sRet; +} + +Status Win32Env::NewSequentialFile( const std::string& fname, SequentialFile** result ) +{ + Status sRet; + std::string path = fname; + ModifyPath(path); + Win32SequentialFile* pFile = new Win32SequentialFile(path); + if(pFile->isEnable()){ + *result = pFile; + }else { + delete pFile; + sRet = Status::IOError(path, Win32::GetLastErrSz()); + } + return sRet; +} + +Status Win32Env::NewRandomAccessFile( const std::string& fname, RandomAccessFile** result ) +{ + Status sRet; + std::string path = fname; + Win32RandomAccessFile* pFile = new Win32RandomAccessFile(ModifyPath(path)); + if(!pFile->isEnable()){ + delete pFile; + *result = NULL; + sRet = Status::IOError(path,"Could not create random access file."); + }else + *result = pFile; + return sRet; +} + +Status Win32Env::NewLogger( const std::string& fname, Logger** result ) +{ + Status sRet; + std::string path = fname; + Win32MapFile* pMapFile = new Win32MapFile(ModifyPath(path)); + if(!pMapFile->isEnable()){ + delete pMapFile; + *result = NULL; + sRet = Status::IOError(path,"could not create a logger."); + }else + *result = new Win32Logger(pMapFile); + return sRet; +} + +Status Win32Env::NewWritableFile( const std::string& fname, WritableFile** result ) +{ + Status sRet; + std::string path = fname; + Win32MapFile* pFile = new Win32MapFile(ModifyPath(path)); + if(!pFile->isEnable()){ + *result = NULL; + sRet = Status::IOError(fname,Win32::GetLastErrSz()); + }else + *result = pFile; + return sRet; +} + +Win32Env::Win32Env() +{ + +} + +Win32Env::~Win32Env() +{ + +} + + +} // Win32 namespace + +static port::OnceType once = LEVELDB_ONCE_INIT; +static Env* default_env; +static void InitDefaultEnv() { default_env = new Win32::Win32Env(); } + +Env* Env::Default() { + port::InitOnce(&once, InitDefaultEnv); + return default_env; +} + +} // namespace leveldb + +#endif // defined(LEVELDB_PLATFORM_WINDOWS) diff --git a/src/leveldb/util/mutexlock.h b/src/leveldb/util/mutexlock.h index c3f3306d3e..1ff5a9efa1 100644 --- a/src/leveldb/util/mutexlock.h +++ b/src/leveldb/util/mutexlock.h @@ -6,6 +6,7 @@ #define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ #include "port/port.h" +#include "port/thread_annotations.h" namespace leveldb { @@ -19,12 +20,13 @@ namespace leveldb { // ... some complex code, possibly with multiple return paths ... // } -class MutexLock { +class SCOPED_LOCKABLE MutexLock { public: - explicit MutexLock(port::Mutex *mu) : mu_(mu) { + explicit MutexLock(port::Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { this->mu_->Lock(); } - ~MutexLock() { this->mu_->Unlock(); } + ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } private: port::Mutex *const mu_; diff --git a/src/leveldb/util/posix_logger.h b/src/leveldb/util/posix_logger.h index 9741b1afad..c063c2b7cb 100644 --- a/src/leveldb/util/posix_logger.h +++ b/src/leveldb/util/posix_logger.h @@ -3,7 +3,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. // // Logger implementation that can be shared by all environments -// where enough posix functionality is available. +// where enough Posix functionality is available. #ifndef STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ #define STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ diff --git a/src/leveldb/util/win_logger.cc b/src/leveldb/util/win_logger.cc deleted file mode 100644 index 834c98cc76..0000000000 --- a/src/leveldb/util/win_logger.cc +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include "util/win_logger.h" - -#include <windows.h> - -namespace leveldb { - -void WinLogger::Logv(const char* format, va_list ap) { - const uint64_t thread_id = static_cast<uint64_t>(::GetCurrentThreadId()); - - // We try twice: the first time with a fixed-size stack allocated buffer, - // and the second time with a much larger dynamically allocated buffer. - char buffer[500]; - - for (int iter = 0; iter < 2; iter++) { - char* base; - int bufsize; - if (iter == 0) { - bufsize = sizeof(buffer); - base = buffer; - } else { - bufsize = 30000; - base = new char[bufsize]; - } - - char* p = base; - char* limit = base + bufsize; - - SYSTEMTIME st; - - // GetSystemTime returns UTC time, we want local time! - ::GetLocalTime(&st); - -#ifdef _MSC_VER - p += _snprintf_s(p, limit - p, _TRUNCATE, - "%04d/%02d/%02d-%02d:%02d:%02d.%03d %llx ", - st.wYear, - st.wMonth, - st.wDay, - st.wHour, - st.wMinute, - st.wSecond, - st.wMilliseconds, - static_cast<long long unsigned int>(thread_id)); -#else -#ifdef __MINGW32__ - p += snprintf(p, limit - p, - "%04d/%02d/%02d-%02d:%02d:%02d.%03d %llx ", - st.wYear, - st.wMonth, - st.wDay, - st.wHour, - st.wMinute, - st.wSecond, - st.wMilliseconds, - static_cast<long long unsigned int>(thread_id)); -#else -#error Unable to detect Windows compiler (neither _MSC_VER nor __MINGW32__ are set) -#endif -#endif - - // Print the message - if (p < limit) { - va_list backup_ap = ap; - p += vsnprintf(p, limit - p, format, backup_ap); - va_end(backup_ap); - } - - // Truncate to available space if necessary - if (p >= limit) { - if (iter == 0) { - continue; // Try again with larger buffer - } else { - p = limit - 1; - } - } - - // Add newline if necessary - if (p == base || p[-1] != '\n') { - *p++ = '\n'; - } - - assert(p <= limit); - fwrite(base, 1, p - base, file_); - fflush(file_); - if (base != buffer) { - delete[] base; - } - break; - } -} - -}
\ No newline at end of file diff --git a/src/leveldb/util/win_logger.h b/src/leveldb/util/win_logger.h deleted file mode 100644 index b155d5c319..0000000000 --- a/src/leveldb/util/win_logger.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -// Logger implementation for Windows - -#ifndef STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_ -#define STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_ - -#include <stdio.h> -#include "leveldb/env.h" - -namespace leveldb { - -class WinLogger : public Logger { - private: - FILE* file_; - public: - explicit WinLogger(FILE* f) : file_(f) { assert(file_); } - virtual ~WinLogger() { - fclose(file_); - } - virtual void Logv(const char* format, va_list ap); - -}; - -} -#endif // STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_ diff --git a/src/main.cpp b/src/main.cpp index 46ed6c56a5..84a5cdc17b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -788,6 +788,7 @@ bool CTxMemPool::accept(CTransaction &tx, bool fCheckInputs, // If updated, erase old tx from wallet if (ptxOld) EraseFromWallets(ptxOld->GetHash()); + SyncWithWallets(hash, tx, NULL, true); printf("CTxMemPool::accept() : accepted %s (poolsz %"PRIszu")\n", hash.ToString().substr(0,10).c_str(), @@ -1612,6 +1613,14 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsViewCache &view, bool fJust // verify that the view's current state corresponds to the previous block assert(pindex->pprev == view.GetBestBlock()); + // Special case for the genesis block, skipping connection of its transactions + // (its coinbase is unspendable) + if (GetHash() == hashGenesisBlock) { + view.SetBestBlock(pindex); + pindexGenesisBlock = pindex; + return true; + } + bool fScriptChecks = pindex->nHeight >= Checkpoints::GetTotalBlocksEstimate(); // Do not allow blocks that contain transactions which 'overwrite' older transactions, @@ -1755,21 +1764,6 @@ bool SetBestChain(CBlockIndex* pindexNew) // Only when all have succeeded, we push it to pcoinsTip. CCoinsViewCache view(*pcoinsTip, true); - // special case for attaching the genesis block - // note that no ConnectBlock is called, so its coinbase output is non-spendable - if (pindexGenesisBlock == NULL && pindexNew->GetBlockHash() == hashGenesisBlock) - { - view.SetBestBlock(pindexNew); - if (!view.Flush()) - return false; - pindexGenesisBlock = pindexNew; - pindexBest = pindexNew; - hashBestChain = pindexNew->GetBlockHash(); - nBestHeight = pindexBest->nHeight; - bnBestChainWork = pindexNew->bnChainWork; - return true; - } - // Find the fork (typically, there is none) CBlockIndex* pfork = view.GetBestBlock(); CBlockIndex* plonger = pindexNew; @@ -3280,6 +3274,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (fDebugNet || (vInv.size() != 1)) printf("received getdata (%"PRIszu" invsz)\n", vInv.size()); + vector<CInv> vNotFound; BOOST_FOREACH(const CInv& inv, vInv) { if (fShutdown) @@ -3303,6 +3298,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (pfrom->pfilter) { CMerkleBlock merkleBlock(block, *pfrom->pfilter); + pfrom->PushMessage("merkleblock", merkleBlock); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didnt send here - @@ -3313,7 +3309,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) if (!pfrom->setInventoryKnown.count(CInv(MSG_TX, pair.second))) pfrom->PushMessage("tx", block.vtx[pair.first]); - pfrom->PushMessage("merkleblock", merkleBlock); } // else // no response @@ -3352,12 +3347,27 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) ss.reserve(1000); ss << tx; pfrom->PushMessage("tx", ss); + pushed = true; } } + if (!pushed) { + vNotFound.push_back(inv); + } } - // Track requests for our stuff + // Track requests for our stuff. Inventory(inv.hash); + + if (!vNotFound.empty()) { + // Let the peer know that we didn't find what it asked for, so it doesn't + // have to wait around forever. Currently only SPV clients actually care + // about this message: it's needed when they are recursively walking the + // dependencies of relevant unconfirmed transactions. SPV clients want to + // do that because they want to know about (and store and rebroadcast and + // risk analyze) the dependencies of transactions relevant to them, without + // having to download the entire memory pool. + pfrom->PushMessage("notfound", vNotFound); + } } } @@ -3447,7 +3457,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) bool fMissingInputs = false; if (tx.AcceptToMemoryPool(true, &fMissingInputs)) { - SyncWithWallets(inv.hash, tx, NULL, true); RelayTransaction(tx, inv.hash, vMsg); mapAlreadyAskedFor.erase(inv); vWorkQueue.push_back(inv.hash); @@ -3470,7 +3479,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (tx.AcceptToMemoryPool(true, &fMissingInputs2)) { printf(" accepted orphan tx %s\n", inv.hash.ToString().substr(0,10).c_str()); - SyncWithWallets(inv.hash, tx, NULL, true); RelayTransaction(tx, inv.hash, vMsg); mapAlreadyAskedFor.erase(inv); vWorkQueue.push_back(inv.hash); @@ -3624,7 +3632,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) // Nodes must NEVER send a data item > 520 bytes (the max size for a script data object, // and thus, the maximum size any matched object can have) in a filteradd message - if (vData.size() > 520) + if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { pfrom->Misbehaving(100); } else { diff --git a/src/makefile.linux-mingw b/src/makefile.linux-mingw index ff565f2a2d..c3cbe90bcd 100644 --- a/src/makefile.linux-mingw +++ b/src/makefile.linux-mingw @@ -21,6 +21,7 @@ LIBPATHS= \ -L"$(DEPSDIR)/openssl-1.0.1c" LIBS= \ + $(CURDIR)/leveldb/libleveldb.a $(CURDIR)/leveldb/libmemenv.a \ -l boost_system-mt-s \ -l boost_filesystem-mt-s \ -l boost_program_options-mt-s \ @@ -33,7 +34,8 @@ LIBS= \ DEFS=-D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -DBOOST_SPIRIT_THREADSAFE DEBUGFLAGS=-g CFLAGS=-O2 -w -Wall -Wextra -Wformat -Wformat-security -Wno-unused-parameter $(DEBUGFLAGS) $(DEFS) $(INCLUDEPATHS) -LDFLAGS=-Wl,--dynamicbase -Wl,--nxcompat +# enable: ASLR, DEP and large address aware +LDFLAGS=-Wl,--dynamicbase -Wl,--nxcompat -Wl,--large-address-aware TESTDEFS = -DTEST_DATA_DIR=$(abspath test/data) @@ -90,11 +92,10 @@ OBJS= \ all: bitcoind.exe -LIBS += $(CURDIR)/leveldb/libleveldb.a $(CURDIR)/leveldb/libmemenv.a DEFS += -I"$(CURDIR)/leveldb/include" DEFS += -I"$(CURDIR)/leveldb/helpers" leveldb/libleveldb.a: - @echo "Building LevelDB ..." && cd leveldb && CC=i586-mingw32msvc-gcc CXX=i586-mingw32msvc-g++ TARGET_OS=OS_WINDOWS_CROSSCOMPILE CXXFLAGS="-I$(INCLUDEPATHS)" LDFLAGS="-L$(LIBPATHS)" $(MAKE) libleveldb.a libmemenv.a && i586-mingw32msvc-ranlib libleveldb.a && i586-mingw32msvc-ranlib libmemenv.a && cd .. + @echo "Building LevelDB ..." && cd leveldb && CC=i586-mingw32msvc-gcc CXX=i586-mingw32msvc-g++ TARGET_OS=OS_WINDOWS_CROSSCOMPILE CXXFLAGS="$(INCLUDEPATHS)" LDFLAGS="$(LIBPATHS)" $(MAKE) libleveldb.a libmemenv.a && i586-mingw32msvc-ranlib libleveldb.a && i586-mingw32msvc-ranlib libmemenv.a && cd .. obj/leveldb.o: leveldb/libleveldb.a obj/build.h: FORCE @@ -123,5 +124,6 @@ clean: -rm -f obj-test/*.o -rm -f test_bitcoin.exe -rm -f obj/build.h + cd leveldb && TARGET_OS=OS_WINDOWS_CROSSCOMPILE $(MAKE) clean && cd .. FORCE: diff --git a/src/makefile.mingw b/src/makefile.mingw index 9a6680bf40..366d32bd86 100644 --- a/src/makefile.mingw +++ b/src/makefile.mingw @@ -2,25 +2,41 @@ # Distributed under the MIT/X11 software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -USE_UPNP:=0 +# Makefile for the MinGW g++ compiler/toolchain +# +# Assumes Berkeley DB, Boost, and OpenSSL have all been compiled and installed +# into /usr/local (/usr/local/include, /usr/local/lib). +# +# If dependencies are somewhere else, run 'make DEPSDIR=/path/' +# +# Boost libraries are given wacky names that include the particular version of +# boost you're using; set BOOST_SUFFIX appropriately. +# +# 'make clean' assumes it is running inside a MSYS shell, and uses 'rm' +# to remove files. + +USE_UPNP:=- USE_IPV6:=1 +DEPSDIR?=/usr/local +BOOST_SUFFIX?=-mgw46-mt-sd-1_52 + INCLUDEPATHS= \ - -I"C:\boost-1.50.0-mgw" \ - -I"C:\db-4.8.30.NC-mgw\build_unix" \ - -I"C:\openssl-1.0.1c-mgw\include" + -I"$(CURDIR)" \ + -I"$(DEPSDIR)/include" LIBPATHS= \ - -L"C:\boost-1.50.0-mgw\stage\lib" \ - -L"C:\db-4.8.30.NC-mgw\build_unix" \ - -L"C:\openssl-1.0.1c-mgw" + -L"$(CURDIR)/leveldb" \ + -L"$(DEPSDIR)/lib" LIBS= \ - -l boost_system-mgw45-mt-s-1_50 \ - -l boost_filesystem-mgw45-mt-s-1_50 \ - -l boost_program_options-mgw45-mt-s-1_50 \ - -l boost_thread-mgw45-mt-s-1_50 \ - -l boost_chrono-mgw45-mt-s-1_50 \ + -l leveldb \ + -l memenv \ + -l boost_system$(BOOST_SUFFIX) \ + -l boost_filesystem$(BOOST_SUFFIX) \ + -l boost_program_options$(BOOST_SUFFIX) \ + -l boost_thread$(BOOST_SUFFIX) \ + -l boost_chrono$(BOOST_SUFFIX) \ -l db_cxx \ -l ssl \ -l crypto @@ -28,7 +44,8 @@ LIBS= \ DEFS=-DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -DBOOST_SPIRIT_THREADSAFE DEBUGFLAGS=-g CFLAGS=-mthreads -O2 -w -Wall -Wextra -Wformat -Wformat-security -Wno-unused-parameter $(DEBUGFLAGS) $(DEFS) $(INCLUDEPATHS) -LDFLAGS=-Wl,--dynamicbase -Wl,--nxcompat +# enable: ASLR, DEP and large address aware +LDFLAGS=-Wl,--dynamicbase -Wl,--nxcompat -Wl,--large-address-aware TESTDEFS = -DTEST_DATA_DIR=$(abspath test/data) @@ -36,8 +53,6 @@ ifndef USE_UPNP override USE_UPNP = - endif ifneq (${USE_UPNP}, -) - INCLUDEPATHS += -I"C:\miniupnpc-1.6-mgw" - LIBPATHS += -L"C:\miniupnpc-1.6-mgw" LIBS += -l miniupnpc -l iphlpapi DEFS += -DSTATICLIB -DUSE_UPNP=$(USE_UPNP) endif @@ -93,12 +108,12 @@ test check: test_bitcoin.exe FORCE # # LevelDB support # -LIBS += $(CURDIR)/leveldb/libleveldb.a $(CURDIR)/leveldb/libmemenv.a DEFS += $(addprefix -I,$(CURDIR)/leveldb/include) DEFS += $(addprefix -I,$(CURDIR)/leveldb/helpers) -# TODO: If this fails, try adding a ranlib libleveldb.a && ranlib libmemenv.a + leveldb/libleveldb.a: - cd leveldb && $(MAKE) libleveldb.a libmemenv.a && cd .. + cd leveldb && $(MAKE) OPT="$(DEBUGFLAGS)" TARGET_OS=NATIVE_WINDOWS libleveldb.a libmemenv.a && cd .. + obj/leveldb.o: leveldb/libleveldb.a obj/%.o: %.cpp $(HEADERS) @@ -113,11 +128,12 @@ obj-test/%.o: test/%.cpp $(HEADERS) g++ -c $(TESTDEFS) $(CFLAGS) -o $@ $< test_bitcoin.exe: $(TESTOBJS) $(filter-out obj/init.o,$(OBJS:obj/%=obj/%)) - g++ $(CFLAGS) $(LDFLAGS) -o $@ $(LIBPATHS) $^ -lboost_unit_test_framework $(LIBS) + g++ $(CFLAGS) $(LDFLAGS) -o $@ $(LIBPATHS) $^ -lboost_unit_test_framework$(BOOST_SUFFIX) $(LIBS) clean: - -del /Q bitcoind test_bitcoin - -del /Q obj\* - -del /Q obj-test\* + rm -f bitcoind.exe test_bitcoin.exe + rm -f obj/* + rm -f obj-test/* + cd leveldb && $(MAKE) TARGET_OS=NATIVE_WINDOWS clean && cd .. FORCE: diff --git a/src/noui.cpp b/src/noui.cpp index 96a8de4ee9..302d059291 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -37,9 +37,15 @@ static bool noui_ThreadSafeAskFee(int64 /*nFeeRequired*/) return true; } +static void noui_InitMessage(const std::string &message) +{ + printf("init message: %s\n", message.c_str()); +} + void noui_connect() { // Connect bitcoind signal handlers uiInterface.ThreadSafeMessageBox.connect(noui_ThreadSafeMessageBox); uiInterface.ThreadSafeAskFee.connect(noui_ThreadSafeAskFee); + uiInterface.InitMessage.connect(noui_InitMessage); } diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 263fd52790..f4696d5a3b 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -62,8 +62,8 @@ AddressBookPage::AddressBookPage(Mode mode, Tabs tab, QWidget *parent) : } // Context menu actions - QAction *copyLabelAction = new QAction(tr("Copy &Label"), this); QAction *copyAddressAction = new QAction(ui->copyToClipboard->text(), this); + QAction *copyLabelAction = new QAction(tr("Copy &Label"), this); QAction *editAction = new QAction(tr("&Edit"), this); QAction *showQRCodeAction = new QAction(ui->showQRCode->text(), this); QAction *signMessageAction = new QAction(ui->signMessage->text(), this); @@ -78,7 +78,9 @@ AddressBookPage::AddressBookPage(Mode mode, Tabs tab, QWidget *parent) : if(tab == SendingTab) contextMenu->addAction(deleteAction); contextMenu->addSeparator(); +#ifdef USE_QRCODE contextMenu->addAction(showQRCodeAction); +#endif if(tab == ReceivingTab) contextMenu->addAction(signMessageAction); else if(tab == SendingTab) @@ -184,36 +186,31 @@ void AddressBookPage::on_signMessage_clicked() { QTableView *table = ui->tableView; QModelIndexList indexes = table->selectionModel()->selectedRows(AddressTableModel::Address); - QString addr; foreach (QModelIndex index, indexes) { - QVariant address = index.data(); - addr = address.toString(); + QString address = index.data().toString(); + emit signMessage(address); } - - emit signMessage(addr); } void AddressBookPage::on_verifyMessage_clicked() { QTableView *table = ui->tableView; QModelIndexList indexes = table->selectionModel()->selectedRows(AddressTableModel::Address); - QString addr; foreach (QModelIndex index, indexes) { - QVariant address = index.data(); - addr = address.toString(); + QString address = index.data().toString(); + emit verifyMessage(address); } - - emit verifyMessage(addr); } void AddressBookPage::on_newAddressButton_clicked() { if(!model) return; + EditAddressDialog dlg( tab == SendingTab ? EditAddressDialog::NewSendingAddress : @@ -230,6 +227,7 @@ void AddressBookPage::on_deleteButton_clicked() QTableView *table = ui->tableView; if(!table->selectionModel()) return; + QModelIndexList indexes = table->selectionModel()->selectedRows(); if(!indexes.isEmpty()) { @@ -341,11 +339,11 @@ void AddressBookPage::on_showQRCode_clicked() foreach (QModelIndex index, indexes) { - QString address = index.data().toString(), label = index.sibling(index.row(), 0).data(Qt::EditRole).toString(); + QString address = index.data().toString(); + QString label = index.sibling(index.row(), 0).data(Qt::EditRole).toString(); QRCodeDialog *dialog = new QRCodeDialog(address, label, tab == ReceivingTab, this); - if(optionsModel) - dialog->setModel(optionsModel); + dialog->setModel(optionsModel); dialog->setAttribute(Qt::WA_DeleteOnClose); dialog->show(); } diff --git a/src/qt/addressbookpage.h b/src/qt/addressbookpage.h index 6d3a734a16..c676d1e941 100644 --- a/src/qt/addressbookpage.h +++ b/src/qt/addressbookpage.h @@ -54,26 +54,31 @@ private: QString returnValue; QSortFilterProxyModel *proxyModel; QMenu *contextMenu; - QAction *deleteAction; + QAction *deleteAction; // to be able to explicitly disable it QString newAddressToSelect; private slots: + /** Delete currently selected address entry */ void on_deleteButton_clicked(); + /** Create a new address for receiving coins and / or add a new address book entry */ void on_newAddressButton_clicked(); /** Copy address of currently selected address entry to clipboard */ void on_copyToClipboard_clicked(); + /** Open the sign message tab in the Sign/Verify Message dialog with currently selected address */ void on_signMessage_clicked(); + /** Open the verify message tab in the Sign/Verify Message dialog with currently selected address */ void on_verifyMessage_clicked(); - void selectionChanged(); + /** Generate a QR Code from the currently selected address */ void on_showQRCode_clicked(); - /** Spawn contextual menu (right mouse menu) for address book entry */ - void contextualMenu(const QPoint &point); - - /** Copy label of currently selected address entry to clipboard */ + /** Copy label of currently selected address entry to clipboard (no button) */ void onCopyLabelAction(); - /** Edit currently selected address entry */ + /** Edit currently selected address entry (no button) */ void onEditAction(); + /** Set button states based on selected tab and selection */ + void selectionChanged(); + /** Spawn contextual menu (right mouse menu) for address book entry */ + void contextualMenu(const QPoint &point); /** New entry/entries were added to address table */ void selectNewAddress(const QModelIndex &parent, int begin, int /*end*/); diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index e65d3915ec..03b09cdceb 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -69,6 +69,8 @@ public: QString::fromStdString(address.ToString()))); } } + // qLowerBound() and qUpperBound() require our cachedAddressTable list to be sorted in asc order + qSort(cachedAddressTable.begin(), cachedAddressTable.end(), AddressTableEntryLessThan()); } void updateEntry(const QString &address, const QString &label, bool isMine, int status) @@ -208,7 +210,7 @@ QVariant AddressTableModel::data(const QModelIndex &index, int role) const return QVariant(); } -bool AddressTableModel::setData(const QModelIndex & index, const QVariant & value, int role) +bool AddressTableModel::setData(const QModelIndex &index, const QVariant &value, int role) { if(!index.isValid()) return false; @@ -221,18 +223,36 @@ bool AddressTableModel::setData(const QModelIndex & index, const QVariant & valu switch(index.column()) { case Label: + // Do nothing, if old label == new label + if(rec->label == value.toString()) + { + editStatus = NO_CHANGES; + return false; + } wallet->SetAddressBookName(CBitcoinAddress(rec->address.toStdString()).Get(), value.toString().toStdString()); - rec->label = value.toString(); break; case Address: + // Do nothing, if old address == new address + if(CBitcoinAddress(rec->address.toStdString()) == CBitcoinAddress(value.toString().toStdString())) + { + editStatus = NO_CHANGES; + return false; + } // Refuse to set invalid address, set error status and return false - if(!walletModel->validateAddress(value.toString())) + else if(!walletModel->validateAddress(value.toString())) { editStatus = INVALID_ADDRESS; return false; } + // Check for duplicate addresses to prevent accidental deletion of addresses, if you try + // to paste an existing address over another address (with a different label) + else if(wallet->mapAddressBook.count(CBitcoinAddress(value.toString().toStdString()).Get())) + { + editStatus = DUPLICATE_ADDRESS; + return false; + } // Double-check that we're not overwriting a receiving address - if(rec->type == AddressTableEntry::Sending) + else if(rec->type == AddressTableEntry::Sending) { { LOCK(wallet->cs_wallet); @@ -244,7 +264,6 @@ bool AddressTableModel::setData(const QModelIndex & index, const QVariant & valu } break; } - return true; } return false; @@ -262,7 +281,7 @@ QVariant AddressTableModel::headerData(int section, Qt::Orientation orientation, return QVariant(); } -Qt::ItemFlags AddressTableModel::flags(const QModelIndex & index) const +Qt::ItemFlags AddressTableModel::flags(const QModelIndex &index) const { if(!index.isValid()) return 0; @@ -279,7 +298,7 @@ Qt::ItemFlags AddressTableModel::flags(const QModelIndex & index) const return retval; } -QModelIndex AddressTableModel::index(int row, int column, const QModelIndex & parent) const +QModelIndex AddressTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); AddressTableEntry *data = priv->index(row); @@ -345,6 +364,7 @@ QString AddressTableModel::addRow(const QString &type, const QString &label, con { return QString(); } + // Add entry { LOCK(wallet->cs_wallet); @@ -353,7 +373,7 @@ QString AddressTableModel::addRow(const QString &type, const QString &label, con return QString::fromStdString(strAddress); } -bool AddressTableModel::removeRows(int row, int count, const QModelIndex & parent) +bool AddressTableModel::removeRows(int row, int count, const QModelIndex &parent) { Q_UNUSED(parent); AddressTableEntry *rec = priv->index(row); diff --git a/src/qt/addresstablemodel.h b/src/qt/addresstablemodel.h index 42974e3e1f..ae3e3b2f04 100644 --- a/src/qt/addresstablemodel.h +++ b/src/qt/addresstablemodel.h @@ -29,26 +29,27 @@ public: /** Return status of edit/insert operation */ enum EditStatus { - OK, - INVALID_ADDRESS, /**< Unparseable address */ - DUPLICATE_ADDRESS, /**< Address already in address book */ - WALLET_UNLOCK_FAILURE, /**< Wallet could not be unlocked to create new receiving address */ - KEY_GENERATION_FAILURE /**< Generating a new public key for a receiving address failed */ + OK, /**< Everything ok */ + NO_CHANGES, /**< No changes were made during edit operation */ + INVALID_ADDRESS, /**< Unparseable address */ + DUPLICATE_ADDRESS, /**< Address already in address book */ + WALLET_UNLOCK_FAILURE, /**< Wallet could not be unlocked to create new receiving address */ + KEY_GENERATION_FAILURE /**< Generating a new public key for a receiving address failed */ }; - static const QString Send; /**< Specifies send address */ - static const QString Receive; /**< Specifies receive address */ + static const QString Send; /**< Specifies send address */ + static const QString Receive; /**< Specifies receive address */ /** @name Methods overridden from QAbstractTableModel @{*/ int rowCount(const QModelIndex &parent) const; int columnCount(const QModelIndex &parent) const; QVariant data(const QModelIndex &index, int role) const; - bool setData(const QModelIndex & index, const QVariant & value, int role); + bool setData(const QModelIndex &index, const QVariant &value, int role); QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex & parent) const; - bool removeRows(int row, int count, const QModelIndex & parent = QModelIndex()); - Qt::ItemFlags flags(const QModelIndex & index) const; + QModelIndex index(int row, int column, const QModelIndex &parent) const; + bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()); + Qt::ItemFlags flags(const QModelIndex &index) const; /*@}*/ /* Add an address to the model. diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index c3701ced7f..e5526a6c09 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -86,6 +86,7 @@ static void InitMessage(const std::string &message) splashref->showMessage(QString::fromStdString(message), Qt::AlignBottom|Qt::AlignHCenter, QColor(255,255,200)); QApplication::instance()->processEvents(); } + printf("init message: %s\n", message.c_str()); } static void QueueShutdown() diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index 0d88aa47cb..5cfcb34b95 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -25,7 +25,7 @@ EditAddressDialog::EditAddressDialog(Mode mode, QWidget *parent) : break; case EditReceivingAddress: setWindowTitle(tr("Edit receiving address")); - ui->addressEdit->setDisabled(true); + ui->addressEdit->setEnabled(false); break; case EditSendingAddress: setWindowTitle(tr("Edit sending address")); @@ -44,6 +44,9 @@ EditAddressDialog::~EditAddressDialog() void EditAddressDialog::setModel(AddressTableModel *model) { this->model = model; + if(!model) + return; + mapper->setModel(model); mapper->addMapping(ui->labelEdit, AddressTableModel::Label); mapper->addMapping(ui->addressEdit, AddressTableModel::Address); @@ -58,6 +61,7 @@ bool EditAddressDialog::saveCurrentRow() { if(!model) return false; + switch(mode) { case NewReceivingAddress: @@ -82,35 +86,39 @@ void EditAddressDialog::accept() { if(!model) return; + if(!saveCurrentRow()) { switch(model->getEditStatus()) { - case AddressTableModel::DUPLICATE_ADDRESS: - QMessageBox::warning(this, windowTitle(), - tr("The entered address \"%1\" is already in the address book.").arg(ui->addressEdit->text()), - QMessageBox::Ok, QMessageBox::Ok); + case AddressTableModel::OK: + // Failed with unknown reason. Just reject. + break; + case AddressTableModel::NO_CHANGES: + // No changes were made during edit operation. Just reject. break; case AddressTableModel::INVALID_ADDRESS: QMessageBox::warning(this, windowTitle(), tr("The entered address \"%1\" is not a valid Bitcoin address.").arg(ui->addressEdit->text()), QMessageBox::Ok, QMessageBox::Ok); - return; + break; + case AddressTableModel::DUPLICATE_ADDRESS: + QMessageBox::warning(this, windowTitle(), + tr("The entered address \"%1\" is already in the address book.").arg(ui->addressEdit->text()), + QMessageBox::Ok, QMessageBox::Ok); + break; case AddressTableModel::WALLET_UNLOCK_FAILURE: QMessageBox::critical(this, windowTitle(), tr("Could not unlock wallet."), QMessageBox::Ok, QMessageBox::Ok); - return; + break; case AddressTableModel::KEY_GENERATION_FAILURE: QMessageBox::critical(this, windowTitle(), tr("New key generation failed."), QMessageBox::Ok, QMessageBox::Ok); - return; - case AddressTableModel::OK: - // Failed with unknown reason. Just reject. break; - } + } return; } QDialog::accept(); diff --git a/src/qt/editaddressdialog.h b/src/qt/editaddressdialog.h index 7ec053f135..0e4183bd52 100644 --- a/src/qt/editaddressdialog.h +++ b/src/qt/editaddressdialog.h @@ -27,15 +27,17 @@ public: }; explicit EditAddressDialog(Mode mode, QWidget *parent = 0); - ~EditAddressDialog(); + ~EditAddressDialog(); void setModel(AddressTableModel *model); void loadRow(int row); - void accept(); - QString getAddress() const; void setAddress(const QString &address); + +public slots: + void accept(); + private: bool saveCurrentRow(); diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index ff70ca24af..d4e73adf96 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -77,7 +77,8 @@ void setupAmountWidget(QLineEdit *widget, QWidget *parent) bool parseBitcoinURI(const QUrl &uri, SendCoinsRecipient *out) { - if(uri.scheme() != QString("bitcoin")) + // return if URI is not valid or is no bitcoin URI + if(!uri.isValid() || uri.scheme() != QString("bitcoin")) return false; SendCoinsRecipient rv; diff --git a/src/script.cpp b/src/script.cpp index 70adf1f9dc..5e5cd096cd 100644 --- a/src/script.cpp +++ b/src/script.cpp @@ -278,7 +278,8 @@ bool IsCanonicalSignature(const valtype &vchSig) { return error("Non-canonical signature: too short"); if (vchSig.size() > 73) return error("Non-canonical signature: too long"); - if (vchSig[vchSig.size() - 1] & 0x7C) + unsigned char nHashType = vchSig[vchSig.size() - 1] & (~(SIGHASH_ANYONECANPAY)); + if (nHashType < SIGHASH_ALL || nHashType > SIGHASH_SINGLE) return error("Non-canonical signature: unknown hashtype byte"); if (vchSig[0] != 0x30) return error("Non-canonical signature: wrong type"); @@ -340,7 +341,7 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, co // if (!script.GetOp(pc, opcode, vchPushValue)) return false; - if (vchPushValue.size() > 520) + if (vchPushValue.size() > MAX_SCRIPT_ELEMENT_SIZE) return false; if (opcode > OP_16 && ++nOpCount > 201) return false; @@ -670,7 +671,7 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, co valtype& vch2 = stacktop(-1); vch1.insert(vch1.end(), vch2.begin(), vch2.end()); popstack(stack); - if (stacktop(-1).size() > 520) + if (stacktop(-1).size() > MAX_SCRIPT_ELEMENT_SIZE) return false; } break; diff --git a/src/script.h b/src/script.h index 0b481eb605..4b29f6273c 100644 --- a/src/script.h +++ b/src/script.h @@ -17,6 +17,8 @@ class CCoins; class CTransaction; +static const unsigned int MAX_SCRIPT_ELEMENT_SIZE = 520; // bytes + /** Signature hash types/flags */ enum { diff --git a/src/wallet.cpp b/src/wallet.cpp index f49bfb5f8f..8b2f03212a 100644 --- a/src/wallet.cpp +++ b/src/wallet.cpp @@ -1636,29 +1636,38 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() { CWalletTx *pcoin = &walletEntry.second; - if (pcoin->vin.size() > 0 && IsMine(pcoin->vin[0])) + if (pcoin->vin.size() > 0) { + bool any_mine = false; // group all input addresses with each other BOOST_FOREACH(CTxIn txin, pcoin->vin) { CTxDestination address; + if(!IsMine(txin)) /* If this input isn't mine, ignore it */ + continue; if(!ExtractDestination(mapWallet[txin.prevout.hash].vout[txin.prevout.n].scriptPubKey, address)) continue; grouping.insert(address); + any_mine = true; } // group change with input addresses - BOOST_FOREACH(CTxOut txout, pcoin->vout) - if (IsChange(txout)) - { - CWalletTx tx = mapWallet[pcoin->vin[0].prevout.hash]; - CTxDestination txoutAddr; - if(!ExtractDestination(txout.scriptPubKey, txoutAddr)) - continue; - grouping.insert(txoutAddr); - } - groupings.insert(grouping); - grouping.clear(); + if (any_mine) + { + BOOST_FOREACH(CTxOut txout, pcoin->vout) + if (IsChange(txout)) + { + CTxDestination txoutAddr; + if(!ExtractDestination(txout.scriptPubKey, txoutAddr)) + continue; + grouping.insert(txoutAddr); + } + } + if (grouping.size() > 0) + { + groupings.insert(grouping); + grouping.clear(); + } } // group lone addrs by themselves |