diff options
37 files changed, 835 insertions, 268 deletions
diff --git a/configure.ac b/configure.ac index 7c0a9f4a94..c5d08a0285 100644 --- a/configure.ac +++ b/configure.ac @@ -590,17 +590,15 @@ fi if test x$use_boost = xyes; then -BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB" +BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB" dnl Boost >= 1.50 uses sleep_for rather than the now-deprecated sleep, however dnl it was broken from 1.50 to 1.52 when backed by nanosleep. Use sleep_for if dnl a working version is available, else fall back to sleep. sleep was removed dnl after 1.56. dnl If neither is available, abort. -dnl If sleep_for is used, boost_chrono becomes a requirement. -if test x$ax_cv_boost_chrono = xyes; then TEMP_LIBS="$LIBS" -LIBS="$BOOST_LIBS $BOOST_CHRONO_LIB $LIBS" +LIBS="$BOOST_LIBS $LIBS" TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -613,12 +611,11 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ choke me #endif ]])], - [boost_sleep=yes; BOOST_LIBS="$BOOST_LIBS $BOOST_CHRONO_LIB"; + [boost_sleep=yes; AC_DEFINE(HAVE_WORKING_BOOST_SLEEP_FOR, 1, [Define this symbol if boost sleep_for works])], [boost_sleep=no]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" -fi if test x$boost_sleep != xyes; then TEMP_LIBS="$LIBS" diff --git a/depends/Makefile b/depends/Makefile index 05ef33f2ee..ef5a20e6c3 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -91,12 +91,12 @@ include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) -$(host_prefix)/.stamp_$(final_build_id): | $(native_packages) $(packages) +$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) - $(AT)echo copying packages: $| + $(AT)echo copying packages: $^ $(AT)echo to: $(@D) - $(AT)cd $(@D); $(foreach package,$|, tar xf $($(package)_cached); ) + $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) @@ -121,8 +121,35 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ $< > $@ $(AT)touch $@ -install: $(host_prefix)/share/config.site -download-one: $(all_sources) + +define check_or_remove_cached + mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \ + $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \ + ( rm -f $($(package)_cached_checksum); \ + if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi ) +endef + +define check_or_remove_sources + mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ + $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ + ( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \ + rm -f $($(package)_all_sources) $($(1)_fetched)) +endef + +check-packages: + @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));) +check-sources: + @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) + +$(host_prefix)/share/config.site: check-packages + +check-packages: check-sources + +install: check-packages $(host_prefix)/share/config.site + + +download-one: check-sources $(all_sources) + download-osx: @$(MAKE) -s HOST=x86_64-apple-darwin11 download-one download-linux: @@ -130,4 +157,5 @@ download-linux: download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win -.PHONY: install cached download-one download-osx download-linux download-win download + +.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources diff --git a/depends/README.md b/depends/README.md new file mode 100644 index 0000000000..2dc0b9e47e --- /dev/null +++ b/depends/README.md @@ -0,0 +1,56 @@ +### Usage + +To build dependencies for the current arch+OS: + + make + +To build for another arch/OS: + + make HOST=host-platform-triplet + +For example: + + make HOST=x86_64-w64-mingw32 -j4 + +A prefix will be generated that's suitable for plugging into Bitcoin's +configure. In the above example, a dir named i686-w64-mingw32 will be +created. To use it for Bitcoin: + + ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32 + +Common `host-platform-triplets` for cross compilation are: + +- `i686-w64-mingw32` for Win32 +- `x86_64-w64-mingw32` for Win64 +- `x86_64-apple-darwin11` for MacOSX +- `arm-linux-gnueabihf` for Linux ARM + +No other options are needed, the paths are automatically configured. + +Dependency Options: +The following can be set when running make: make FOO=bar + + SOURCES_PATH: downloaded sources will be placed here + BASE_CACHE: built packages will be placed here + SDK_PATH: Path where sdk's can be found (used by OSX) + FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up + NO_QT: Don't download/build/cache qt and its dependencies + NO_WALLET: Don't download/build/cache libs needed to enable the wallet + NO_UPNP: Don't download/build/cache packages needed for enabling upnp + DEBUG: disable some optimizations and enable more runtime checking + +If some packages are not built, for example `make NO_WALLET=1`, the appropriate +options will be passed to bitcoin's configure. In this case, `--disable-wallet`. + +Additional targets: + + download: run 'make download' to fetch all sources without building them + download-osx: run 'make download-osx' to fetch all sources needed for osx builds + download-win: run 'make download-win' to fetch all sources needed for win builds + download-linux: run 'make download-linux' to fetch all sources needed for linux builds + +### Other documentation + +- [description.md](description.md): General description of the depends system +- [packages.md](packages.md): Steps for adding packages + diff --git a/depends/README.usage b/depends/README.usage deleted file mode 100644 index 24e1231d82..0000000000 --- a/depends/README.usage +++ /dev/null @@ -1,34 +0,0 @@ -To build dependencies for the current arch+OS: - make -To build for another arch/OS: - make HOST=host-platform-triplet && make HOST=host-platform-triplet - (For example: make HOST=i686-w64-mingw32 -j4) - -A prefix will be generated that's suitable for plugging into Bitcoin's -configure. In the above example, a dir named i686-w64-mingw32 will be -created. To use it for Bitcoin: - -./configure --prefix=`pwd`/depends/i686-w64-mingw32 - -No other options are needed, the paths are automatically configured. - -Dependency Options: -The following can be set when running make: make FOO=bar - -SOURCES_PATH: downloaded sources will be placed here -BASE_CACHE: built packages will be placed here -SDK_PATH: Path where sdk's can be found (used by OSX) -FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up -NO_QT: Don't download/build/cache qt and its dependencies -NO_WALLET: Don't download/build/cache libs needed to enable the wallet -NO_UPNP: Don't download/build/cache packages needed for enabling upnp -DEBUG: disable some optimizations and enable more runtime checking - -If some packages are not built, for example 'make NO_WALLET=1', the appropriate -options will be passed to bitcoin's configure. In this case, --disable-wallet. - -Additional targets: -download: run 'make download' to fetch all sources without building them -download-osx: run 'make download-osx' to fetch all sources needed for osx builds -download-win: run 'make download-win' to fetch all sources needed for win builds -download-linux: run 'make download-linux' to fetch all sources needed for linux builds diff --git a/depends/README b/depends/description.md index 55e7222697..74f9ef3f20 100644 --- a/depends/README +++ b/depends/description.md @@ -1,9 +1,7 @@ -This is a system of building and caching dependencies necessary for building -Bitcoin. - +This is a system of building and caching dependencies necessary for building Bitcoin. There are several features that make it different from most similar systems: -- It is designed to be builder and host agnostic +### It is designed to be builder and host agnostic In theory, binaries for any target OS/architecture can be created, from a builder running any OS/architecture. In practice, build-side tools must be @@ -11,18 +9,18 @@ specified when the defaults don't fit, and packages must be amended to work on new hosts. For now, a build architecture of x86_64 is assumed, either on Linux or OSX. -- No reliance on timestamps +### No reliance on timestamps File presence is used to determine what needs to be built. This makes the results distributable and easily digestable by automated builders. -- Each build only has its specified dependencies available at build-time. +### Each build only has its specified dependencies available at build-time. For each build, the sysroot is wiped and the (recursive) dependencies are installed. This makes each build deterministic, since there will never be any unknown files available to cause side-effects. -- Each package is cached and only rebuilt as needed. +### Each package is cached and only rebuilt as needed. Before building, a unique build-id is generated for each package. This id consists of a hash of all files used to build the package (Makefiles, packages, @@ -32,7 +30,7 @@ any other package that depends on it. If any of the main makefiles (Makefile, funcs.mk, etc) are changed, all packages will be rebuilt. After building, the results are cached into a tarball that can be re-used and distributed. -- Package build results are (relatively) deterministic. +### Package build results are (relatively) deterministic. Each package is configured and patched so that it will yield the same build-results with each consequent build, within a reasonable set of @@ -41,13 +39,13 @@ beyond the scope of this system. Additionally, the toolchain itself must be capable of deterministic results. When revisions are properly bumped, a cached build should represent an exact single payload. -- Sources are fetched and verified automatically +### Sources are fetched and verified automatically Each package must define its source location and checksum. The build will fail if the fetched source does not match. Sources may be pre-seeded and/or cached as desired. -- Self-cleaning +### Self-cleaning Build and staging dirs are wiped after use, and any previous version of a cached result is removed following a successful build. Automated builders diff --git a/depends/funcs.mk b/depends/funcs.mk index b407737f7f..050a9b1321 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -53,12 +53,14 @@ $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix) $(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) $(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) $(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir) +$(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps -$(1)_fetched=$$($(1)_source_dir)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name) +$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned @@ -154,7 +156,10 @@ endef define int_add_cmds $($(1)_fetched): $(AT)mkdir -p $$(@D) $(SOURCES_PATH) + $(AT)rm -f $$@ + $(AT)touch $$@ $(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) $(AT)touch $$@ $($(1)_extracted): | $($(1)_fetched) $(AT)echo Extracting $(1)... @@ -195,10 +200,12 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) $(AT)rm -rf $$(@D) && mkdir -p $$(@D) $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) $(AT)rm -rf $($(1)_staging_dir) +$($(1)_cached_checksum): $($(1)_cached) + $(AT)cd $$(@D); $(build_SHA256SUM) $$(<F) > $$(@) .PHONY: $(1) -$(1): | $($(1)_cached) -.SECONDARY: $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) +$(1): | $($(1)_cached_checksum) +.SECONDARY: $($(1)_cached) $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) endef diff --git a/depends/README.packages b/depends/packages.md index 5ab7ed7dee..7c80362509 100644 --- a/depends/README.packages +++ b/depends/packages.md @@ -4,119 +4,137 @@ variables, and defining build commands. The package "mylib" will be used here as an example General tips: -mylib_foo is written as $(package)_foo in order to make recipes more similar. +- mylib_foo is written as $(package)_foo in order to make recipes more similar. -Identifiers: +## Identifiers Each package is required to define at least these variables: - $(package)_version: + + $(package)_version: Version of the upstream library or program. If there is no version, a placeholder such as 1.0 can be used. - $(package)_download_path: + + $(package)_download_path: Location of the upstream source, without the file-name. Usually http or ftp. - $(package)_file_name: + + $(package)_file_name: The upstream source filename available at the download path. - $(package)_sha256_hash: + + $(package)_sha256_hash: The sha256 hash of the upstream file These variables are optional: - $(package)_build_subdir: + + $(package)_build_subdir: cd to this dir before running configure/build/stage commands. - $(package)_download_file: + + $(package)_download_file: The file-name of the upstream source if it differs from how it should be stored locally. This can be used to avoid storing file-names with strange characters. - $(package)_dependencies: + + $(package)_dependencies: Names of any other packages that this one depends on. - $(package)_patches: + + $(package)_patches: Filenames of any patches needed to build the package + $(package)_extra_sources: + Any extra files that will be fetched via $(package)_fetch_cmds. These are + specified so that they can be fetched and verified via 'make download'. + -Build Variables: +## Build Variables: After defining the main identifiers, build variables may be added or customized before running the build commands. They should be added to a function called $(package)_set_vars. For example: -define $(package)_set_vars -... -endef + define $(package)_set_vars + ... + endef Most variables can be prefixed with the host, architecture, or both, to make the modifications specific to that case. For example: - Universal: $(package)_cc=gcc - Linux only: $(package)_linux_cc=gcc - x86_64 only: $(package)_x86_64_cc = gcc - x86_64 linux only: $(package)_x86_64_linux_cc = gcc + Universal: $(package)_cc=gcc + Linux only: $(package)_linux_cc=gcc + x86_64 only: $(package)_x86_64_cc = gcc + x86_64 linux only: $(package)_x86_64_linux_cc = gcc These variables may be set to override or append their default values. - $(package)_cc - $(package)_cxx - $(package)_objc - $(package)_objcxx - $(package)_ar - $(package)_ranlib - $(package)_libtool - $(package)_nm - $(package)_cflags - $(package)_cxxflags - $(package)_ldflags - $(package)_cppflags - $(package)_config_env - $(package)_build_env - $(package)_stage_env - $(package)_build_opts - $(package)_config_opts + + $(package)_cc + $(package)_cxx + $(package)_objc + $(package)_objcxx + $(package)_ar + $(package)_ranlib + $(package)_libtool + $(package)_nm + $(package)_cflags + $(package)_cxxflags + $(package)_ldflags + $(package)_cppflags + $(package)_config_env + $(package)_build_env + $(package)_stage_env + $(package)_build_opts + $(package)_config_opts The *_env variables are used to add environment variables to the respective commands. Many variables respect a debug/release suffix as well, in order to use them for only the appropriate build config. For example: - $(package)_cflags_release = -O3 - $(package)_cflags_i686_debug = -g - $(package)_config_opts_release = --disable-debug + + $(package)_cflags_release = -O3 + $(package)_cflags_i686_debug = -g + $(package)_config_opts_release = --disable-debug These will be used in addition to the options that do not specify debug/release. All builds are considered to be release unless DEBUG=1 is set by -the user. +the user. Other variables may be defined as needed. -Other variables may be defined as needed. - -Build commands: +## Build commands: For each build, a unique build dir and staging dir are created. For example, - work/build/mylib/1.0-1adac830f6e and work/staging/mylib/1.0-1adac830f6e. + `work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. The following build commands are available for each recipe: - $(package)_fetch_cmds: + $(package)_fetch_cmds: Runs from: build dir Fetch the source file. If undefined, it will be fetched and verified against its hash. - $(package)_extract_cmds: + + $(package)_extract_cmds: Runs from: build dir Verify the source file against its hash and extract it. If undefined, the source is assumed to be a tarball. - $(package)_preprocess_cmds: + + $(package)_preprocess_cmds: Runs from: build dir/$(package)_build_subdir Preprocess the source as necessary. If undefined, does nothing. - $(package)_config_cmds: + + $(package)_config_cmds: Runs from: build dir/$(package)_build_subdir Configure the source. If undefined, does nothing. - $(package)_build_cmds: + + $(package)_build_cmds: Runs from: build dir/$(package)_build_subdir Build the source. If undefined, does nothing. - $(package)_stage_cmds: + + $(package)_stage_cmds: Runs from: build dir/$(package)_build_subdir Stage the build results. If undefined, does nothing. The following variables are available for each recipe: - $(1)_staging_dir: package's destination sysroot path - $(1)_staging_prefix_dir: prefix path inside of the package's staging dir - $(1)_extract_dir: path to the package's extracted sources - $(1)_build_dir: path where configure/build/stage commands will be run - $(1)_patch_dir: path where the package's patches (if any) are found + + $(1)_staging_dir: package's destination sysroot path + $(1)_staging_prefix_dir: prefix path inside of the package's staging dir + $(1)_extract_dir: path to the package's extracted sources + $(1)_build_dir: path where configure/build/stage commands will be run + $(1)_patch_dir: path where the package's patches (if any) are found Notes on build commands: @@ -125,4 +143,5 @@ configure step to (usually) correctly configure automatically. Any $($(package)_config_opts) will be appended. Most autotools projects can be properly staged using: - $(MAKE) DESTDIR=$($(package)_staging_dir) install + + $(MAKE) DESTDIR=$($(package)_staging_dir) install diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 951ad4fb29..1c1bcf199a 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -9,6 +9,8 @@ $(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_versi $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_sha256_hash=60d8f69f032d62ef61bf527857ebb933741ec3352d4d328c5516aa520662dab7 +$(package)_extra_sources=$($(package)_clang_file_name) + define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) diff --git a/doc/assets-attribution.md b/doc/assets-attribution.md index c860cdc534..ebba64a61a 100644 --- a/doc/assets-attribution.md +++ b/doc/assets-attribution.md @@ -22,6 +22,7 @@ The following is a list of assets used in the bitcoin source and their proper at src/qt/res/icons/receive.png, src/qt/res/icons/remove.png, src/qt/res/icons/send.png, src/qt/res/icons/synced.png, src/qt/res/icons/transaction*.png, src/qt/res/icons/tx_output.png, + src/qt/res/icons/warning.png Jonas Schnelli ----------------------- diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py index f26cbee1e2..85fd1c982a 100755 --- a/qa/rpc-tests/pruning.py +++ b/qa/rpc-tests/pruning.py @@ -7,7 +7,8 @@ # Test pruning code # ******** # WARNING: -# This test uses 4GB of disk space and takes in excess of 30 mins to run +# This test uses 4GB of disk space. +# This test takes 30 mins or more (up to 2 hours) # ******** from test_framework import BitcoinTestFramework @@ -51,11 +52,11 @@ class PruneTest(BitcoinTestFramework): self.is_network_split = False # Create nodes 0 and 1 to mine - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)) + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) # Create node 2 to test pruning - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=300)) + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900)) self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/" self.address[0] = self.nodes[0].getnewaddress() @@ -108,7 +109,7 @@ class PruneTest(BitcoinTestFramework): # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300) + self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900) # Mine 24 blocks in node 1 self.utxo = self.nodes[1].listunspent() for i in xrange(24): @@ -135,7 +136,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) stop_node(self.nodes[1],1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300) + self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) height = self.nodes[1].getblockcount() print "Current block height:", height @@ -158,7 +159,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node1 to clear those giant tx's from mempool stop_node(self.nodes[1],1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300) + self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) print "Generating new longer chain of 300 more blocks" self.nodes[1].generate(300) @@ -223,7 +224,7 @@ class PruneTest(BitcoinTestFramework): waitstart = time.time() while self.nodes[2].getblockcount() < goalbestheight: time.sleep(0.1) - if time.time() - waitstart > 300: + if time.time() - waitstart > 900: raise AssertionError("Node 2 didn't reorg to proper height") assert(self.nodes[2].getbestblockhash() == goalbesthash) # Verify we can now have the data for a block previously pruned @@ -256,7 +257,7 @@ class PruneTest(BitcoinTestFramework): def run_test(self): - print "Warning! This test requires 4GB of disk space and takes over 30 mins" + print "Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)" print "Mining a big blockchain of 995 blocks" self.create_big_chain() # Chain diagram key: diff --git a/src/Makefile.am b/src/Makefile.am index 24fd76e477..93ad26acf8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -102,6 +102,7 @@ BITCOIN_CORE_H = \ leveldbwrapper.h \ limitedmap.h \ main.h \ + memusage.h \ merkleblock.h \ miner.h \ mruset.h \ @@ -118,6 +119,7 @@ BITCOIN_CORE_H = \ rpcclient.h \ rpcprotocol.h \ rpcserver.h \ + scheduler.h \ script/interpreter.h \ script/script.h \ script/script_error.h \ @@ -259,6 +261,7 @@ libbitcoin_common_a_SOURCES = \ primitives/transaction.cpp \ protocol.cpp \ pubkey.cpp \ + scheduler.cpp \ script/interpreter.cpp \ script/script.cpp \ script/script_error.cpp \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 31fe3a9f69..6b7c42285d 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -256,6 +256,7 @@ RES_ICONS = \ qt/res/icons/tx_input.png \ qt/res/icons/tx_output.png \ qt/res/icons/tx_mined.png \ + qt/res/icons/warning.png \ qt/res/icons/verify.png BITCOIN_QT_CPP = \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 65112f6f5d..0997148117 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -61,6 +61,7 @@ BITCOIN_TESTS =\ test/pow_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ + test/scheduler_tests.cpp \ test/script_P2SH_tests.cpp \ test/script_tests.cpp \ test/scriptnum_tests.cpp \ diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index eeca8655c9..cce687ac98 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -8,6 +8,7 @@ #include "init.h" #include "main.h" #include "noui.h" +#include "scheduler.h" #include "util.h" #include <boost/algorithm/string/predicate.hpp> @@ -55,6 +56,7 @@ void WaitForShutdown(boost::thread_group* threadGroup) bool AppInit(int argc, char* argv[]) { boost::thread_group threadGroup; + CScheduler scheduler; bool fRet = false; @@ -142,7 +144,7 @@ bool AppInit(int argc, char* argv[]) #endif SoftSetBoolArg("-server", true); - fRet = AppInit2(threadGroup); + fRet = AppInit2(threadGroup, scheduler); } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInit()"); diff --git a/src/chainparams.h b/src/chainparams.h index bb95b3aa4a..6274bf3eb4 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -50,9 +50,6 @@ public: const std::vector<unsigned char>& AlertKey() const { return vAlertPubKey; } int GetDefaultPort() const { return nDefaultPort; } int SubsidyHalvingInterval() const { return consensus.nSubsidyHalvingInterval; } - int EnforceBlockUpgradeMajority() const { return consensus.nMajorityEnforceBlockUpgrade; } - int RejectBlockOutdatedMajority() const { return consensus.nMajorityRejectBlockOutdated; } - int ToCheckBlockUpgradeMajority() const { return consensus.nMajorityWindow; } /** Used if GenerateBitcoins is called with a negative number of threads */ int DefaultMinerThreads() const { return nMinerThreads; } diff --git a/src/coins.cpp b/src/coins.cpp index d79e29951b..a41d5a310d 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -4,6 +4,7 @@ #include "coins.h" +#include "memusage.h" #include "random.h" #include <assert.h> @@ -57,13 +58,17 @@ bool CCoinsViewBacked::GetStats(CCoinsStats &stats) const { return base->GetStat CCoinsKeyHasher::CCoinsKeyHasher() : salt(GetRandHash()) {} -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false) { } +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } CCoinsViewCache::~CCoinsViewCache() { assert(!hasModifier); } +size_t CCoinsViewCache::DynamicMemoryUsage() const { + return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; +} + CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { CCoinsMap::iterator it = cacheCoins.find(txid); if (it != cacheCoins.end()) @@ -78,6 +83,7 @@ CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } + cachedCoinsUsage += memusage::DynamicUsage(ret->second.coins); return ret; } @@ -93,6 +99,7 @@ bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { assert(!hasModifier); std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); + size_t cachedCoinUsage = 0; if (ret.second) { if (!base->GetCoins(txid, ret.first->second.coins)) { // The parent view does not have this entry; mark it as fresh. @@ -102,10 +109,12 @@ CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { // The parent view only has a pruned entry for this; mark it as fresh. ret.first->second.flags = CCoinsCacheEntry::FRESH; } + } else { + cachedCoinUsage = memusage::DynamicUsage(ret.first->second.coins); } // Assume that whenever ModifyCoins is called, the entry will be modified. ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first); + return CCoinsModifier(*this, ret.first, cachedCoinUsage); } const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { @@ -150,6 +159,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn assert(it->second.flags & CCoinsCacheEntry::FRESH); CCoinsCacheEntry& entry = cacheCoins[it->first]; entry.coins.swap(it->second.coins); + cachedCoinsUsage += memusage::DynamicUsage(entry.coins); entry.flags = CCoinsCacheEntry::DIRTY | CCoinsCacheEntry::FRESH; } } else { @@ -157,10 +167,13 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. + cachedCoinsUsage -= memusage::DynamicUsage(itUs->second.coins); cacheCoins.erase(itUs); } else { // A normal modification. + cachedCoinsUsage -= memusage::DynamicUsage(itUs->second.coins); itUs->second.coins.swap(it->second.coins); + cachedCoinsUsage += memusage::DynamicUsage(itUs->second.coins); itUs->second.flags |= CCoinsCacheEntry::DIRTY; } } @@ -175,6 +188,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn bool CCoinsViewCache::Flush() { bool fOk = base->BatchWrite(cacheCoins, hashBlock); cacheCoins.clear(); + cachedCoinsUsage = 0; return fOk; } @@ -232,7 +246,7 @@ double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight) const return tx.ComputePriority(dResult); } -CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_) : cache(cache_), it(it_) { +CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage) : cache(cache_), it(it_), cachedCoinUsage(usage) { assert(!cache.hasModifier); cache.hasModifier = true; } @@ -242,7 +256,11 @@ CCoinsModifier::~CCoinsModifier() assert(cache.hasModifier); cache.hasModifier = false; it->second.coins.Cleanup(); + cache.cachedCoinsUsage -= cachedCoinUsage; // Subtract the old usage if ((it->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { cache.cacheCoins.erase(it); + } else { + // If the coin still exists after the modification, add the new usage + cache.cachedCoinsUsage += memusage::DynamicUsage(it->second.coins); } } diff --git a/src/coins.h b/src/coins.h index fe2eaa08e5..a4671645df 100644 --- a/src/coins.h +++ b/src/coins.h @@ -7,6 +7,7 @@ #define BITCOIN_COINS_H #include "compressor.h" +#include "memusage.h" #include "serialize.h" #include "uint256.h" @@ -252,6 +253,15 @@ public: return false; return true; } + + size_t DynamicMemoryUsage() const { + size_t ret = memusage::DynamicUsage(vout); + BOOST_FOREACH(const CTxOut &out, vout) { + const std::vector<unsigned char> *script = &out.scriptPubKey; + ret += memusage::DynamicUsage(*script); + } + return ret; + } }; class CCoinsKeyHasher @@ -356,7 +366,8 @@ class CCoinsModifier private: CCoinsViewCache& cache; CCoinsMap::iterator it; - CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_); + size_t cachedCoinUsage; // Cached memory usage of the CCoins object before modification + CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage); public: CCoins* operator->() { return &it->second.coins; } @@ -372,6 +383,7 @@ protected: /* Whether this cache has an active modifier. */ bool hasModifier; + /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". @@ -379,6 +391,9 @@ protected: mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; + /* Cached dynamic memory usage for the inner CCoins objects. */ + mutable size_t cachedCoinsUsage; + public: CCoinsViewCache(CCoinsView *baseIn); ~CCoinsViewCache(); @@ -414,6 +429,9 @@ public: //! Calculate the size of the cache (in number of transactions) unsigned int GetCacheSize() const; + //! Calculate the size of the cache (in bytes) + size_t DynamicMemoryUsage() const; + /** * Amount of bitcoins coming in to a transaction * Note that lightweight clients may not know anything besides the hash of previous transactions, diff --git a/src/init.cpp b/src/init.cpp index e2bc36a8e4..3c3b3fb231 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -19,6 +19,7 @@ #include "net.h" #include "rpcserver.h" #include "script/standard.h" +#include "scheduler.h" #include "txdb.h" #include "ui_interface.h" #include "util.h" @@ -564,7 +565,7 @@ bool InitSanityCheck(void) /** Initialize bitcoin. * @pre Parameters should be parsed and config file should be read. */ -bool AppInit2(boost::thread_group& threadGroup) +bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -890,6 +891,10 @@ bool AppInit2(boost::thread_group& threadGroup) threadGroup.create_thread(&ThreadScriptCheck); } + // Start the lightweight task scheduler thread + CScheduler::Function serviceLoop = boost::bind(&CScheduler::serviceQueue, &scheduler); + threadGroup.create_thread(boost::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections * that the server is there and will be ready later). Warmup mode will @@ -1056,18 +1061,20 @@ bool AppInit2(boost::thread_group& threadGroup) } // cache size calculations - size_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); - if (nTotalCache < (nMinDbCache << 20)) - nTotalCache = (nMinDbCache << 20); // total cache cannot be less than nMinDbCache - else if (nTotalCache > (nMaxDbCache << 20)) - nTotalCache = (nMaxDbCache << 20); // total cache cannot be greater than nMaxDbCache - size_t nBlockTreeDBCache = nTotalCache / 8; + int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); + nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache + nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache + int64_t nBlockTreeDBCache = nTotalCache / 8; if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false)) nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB nTotalCache -= nBlockTreeDBCache; - size_t nCoinDBCache = nTotalCache / 2; // use half of the remaining cache for coindb cache + int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nTotalCache -= nCoinDBCache; - nCoinCacheSize = nTotalCache / 300; // coins in memory require around 300 bytes + nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache + LogPrintf("Cache configuration:\n"); + LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); + LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); + LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024)); bool fLoaded = false; while (!fLoaded) { @@ -1373,7 +1380,7 @@ bool AppInit2(boost::thread_group& threadGroup) LogPrintf("mapAddressBook.size() = %u\n", pwalletMain ? pwalletMain->mapAddressBook.size() : 0); #endif - StartNode(threadGroup); + StartNode(threadGroup, scheduler); #ifdef ENABLE_WALLET // Generate coins in the background diff --git a/src/init.h b/src/init.h index d451f65be9..dcb2b29360 100644 --- a/src/init.h +++ b/src/init.h @@ -8,6 +8,7 @@ #include <string> +class CScheduler; class CWallet; namespace boost @@ -20,7 +21,7 @@ extern CWallet* pwalletMain; void StartShutdown(); bool ShutdownRequested(); void Shutdown(); -bool AppInit2(boost::thread_group& threadGroup); +bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler); /** The help message mode determines what help message to show */ enum HelpMessageMode { diff --git a/src/main.cpp b/src/main.cpp index 79ee4e55ec..5fd9c62486 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -57,7 +57,7 @@ bool fPruneMode = false; bool fIsBareMultisigStd = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = true; -unsigned int nCoinCacheSize = 5000; +size_t nCoinCacheUsage = 5000 * 300; uint64_t nPruneTarget = 0; /** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ @@ -75,10 +75,9 @@ void EraseOrphansFor(NodeId peer); /** * Returns true if there are nRequired or more blocks of minVersion or above - * in the last Params().ToCheckBlockUpgradeMajority() blocks, starting at pstart - * and going backwards. + * in the last Consensus::Params::nMajorityWindow blocks, starting at pstart and going backwards. */ -static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned int nRequired); +static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams); static void CheckBlockIndex(); /** Constant stuff for coinbase transactions we create: */ @@ -1747,7 +1746,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; // Start enforcing the DERSIG (BIP66) rules, for block.nVersion=3 blocks, when 75% of the network has upgraded: - if (block.nVersion >= 3 && IsSuperMajority(3, pindex->pprev, Params().EnforceBlockUpgradeMajority())) { + if (block.nVersion >= 3 && IsSuperMajority(3, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { flags |= SCRIPT_VERIFY_DERSIG; } @@ -1880,6 +1879,8 @@ enum FlushStateMode { bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { LOCK2(cs_main, cs_LastBlockFile); static int64_t nLastWrite = 0; + static int64_t nLastFlush = 0; + static int64_t nLastSetChain = 0; std::set<int> setFilesToPrune; bool fFlushForPrune = false; try { @@ -1893,16 +1894,32 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { } } } - if ((mode == FLUSH_STATE_ALWAYS) || - ((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) || - (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000) || - fFlushForPrune) { - // Typical CCoins structures on disk are around 100 bytes in size. - // Pushing a new one to the database can cause it to be written - // twice (once in the log, and once in the tables). This is already - // an overestimation, as most will delete an existing entry or - // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(100 * 2 * 2 * pcoinsTip->GetCacheSize())) + int64_t nNow = GetTimeMicros(); + // Avoid writing/flushing immediately after startup. + if (nLastWrite == 0) { + nLastWrite = nNow; + } + if (nLastFlush == 0) { + nLastFlush = nNow; + } + if (nLastSetChain == 0) { + nLastSetChain = nNow; + } + size_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + // The cache is large and close to the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize * (10.0/9) > nCoinCacheUsage; + // The cache is over the limit, we have to write now. + bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nCoinCacheUsage; + // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. + bool fPeriodicWrite = mode == FLUSH_STATE_PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; + // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. + bool fPeriodicFlush = mode == FLUSH_STATE_PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; + // Combine all conditions that result in a full cache flush. + bool fDoFullFlush = (mode == FLUSH_STATE_ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; + // Write blocks and block index to disk. + if (fDoFullFlush || fPeriodicWrite) { + // Depend on nMinDiskSpace to ensure we can write block index + if (!CheckDiskSpace(0)) return state.Error("out of disk space"); // First make sure all block and undo data is flushed to disk. FlushBlockFile(); @@ -1924,21 +1941,31 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { return state.Abort("Files to write to block index database"); } } - // Flush the chainstate (which may refer to block index entries). - if (!pcoinsTip->Flush()) - return state.Abort("Failed to write to coin database"); - // Finally remove any pruned files if (fFlushForPrune) { UnlinkPrunedFiles(setFilesToPrune); fCheckForPruning = false; } - + nLastWrite = nNow; + } + // Flush best chain related state. This can only be done if the blocks / block index write was also done. + if (fDoFullFlush) { + // Typical CCoins structures on disk are around 128 bytes in size. + // Pushing a new one to the database can cause it to be written + // twice (once in the log, and once in the tables). This is already + // an overestimation, as most will delete an existing entry or + // overwrite one. Still, use a conservative safety factor of 2. + if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + return state.Error("out of disk space"); + // Flush the chainstate (which may refer to block index entries). + if (!pcoinsTip->Flush()) + return state.Abort("Failed to write to coin database"); + nLastFlush = nNow; + } + if ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000) { // Update best block in wallet (so we can detect restored wallets). - if (mode != FLUSH_STATE_IF_NEEDED) { - GetMainSignals().SetBestChain(chainActive.GetLocator()); - } - nLastWrite = GetTimeMicros(); + GetMainSignals().SetBestChain(chainActive.GetLocator()); + nLastSetChain = nNow; } } catch (const std::runtime_error& e) { return state.Abort(std::string("System error while flushing: ") + e.what()); @@ -1966,10 +1993,10 @@ void static UpdateTip(CBlockIndex *pindexNew) { nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); - LogPrintf("%s: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f cache=%u\n", __func__, + LogPrintf("%s: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f cache=%.1fMiB(%utx)\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), (unsigned int)pcoinsTip->GetCacheSize()); + Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); cvBlockChange.notify_all(); @@ -2644,18 +2671,14 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta } // Reject block.nVersion=1 blocks when 95% (75% on testnet) of the network has upgraded: - if (block.nVersion < 2 && IsSuperMajority(2, pindexPrev, consensusParams.nMajorityRejectBlockOutdated)) - { + if (block.nVersion < 2 && IsSuperMajority(2, pindexPrev, consensusParams.nMajorityRejectBlockOutdated, consensusParams)) return state.Invalid(error("%s: rejected nVersion=1 block", __func__), REJECT_OBSOLETE, "bad-version"); - } // Reject block.nVersion=2 blocks when 95% (75% on testnet) of the network has upgraded: - if (block.nVersion < 3 && IsSuperMajority(3, pindexPrev, consensusParams.nMajorityRejectBlockOutdated)) - { + if (block.nVersion < 3 && IsSuperMajority(3, pindexPrev, consensusParams.nMajorityRejectBlockOutdated, consensusParams)) return state.Invalid(error("%s : rejected nVersion=2 block", __func__), REJECT_OBSOLETE, "bad-version"); - } return true; } @@ -2663,6 +2686,7 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex * const pindexPrev) { const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; + const Consensus::Params& consensusParams = Params().GetConsensus(); // Check that all transactions are finalized BOOST_FOREACH(const CTransaction& tx, block.vtx) @@ -2672,7 +2696,7 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): - if (block.nVersion >= 2 && IsSuperMajority(2, pindexPrev, Params().EnforceBlockUpgradeMajority())) + if (block.nVersion >= 2 && IsSuperMajority(2, pindexPrev, consensusParams.nMajorityEnforceBlockUpgrade, consensusParams)) { CScript expect = CScript() << nHeight; if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || @@ -2779,11 +2803,10 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, return true; } -static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned int nRequired) +static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams) { - unsigned int nToCheck = Params().ToCheckBlockUpgradeMajority(); unsigned int nFound = 0; - for (unsigned int i = 0; i < nToCheck && nFound < nRequired && pstart != NULL; i++) + for (int i = 0; i < consensusParams.nMajorityWindow && nFound < nRequired && pstart != NULL; i++) { if (pstart->nVersion >= minVersion) ++nFound; @@ -2932,7 +2955,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) return; } - unsigned int nLastBlockWeMustKeep = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + unsigned int nLastBlockWeCanPrune = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files // So we should leave a buffer under our target to account for another allocation @@ -2952,7 +2975,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) break; // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip - if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeMustKeep) + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) break; PruneOneBlockFile(fileNumber); @@ -2963,10 +2986,10 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) } } - LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB min_must_keep=%d removed %d blk/rev pairs\n", + LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", nPruneTarget/1024/1024, nCurrentUsage/1024/1024, ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, - nLastBlockWeMustKeep, count); + nLastBlockWeCanPrune, count); } bool CheckDiskSpace(uint64_t nAdditionalBytes) @@ -3197,7 +3220,7 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && pindex == pindexState && (coins.GetCacheSize() + pcoinsTip->GetCacheSize()) <= nCoinCacheSize) { + if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { bool fClean = true; if (!DisconnectBlock(block, state, pindex, coins, &fClean)) return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); diff --git a/src/main.h b/src/main.h index 90b809e85e..fcbc4075d2 100644 --- a/src/main.h +++ b/src/main.h @@ -82,8 +82,10 @@ static const unsigned int MAX_HEADERS_RESULTS = 2000; * degree of disordering of blocks on disk (which make reindexing and in the future perhaps pruning * harder). We'll probably want to make this a per-peer adaptive value at some point. */ static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; -/** Time to wait (in seconds) between writing blockchain state to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 3600; +/** Time to wait (in seconds) between writing blocks/block index to disk. */ +static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; +/** Time to wait (in seconds) between flushing chainstate to disk. */ +static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; /** Maximum length of reject messages. */ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; @@ -119,7 +121,7 @@ extern bool fTxIndex; extern bool fIsBareMultisigStd; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; -extern unsigned int nCoinCacheSize; +extern size_t nCoinCacheUsage; extern CFeeRate minRelayTxFee; /** Best header we've seen so far (used for getheaders queries' starting points). */ diff --git a/src/memusage.h b/src/memusage.h new file mode 100644 index 0000000000..9f7de9e2e1 --- /dev/null +++ b/src/memusage.h @@ -0,0 +1,111 @@ +// Copyright (c) 2015 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_MEMUSAGE_H +#define BITCOIN_MEMUSAGE_H + +#include <stdlib.h> + +#include <map> +#include <set> +#include <vector> + +#include <boost/unordered_set.hpp> +#include <boost/unordered_map.hpp> + +namespace memusage +{ + +/** Compute the total memory used by allocating alloc bytes. */ +static size_t MallocUsage(size_t alloc); + +/** Compute the memory used for dynamically allocated but owned data structures. + * For generic data types, this is *not* recursive. DynamicUsage(vector<vector<int> >) + * will compute the memory used for the vector<int>'s, but not for the ints inside. + * This is for efficiency reasons, as these functions are intended to be fast. If + * application data structures require more accurate inner accounting, they should + * do the recursion themselves, or use more efficient caching + updating on modification. + */ +template<typename X> static size_t DynamicUsage(const std::vector<X>& v); +template<typename X> static size_t DynamicUsage(const std::set<X>& s); +template<typename X, typename Y> static size_t DynamicUsage(const std::map<X, Y>& m); +template<typename X, typename Y> static size_t DynamicUsage(const boost::unordered_set<X, Y>& s); +template<typename X, typename Y, typename Z> static size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& s); +template<typename X> static size_t DynamicUsage(const X& x); + +static inline size_t MallocUsage(size_t alloc) +{ + // Measured on libc6 2.19 on Linux. + if (sizeof(void*) == 8) { + return ((alloc + 31) >> 4) << 4; + } else if (sizeof(void*) == 4) { + return ((alloc + 15) >> 3) << 3; + } else { + assert(0); + } +} + +// STL data structures + +template<typename X> +struct stl_tree_node +{ +private: + int color; + void* parent; + void* left; + void* right; + X x; +}; + +template<typename X> +static inline size_t DynamicUsage(const std::vector<X>& v) +{ + return MallocUsage(v.capacity() * sizeof(X)); +} + +template<typename X> +static inline size_t DynamicUsage(const std::set<X>& s) +{ + return MallocUsage(sizeof(stl_tree_node<X>)) * s.size(); +} + +template<typename X, typename Y> +static inline size_t DynamicUsage(const std::map<X, Y>& m) +{ + return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size(); +} + +// Boost data structures + +template<typename X> +struct boost_unordered_node : private X +{ +private: + void* ptr; +}; + +template<typename X, typename Y> +static inline size_t DynamicUsage(const boost::unordered_set<X, Y>& s) +{ + return MallocUsage(sizeof(boost_unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); +} + +template<typename X, typename Y, typename Z> +static inline size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& m) +{ + return MallocUsage(sizeof(boost_unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); +} + +// Dispatch to class method as fallback + +template<typename X> +static inline size_t DynamicUsage(const X& x) +{ + return x.DynamicMemoryUsage(); +} + +} + +#endif diff --git a/src/net.cpp b/src/net.cpp index 2de04fc574..6849d79263 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -13,6 +13,7 @@ #include "chainparams.h" #include "clientversion.h" #include "primitives/transaction.h" +#include "scheduler.h" #include "ui_interface.h" #include "crypto/common.h" @@ -1590,7 +1591,7 @@ void static Discover(boost::thread_group& threadGroup) #endif } -void StartNode(boost::thread_group& threadGroup) +void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler) { uiInterface.InitMessage(_("Loading addresses...")); // Load addresses for peers.dat @@ -1640,7 +1641,7 @@ void StartNode(boost::thread_group& threadGroup) threadGroup.create_thread(boost::bind(&TraceThread<void (*)()>, "msghand", &ThreadMessageHandler)); // Dump network addresses - threadGroup.create_thread(boost::bind(&LoopForever<void (*)()>, "dumpaddr", &DumpAddresses, DUMP_ADDRESSES_INTERVAL * 1000)); + scheduler.scheduleEvery(&DumpAddresses, DUMP_ADDRESSES_INTERVAL); } bool StopNode() @@ -32,6 +32,7 @@ class CAddrMan; class CBlockIndex; +class CScheduler; class CNode; namespace boost { @@ -72,7 +73,7 @@ bool OpenNetworkConnection(const CAddress& addrConnect, CSemaphoreGrant *grantOu void MapPort(bool fUseUPnP); unsigned short GetListenPort(); bool BindListenPort(const CService &bindAddr, std::string& strError, bool fWhitelisted = false); -void StartNode(boost::thread_group& threadGroup); +void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler); bool StopNode(); void SocketSendData(CNode *pnode); diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 018169cfdc..8740b98b70 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -26,6 +26,7 @@ #include "init.h" #include "main.h" #include "rpcserver.h" +#include "scheduler.h" #include "ui_interface.h" #include "util.h" @@ -178,6 +179,7 @@ signals: private: boost::thread_group threadGroup; + CScheduler scheduler; /// Pass fatal exception message to UI thread void handleRunawayException(const std::exception *e); @@ -258,7 +260,7 @@ void BitcoinCore::initialize() try { qDebug() << __func__ << ": Running AppInit2 in thread"; - int rv = AppInit2(threadGroup); + int rv = AppInit2(threadGroup, scheduler); if(rv) { /* Start a dummy RPC thread if no RPC thread is active yet diff --git a/src/qt/bitcoin.qrc b/src/qt/bitcoin.qrc index 63af146fd0..c899e95506 100644 --- a/src/qt/bitcoin.qrc +++ b/src/qt/bitcoin.qrc @@ -45,6 +45,7 @@ <file alias="about">res/icons/about.png</file> <file alias="about_qt">res/icons/about_qt.png</file> <file alias="verify">res/icons/verify.png</file> + <file alias="warning">res/icons/warning.png</file> </qresource> <qresource prefix="/movies"> <file alias="spinner-000">res/movies/spinner-000.png</file> diff --git a/src/qt/forms/overviewpage.ui b/src/qt/forms/overviewpage.ui index 53d416ef38..6d792d1475 100644 --- a/src/qt/forms/overviewpage.ui +++ b/src/qt/forms/overviewpage.ui @@ -59,21 +59,35 @@ </widget> </item> <item> - <widget class="QLabel" name="labelWalletStatus"> - <property name="cursor"> - <cursorShape>WhatsThisCursor</cursorShape> + <widget class="QPushButton" name="labelWalletStatus"> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="maximumSize"> + <size> + <width>30</width> + <height>16777215</height> + </size> </property> <property name="toolTip"> <string>The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet.</string> </property> - <property name="styleSheet"> - <string notr="true">QLabel { color: red; }</string> - </property> <property name="text"> - <string notr="true">(out of sync)</string> + <string/> </property> - <property name="alignment"> - <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set> + <property name="icon"> + <iconset resource="../bitcoin.qrc"> + <normaloff>:/icons/warning</normaloff> + <disabledoff>:/icons/warning</disabledoff>:/icons/warning</iconset> + </property> + <property name="iconSize"> + <size> + <width>24</width> + <height>24</height> + </size> + </property> + <property name="flat"> + <bool>true</bool> </property> </widget> </item> @@ -431,21 +445,35 @@ </widget> </item> <item> - <widget class="QLabel" name="labelTransactionsStatus"> - <property name="cursor"> - <cursorShape>WhatsThisCursor</cursorShape> + <widget class="QPushButton" name="labelTransactionsStatus"> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="maximumSize"> + <size> + <width>30</width> + <height>16777215</height> + </size> </property> <property name="toolTip"> <string>The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet.</string> </property> - <property name="styleSheet"> - <string notr="true">QLabel { color: red; }</string> - </property> <property name="text"> - <string notr="true">(out of sync)</string> + <string/> </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> + <property name="icon"> + <iconset resource="../bitcoin.qrc"> + <normaloff>:/icons/warning</normaloff> + <disabledoff>:/icons/warning</disabledoff>:/icons/warning</iconset> + </property> + <property name="iconSize"> + <size> + <width>24</width> + <height>24</height> + </size> + </property> + <property name="flat"> + <bool>true</bool> </property> </widget> </item> diff --git a/src/qt/forms/sendcoinsentry.ui b/src/qt/forms/sendcoinsentry.ui index 48d0dd093c..df06f36833 100644 --- a/src/qt/forms/sendcoinsentry.ui +++ b/src/qt/forms/sendcoinsentry.ui @@ -21,15 +21,21 @@ <string>This is a normal payment.</string> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout"> - <property name="spacing"> + <property name="topMargin"> + <number>8</number> + </property> + <property name="bottomMargin"> + <number>4</number> + </property> + <property name="horizontalSpacing"> <number>12</number> </property> + <property name="verticalSpacing"> + <number>8</number> + </property> <item row="0" column="0"> <widget class="QLabel" name="payToLabel"> <property name="text"> @@ -193,6 +199,13 @@ </property> </widget> </item> + <item row="4" column="0" colspan="2"> + <widget class="Line" name="line"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + </widget> + </item> </layout> </widget> <widget class="QFrame" name="SendCoins_UnauthenticatedPaymentRequest"> @@ -618,10 +631,7 @@ <bool>true</bool> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout_is"> <property name="spacing"> @@ -1150,10 +1160,7 @@ <bool>true</bool> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout_s"> <property name="spacing"> diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 4fa15db9c6..a422ff9a71 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -18,8 +18,8 @@ #include <QAbstractItemDelegate> #include <QPainter> -#define DECORATION_SIZE 64 -#define NUM_ITEMS 3 +#define DECORATION_SIZE 54 +#define NUM_ITEMS 5 class TxViewDelegate : public QAbstractItemDelegate { @@ -129,10 +129,6 @@ OverviewPage::OverviewPage(QWidget *parent) : connect(ui->listTransactions, SIGNAL(clicked(QModelIndex)), this, SLOT(handleTransactionClicked(QModelIndex))); - // init "out of sync" warning labels - ui->labelWalletStatus->setText("(" + tr("out of sync") + ")"); - ui->labelTransactionsStatus->setText("(" + tr("out of sync") + ")"); - // start with displaying the "out of sync" warnings showOutOfSyncWarning(true); } diff --git a/src/qt/res/icons/warning.png b/src/qt/res/icons/warning.png Binary files differnew file mode 100644 index 0000000000..723a30a658 --- /dev/null +++ b/src/qt/res/icons/warning.png diff --git a/src/qt/scicon.cpp b/src/qt/scicon.cpp index a0ffcd82a9..b2f2399b24 100644 --- a/src/qt/scicon.cpp +++ b/src/qt/scicon.cpp @@ -27,12 +27,17 @@ static void MakeSingleColorImage(QImage& img, const QColor& colorbase) QImage SingleColorImage(const QString& filename, const QColor& colorbase) { QImage img(filename); +#if !defined(WIN32) && !defined(MAC_OSX) MakeSingleColorImage(img, colorbase); +#endif return img; } QIcon SingleColorIcon(const QIcon& ico, const QColor& colorbase) { +#if defined(WIN32) || defined(MAC_OSX) + return ico; +#else QIcon new_ico; QSize sz; Q_FOREACH(sz, ico.availableSizes()) @@ -42,6 +47,7 @@ QIcon SingleColorIcon(const QIcon& ico, const QColor& colorbase) new_ico.addPixmap(QPixmap::fromImage(img)); } return new_ico; +#endif } QIcon SingleColorIcon(const QString& filename, const QColor& colorbase) @@ -51,6 +57,9 @@ QIcon SingleColorIcon(const QString& filename, const QColor& colorbase) QColor SingleColor() { +#if defined(WIN32) || defined(MAC_OSX) + return QColor(0,0,0); +#else const QColor colorHighlightBg(QApplication::palette().color(QPalette::Highlight)); const QColor colorHighlightFg(QApplication::palette().color(QPalette::HighlightedText)); const QColor colorText(QApplication::palette().color(QPalette::WindowText)); @@ -61,6 +70,7 @@ QColor SingleColor() else colorbase = colorHighlightFg; return colorbase; +#endif } QIcon SingleColorIcon(const QString& filename) diff --git a/src/scheduler.cpp b/src/scheduler.cpp new file mode 100644 index 0000000000..8b55888ae8 --- /dev/null +++ b/src/scheduler.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "scheduler.h" + +#include <assert.h> +#include <boost/bind.hpp> +#include <utility> + +CScheduler::CScheduler() : nThreadsServicingQueue(0) +{ +} + +CScheduler::~CScheduler() +{ + assert(nThreadsServicingQueue == 0); +} + + +#if BOOST_VERSION < 105000 +static boost::system_time toPosixTime(const boost::chrono::system_clock::time_point& t) +{ + return boost::posix_time::from_time_t(boost::chrono::system_clock::to_time_t(t)); +} +#endif + +void CScheduler::serviceQueue() +{ + boost::unique_lock<boost::mutex> lock(newTaskMutex); + ++nThreadsServicingQueue; + + // newTaskMutex is locked throughout this loop EXCEPT + // when the thread is waiting or when the user's function + // is called. + while (1) { + try { + while (taskQueue.empty()) { + // Wait until there is something to do. + newTaskScheduled.wait(lock); + } +// Wait until either there is a new task, or until +// the time of the first item on the queue: + +// wait_until needs boost 1.50 or later; older versions have timed_wait: +#if BOOST_VERSION < 105000 + while (!taskQueue.empty() && newTaskScheduled.timed_wait(lock, toPosixTime(taskQueue.begin()->first))) { + // Keep waiting until timeout + } +#else + while (!taskQueue.empty() && newTaskScheduled.wait_until(lock, taskQueue.begin()->first) != boost::cv_status::timeout) { + // Keep waiting until timeout + } +#endif + // If there are multiple threads, the queue can empty while we're waiting (another + // thread may service the task we were waiting on). + if (taskQueue.empty()) + continue; + + Function f = taskQueue.begin()->second; + taskQueue.erase(taskQueue.begin()); + + // Unlock before calling f, so it can reschedule itself or another task + // without deadlocking: + lock.unlock(); + f(); + lock.lock(); + } catch (...) { + --nThreadsServicingQueue; + throw; + } + } +} + +void CScheduler::schedule(CScheduler::Function f, boost::chrono::system_clock::time_point t) +{ + { + boost::unique_lock<boost::mutex> lock(newTaskMutex); + taskQueue.insert(std::make_pair(t, f)); + } + newTaskScheduled.notify_one(); +} + +void CScheduler::scheduleFromNow(CScheduler::Function f, int64_t deltaSeconds) +{ + schedule(f, boost::chrono::system_clock::now() + boost::chrono::seconds(deltaSeconds)); +} + +static void Repeat(CScheduler* s, CScheduler::Function f, int64_t deltaSeconds) +{ + f(); + s->scheduleFromNow(boost::bind(&Repeat, s, f, deltaSeconds), deltaSeconds); +} + +void CScheduler::scheduleEvery(CScheduler::Function f, int64_t deltaSeconds) +{ + scheduleFromNow(boost::bind(&Repeat, this, f, deltaSeconds), deltaSeconds); +} diff --git a/src/scheduler.h b/src/scheduler.h new file mode 100644 index 0000000000..bb383ab9f9 --- /dev/null +++ b/src/scheduler.h @@ -0,0 +1,70 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SCHEDULER_H +#define BITCOIN_SCHEDULER_H + +// +// NOTE: +// boost::thread / boost::function / boost::chrono should be ported to +// std::thread / std::function / std::chrono when we support C++11. +// +#include <boost/function.hpp> +#include <boost/chrono/chrono.hpp> +#include <boost/thread.hpp> +#include <map> + +// +// Simple class for background tasks that should be run +// periodically or once "after a while" +// +// Usage: +// +// CScheduler* s = new CScheduler(); +// s->scheduleFromNow(doSomething, 11); // Assuming a: void doSomething() { } +// s->scheduleFromNow(boost::bind(Class::func, this, argument), 3); +// boost::thread* t = new boost::thread(boost::bind(CScheduler::serviceQueue, s)); +// +// ... then at program shutdown, clean up the thread running serviceQueue: +// t->interrupt(); +// t->join(); +// delete t; +// delete s; // Must be done after thread is interrupted/joined. +// + +class CScheduler +{ +public: + CScheduler(); + ~CScheduler(); + + typedef boost::function<void(void)> Function; + + // Call func at/after time t + void schedule(Function f, boost::chrono::system_clock::time_point t); + + // Convenience method: call f once deltaSeconds from now + void scheduleFromNow(Function f, int64_t deltaSeconds); + + // Another convenience method: call f approximately + // every deltaSeconds forever, starting deltaSeconds from now. + // To be more precise: every time f is finished, it + // is rescheduled to run deltaSeconds later. If you + // need more accurate scheduling, don't use this method. + void scheduleEvery(Function f, int64_t deltaSeconds); + + // To keep things as simple as possible, there is no unschedule. + + // Services the queue 'forever'. Should be run in a thread, + // and interrupted using boost::interrupt_thread + void serviceQueue(); + +private: + std::multimap<boost::chrono::system_clock::time_point, Function> taskQueue; + boost::condition_variable newTaskScheduled; + boost::mutex newTaskMutex; + int nThreadsServicingQueue; +}; + +#endif diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 2e2cc2214b..34b311b804 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -59,6 +59,24 @@ public: bool GetStats(CCoinsStats& stats) const { return false; } }; + +class CCoinsViewCacheTest : public CCoinsViewCache +{ +public: + CCoinsViewCacheTest(CCoinsView* base) : CCoinsViewCache(base) {} + + void SelfTest() const + { + // Manually recompute the dynamic usage of the whole data, and compare it. + size_t ret = memusage::DynamicUsage(cacheCoins); + for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { + ret += memusage::DynamicUsage(it->second.coins); + } + BOOST_CHECK_EQUAL(memusage::DynamicUsage(*this), ret); + } + +}; + } BOOST_FIXTURE_TEST_SUITE(coins_tests, BasicTestingSetup) @@ -90,8 +108,8 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. - std::vector<CCoinsViewCache*> stack; // A stack of CCoinsViewCaches on top. - stack.push_back(new CCoinsViewCache(&base)); // Start with one cache. + std::vector<CCoinsViewCacheTest*> stack; // A stack of CCoinsViewCaches on top. + stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. // Use a limited set of random transaction ids, so we do test overwriting entries. std::vector<uint256> txids; @@ -136,6 +154,9 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) missed_an_entry = true; } } + BOOST_FOREACH(const CCoinsViewCacheTest *test, stack) { + test->SelfTest(); + } } if (insecure_rand() % 100 == 0) { @@ -152,7 +173,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) } else { removed_all_caches = true; } - stack.push_back(new CCoinsViewCache(tip)); + stack.push_back(new CCoinsViewCacheTest(tip)); if (stack.size() == 4) { reached_4_caches = true; } diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp new file mode 100644 index 0000000000..a26d0afaed --- /dev/null +++ b/src/test/scheduler_tests.cpp @@ -0,0 +1,110 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "random.h" +#include "scheduler.h" + +#include "test/test_bitcoin.h" + +#include <boost/bind.hpp> +#include <boost/random/mersenne_twister.hpp> +#include <boost/random/uniform_int_distribution.hpp> +#include <boost/thread.hpp> +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(scheduler_tests) + +static void microTask(CScheduler& s, boost::mutex& mutex, int& counter, int delta, boost::chrono::system_clock::time_point rescheduleTime) +{ + { + boost::unique_lock<boost::mutex> lock(mutex); + counter += delta; + } + boost::chrono::system_clock::time_point noTime = boost::chrono::system_clock::time_point::min(); + if (rescheduleTime != noTime) { + CScheduler::Function f = boost::bind(µTask, boost::ref(s), boost::ref(mutex), boost::ref(counter), -delta + 1, noTime); + s.schedule(f, rescheduleTime); + } +} + +static void MicroSleep(uint64_t n) +{ +#if defined(HAVE_WORKING_BOOST_SLEEP_FOR) + boost::this_thread::sleep_for(boost::chrono::microseconds(n)); +#elif defined(HAVE_WORKING_BOOST_SLEEP) + boost::this_thread::sleep(boost::posix_time::microseconds(n)); +#else + //should never get here + #error missing boost sleep implementation +#endif +} + +BOOST_AUTO_TEST_CASE(manythreads) +{ + // Stress test: hundreds of microsecond-scheduled tasks, + // serviced by 10 threads. + // + // So... ten shared counters, which if all the tasks execute + // properly will sum to the number of tasks done. + // Each task adds or subtracts from one of the counters a + // random amount, and then schedules another task 0-1000 + // microseconds in the future to subtract or add from + // the counter -random_amount+1, so in the end the shared + // counters should sum to the number of initial tasks performed. + CScheduler microTasks; + + boost::thread_group microThreads; + for (int i = 0; i < 5; i++) + microThreads.create_thread(boost::bind(&CScheduler::serviceQueue, µTasks)); + + boost::mutex counterMutex[10]; + int counter[10] = { 0 }; + boost::random::mt19937 rng(insecure_rand()); + boost::random::uniform_int_distribution<> zeroToNine(0, 9); + boost::random::uniform_int_distribution<> randomMsec(-11, 1000); + boost::random::uniform_int_distribution<> randomDelta(-1000, 1000); + + boost::chrono::system_clock::time_point start = boost::chrono::system_clock::now(); + boost::chrono::system_clock::time_point now = start; + + for (int i = 0; i < 100; i++) { + boost::chrono::system_clock::time_point t = now + boost::chrono::microseconds(randomMsec(rng)); + boost::chrono::system_clock::time_point tReschedule = now + boost::chrono::microseconds(500 + randomMsec(rng)); + int whichCounter = zeroToNine(rng); + CScheduler::Function f = boost::bind(µTask, boost::ref(microTasks), + boost::ref(counterMutex[whichCounter]), boost::ref(counter[whichCounter]), + randomDelta(rng), tReschedule); + microTasks.schedule(f, t); + } + + MicroSleep(600); + now = boost::chrono::system_clock::now(); + // More threads and more tasks: + for (int i = 0; i < 5; i++) + microThreads.create_thread(boost::bind(&CScheduler::serviceQueue, µTasks)); + for (int i = 0; i < 100; i++) { + boost::chrono::system_clock::time_point t = now + boost::chrono::microseconds(randomMsec(rng)); + boost::chrono::system_clock::time_point tReschedule = now + boost::chrono::microseconds(500 + randomMsec(rng)); + int whichCounter = zeroToNine(rng); + CScheduler::Function f = boost::bind(µTask, boost::ref(microTasks), + boost::ref(counterMutex[whichCounter]), boost::ref(counter[whichCounter]), + randomDelta(rng), tReschedule); + microTasks.schedule(f, t); + } + + // All 2,000 tasks should be finished within 2 milliseconds. Sleep a bit longer. + MicroSleep(2100); + + microThreads.interrupt_all(); + microThreads.join_all(); + + int counterSum = 0; + for (int i = 0; i < 10; i++) { + BOOST_CHECK(counter[i] != 0); + counterSum += counter[i]; + } + BOOST_CHECK_EQUAL(counterSum, 200); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.h b/src/txdb.h index 86e1c5d831..bef5dc9fd1 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -22,7 +22,7 @@ class uint256; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 100; //! max. -dbcache in (MiB) -static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 4096 : 1024; +static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache in (MiB) static const int64_t nMinDbCache = 4; diff --git a/src/util.h b/src/util.h index 483d9d7858..4cc0faf4d7 100644 --- a/src/util.h +++ b/src/util.h @@ -203,43 +203,6 @@ void SetThreadPriority(int nPriority); void RenameThread(const char* name); /** - * Standard wrapper for do-something-forever thread functions. - * "Forever" really means until the thread is interrupted. - * Use it like: - * new boost::thread(boost::bind(&LoopForever<void (*)()>, "dumpaddr", &DumpAddresses, 900000)); - * or maybe: - * boost::function<void()> f = boost::bind(&FunctionWithArg, argument); - * threadGroup.create_thread(boost::bind(&LoopForever<boost::function<void()> >, "nothing", f, milliseconds)); - */ -template <typename Callable> void LoopForever(const char* name, Callable func, int64_t msecs) -{ - std::string s = strprintf("bitcoin-%s", name); - RenameThread(s.c_str()); - LogPrintf("%s thread start\n", name); - try - { - while (1) - { - MilliSleep(msecs); - func(); - } - } - catch (const boost::thread_interrupted&) - { - LogPrintf("%s thread stop\n", name); - throw; - } - catch (const std::exception& e) { - PrintExceptionContinue(&e, name); - throw; - } - catch (...) { - PrintExceptionContinue(NULL, name); - throw; - } -} - -/** * .. and a wrapper that just calls func once */ template <typename Callable> void TraceThread(const char* name, Callable func) |
