diff options
40 files changed, 1874 insertions, 642 deletions
diff --git a/configure.ac b/configure.ac index 7c0a9f4a94..c5d08a0285 100644 --- a/configure.ac +++ b/configure.ac @@ -590,17 +590,15 @@ fi if test x$use_boost = xyes; then -BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB" +BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB" dnl Boost >= 1.50 uses sleep_for rather than the now-deprecated sleep, however dnl it was broken from 1.50 to 1.52 when backed by nanosleep. Use sleep_for if dnl a working version is available, else fall back to sleep. sleep was removed dnl after 1.56. dnl If neither is available, abort. -dnl If sleep_for is used, boost_chrono becomes a requirement. -if test x$ax_cv_boost_chrono = xyes; then TEMP_LIBS="$LIBS" -LIBS="$BOOST_LIBS $BOOST_CHRONO_LIB $LIBS" +LIBS="$BOOST_LIBS $LIBS" TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -613,12 +611,11 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ choke me #endif ]])], - [boost_sleep=yes; BOOST_LIBS="$BOOST_LIBS $BOOST_CHRONO_LIB"; + [boost_sleep=yes; AC_DEFINE(HAVE_WORKING_BOOST_SLEEP_FOR, 1, [Define this symbol if boost sleep_for works])], [boost_sleep=no]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" -fi if test x$boost_sleep != xyes; then TEMP_LIBS="$LIBS" diff --git a/depends/Makefile b/depends/Makefile index 05ef33f2ee..ef5a20e6c3 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -91,12 +91,12 @@ include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) -$(host_prefix)/.stamp_$(final_build_id): | $(native_packages) $(packages) +$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) - $(AT)echo copying packages: $| + $(AT)echo copying packages: $^ $(AT)echo to: $(@D) - $(AT)cd $(@D); $(foreach package,$|, tar xf $($(package)_cached); ) + $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) @@ -121,8 +121,35 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ $< > $@ $(AT)touch $@ -install: $(host_prefix)/share/config.site -download-one: $(all_sources) + +define check_or_remove_cached + mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \ + $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \ + ( rm -f $($(package)_cached_checksum); \ + if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi ) +endef + +define check_or_remove_sources + mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ + $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ + ( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \ + rm -f $($(package)_all_sources) $($(1)_fetched)) +endef + +check-packages: + @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));) +check-sources: + @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) + +$(host_prefix)/share/config.site: check-packages + +check-packages: check-sources + +install: check-packages $(host_prefix)/share/config.site + + +download-one: check-sources $(all_sources) + download-osx: @$(MAKE) -s HOST=x86_64-apple-darwin11 download-one download-linux: @@ -130,4 +157,5 @@ download-linux: download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win -.PHONY: install cached download-one download-osx download-linux download-win download + +.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources diff --git a/depends/README.md b/depends/README.md new file mode 100644 index 0000000000..2dc0b9e47e --- /dev/null +++ b/depends/README.md @@ -0,0 +1,56 @@ +### Usage + +To build dependencies for the current arch+OS: + + make + +To build for another arch/OS: + + make HOST=host-platform-triplet + +For example: + + make HOST=x86_64-w64-mingw32 -j4 + +A prefix will be generated that's suitable for plugging into Bitcoin's +configure. In the above example, a dir named i686-w64-mingw32 will be +created. To use it for Bitcoin: + + ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32 + +Common `host-platform-triplets` for cross compilation are: + +- `i686-w64-mingw32` for Win32 +- `x86_64-w64-mingw32` for Win64 +- `x86_64-apple-darwin11` for MacOSX +- `arm-linux-gnueabihf` for Linux ARM + +No other options are needed, the paths are automatically configured. + +Dependency Options: +The following can be set when running make: make FOO=bar + + SOURCES_PATH: downloaded sources will be placed here + BASE_CACHE: built packages will be placed here + SDK_PATH: Path where sdk's can be found (used by OSX) + FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up + NO_QT: Don't download/build/cache qt and its dependencies + NO_WALLET: Don't download/build/cache libs needed to enable the wallet + NO_UPNP: Don't download/build/cache packages needed for enabling upnp + DEBUG: disable some optimizations and enable more runtime checking + +If some packages are not built, for example `make NO_WALLET=1`, the appropriate +options will be passed to bitcoin's configure. In this case, `--disable-wallet`. + +Additional targets: + + download: run 'make download' to fetch all sources without building them + download-osx: run 'make download-osx' to fetch all sources needed for osx builds + download-win: run 'make download-win' to fetch all sources needed for win builds + download-linux: run 'make download-linux' to fetch all sources needed for linux builds + +### Other documentation + +- [description.md](description.md): General description of the depends system +- [packages.md](packages.md): Steps for adding packages + diff --git a/depends/README.usage b/depends/README.usage deleted file mode 100644 index 24e1231d82..0000000000 --- a/depends/README.usage +++ /dev/null @@ -1,34 +0,0 @@ -To build dependencies for the current arch+OS: - make -To build for another arch/OS: - make HOST=host-platform-triplet && make HOST=host-platform-triplet - (For example: make HOST=i686-w64-mingw32 -j4) - -A prefix will be generated that's suitable for plugging into Bitcoin's -configure. In the above example, a dir named i686-w64-mingw32 will be -created. To use it for Bitcoin: - -./configure --prefix=`pwd`/depends/i686-w64-mingw32 - -No other options are needed, the paths are automatically configured. - -Dependency Options: -The following can be set when running make: make FOO=bar - -SOURCES_PATH: downloaded sources will be placed here -BASE_CACHE: built packages will be placed here -SDK_PATH: Path where sdk's can be found (used by OSX) -FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up -NO_QT: Don't download/build/cache qt and its dependencies -NO_WALLET: Don't download/build/cache libs needed to enable the wallet -NO_UPNP: Don't download/build/cache packages needed for enabling upnp -DEBUG: disable some optimizations and enable more runtime checking - -If some packages are not built, for example 'make NO_WALLET=1', the appropriate -options will be passed to bitcoin's configure. In this case, --disable-wallet. - -Additional targets: -download: run 'make download' to fetch all sources without building them -download-osx: run 'make download-osx' to fetch all sources needed for osx builds -download-win: run 'make download-win' to fetch all sources needed for win builds -download-linux: run 'make download-linux' to fetch all sources needed for linux builds diff --git a/depends/README b/depends/description.md index 55e7222697..74f9ef3f20 100644 --- a/depends/README +++ b/depends/description.md @@ -1,9 +1,7 @@ -This is a system of building and caching dependencies necessary for building -Bitcoin. - +This is a system of building and caching dependencies necessary for building Bitcoin. There are several features that make it different from most similar systems: -- It is designed to be builder and host agnostic +### It is designed to be builder and host agnostic In theory, binaries for any target OS/architecture can be created, from a builder running any OS/architecture. In practice, build-side tools must be @@ -11,18 +9,18 @@ specified when the defaults don't fit, and packages must be amended to work on new hosts. For now, a build architecture of x86_64 is assumed, either on Linux or OSX. -- No reliance on timestamps +### No reliance on timestamps File presence is used to determine what needs to be built. This makes the results distributable and easily digestable by automated builders. -- Each build only has its specified dependencies available at build-time. +### Each build only has its specified dependencies available at build-time. For each build, the sysroot is wiped and the (recursive) dependencies are installed. This makes each build deterministic, since there will never be any unknown files available to cause side-effects. -- Each package is cached and only rebuilt as needed. +### Each package is cached and only rebuilt as needed. Before building, a unique build-id is generated for each package. This id consists of a hash of all files used to build the package (Makefiles, packages, @@ -32,7 +30,7 @@ any other package that depends on it. If any of the main makefiles (Makefile, funcs.mk, etc) are changed, all packages will be rebuilt. After building, the results are cached into a tarball that can be re-used and distributed. -- Package build results are (relatively) deterministic. +### Package build results are (relatively) deterministic. Each package is configured and patched so that it will yield the same build-results with each consequent build, within a reasonable set of @@ -41,13 +39,13 @@ beyond the scope of this system. Additionally, the toolchain itself must be capable of deterministic results. When revisions are properly bumped, a cached build should represent an exact single payload. -- Sources are fetched and verified automatically +### Sources are fetched and verified automatically Each package must define its source location and checksum. The build will fail if the fetched source does not match. Sources may be pre-seeded and/or cached as desired. -- Self-cleaning +### Self-cleaning Build and staging dirs are wiped after use, and any previous version of a cached result is removed following a successful build. Automated builders diff --git a/depends/funcs.mk b/depends/funcs.mk index b407737f7f..050a9b1321 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -53,12 +53,14 @@ $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix) $(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) $(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) $(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir) +$(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps -$(1)_fetched=$$($(1)_source_dir)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name) +$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned @@ -154,7 +156,10 @@ endef define int_add_cmds $($(1)_fetched): $(AT)mkdir -p $$(@D) $(SOURCES_PATH) + $(AT)rm -f $$@ + $(AT)touch $$@ $(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) $(AT)touch $$@ $($(1)_extracted): | $($(1)_fetched) $(AT)echo Extracting $(1)... @@ -195,10 +200,12 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) $(AT)rm -rf $$(@D) && mkdir -p $$(@D) $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) $(AT)rm -rf $($(1)_staging_dir) +$($(1)_cached_checksum): $($(1)_cached) + $(AT)cd $$(@D); $(build_SHA256SUM) $$(<F) > $$(@) .PHONY: $(1) -$(1): | $($(1)_cached) -.SECONDARY: $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) +$(1): | $($(1)_cached_checksum) +.SECONDARY: $($(1)_cached) $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) endef diff --git a/depends/README.packages b/depends/packages.md index 5ab7ed7dee..7c80362509 100644 --- a/depends/README.packages +++ b/depends/packages.md @@ -4,119 +4,137 @@ variables, and defining build commands. The package "mylib" will be used here as an example General tips: -mylib_foo is written as $(package)_foo in order to make recipes more similar. +- mylib_foo is written as $(package)_foo in order to make recipes more similar. -Identifiers: +## Identifiers Each package is required to define at least these variables: - $(package)_version: + + $(package)_version: Version of the upstream library or program. If there is no version, a placeholder such as 1.0 can be used. - $(package)_download_path: + + $(package)_download_path: Location of the upstream source, without the file-name. Usually http or ftp. - $(package)_file_name: + + $(package)_file_name: The upstream source filename available at the download path. - $(package)_sha256_hash: + + $(package)_sha256_hash: The sha256 hash of the upstream file These variables are optional: - $(package)_build_subdir: + + $(package)_build_subdir: cd to this dir before running configure/build/stage commands. - $(package)_download_file: + + $(package)_download_file: The file-name of the upstream source if it differs from how it should be stored locally. This can be used to avoid storing file-names with strange characters. - $(package)_dependencies: + + $(package)_dependencies: Names of any other packages that this one depends on. - $(package)_patches: + + $(package)_patches: Filenames of any patches needed to build the package + $(package)_extra_sources: + Any extra files that will be fetched via $(package)_fetch_cmds. These are + specified so that they can be fetched and verified via 'make download'. + -Build Variables: +## Build Variables: After defining the main identifiers, build variables may be added or customized before running the build commands. They should be added to a function called $(package)_set_vars. For example: -define $(package)_set_vars -... -endef + define $(package)_set_vars + ... + endef Most variables can be prefixed with the host, architecture, or both, to make the modifications specific to that case. For example: - Universal: $(package)_cc=gcc - Linux only: $(package)_linux_cc=gcc - x86_64 only: $(package)_x86_64_cc = gcc - x86_64 linux only: $(package)_x86_64_linux_cc = gcc + Universal: $(package)_cc=gcc + Linux only: $(package)_linux_cc=gcc + x86_64 only: $(package)_x86_64_cc = gcc + x86_64 linux only: $(package)_x86_64_linux_cc = gcc These variables may be set to override or append their default values. - $(package)_cc - $(package)_cxx - $(package)_objc - $(package)_objcxx - $(package)_ar - $(package)_ranlib - $(package)_libtool - $(package)_nm - $(package)_cflags - $(package)_cxxflags - $(package)_ldflags - $(package)_cppflags - $(package)_config_env - $(package)_build_env - $(package)_stage_env - $(package)_build_opts - $(package)_config_opts + + $(package)_cc + $(package)_cxx + $(package)_objc + $(package)_objcxx + $(package)_ar + $(package)_ranlib + $(package)_libtool + $(package)_nm + $(package)_cflags + $(package)_cxxflags + $(package)_ldflags + $(package)_cppflags + $(package)_config_env + $(package)_build_env + $(package)_stage_env + $(package)_build_opts + $(package)_config_opts The *_env variables are used to add environment variables to the respective commands. Many variables respect a debug/release suffix as well, in order to use them for only the appropriate build config. For example: - $(package)_cflags_release = -O3 - $(package)_cflags_i686_debug = -g - $(package)_config_opts_release = --disable-debug + + $(package)_cflags_release = -O3 + $(package)_cflags_i686_debug = -g + $(package)_config_opts_release = --disable-debug These will be used in addition to the options that do not specify debug/release. All builds are considered to be release unless DEBUG=1 is set by -the user. +the user. Other variables may be defined as needed. -Other variables may be defined as needed. - -Build commands: +## Build commands: For each build, a unique build dir and staging dir are created. For example, - work/build/mylib/1.0-1adac830f6e and work/staging/mylib/1.0-1adac830f6e. + `work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. The following build commands are available for each recipe: - $(package)_fetch_cmds: + $(package)_fetch_cmds: Runs from: build dir Fetch the source file. If undefined, it will be fetched and verified against its hash. - $(package)_extract_cmds: + + $(package)_extract_cmds: Runs from: build dir Verify the source file against its hash and extract it. If undefined, the source is assumed to be a tarball. - $(package)_preprocess_cmds: + + $(package)_preprocess_cmds: Runs from: build dir/$(package)_build_subdir Preprocess the source as necessary. If undefined, does nothing. - $(package)_config_cmds: + + $(package)_config_cmds: Runs from: build dir/$(package)_build_subdir Configure the source. If undefined, does nothing. - $(package)_build_cmds: + + $(package)_build_cmds: Runs from: build dir/$(package)_build_subdir Build the source. If undefined, does nothing. - $(package)_stage_cmds: + + $(package)_stage_cmds: Runs from: build dir/$(package)_build_subdir Stage the build results. If undefined, does nothing. The following variables are available for each recipe: - $(1)_staging_dir: package's destination sysroot path - $(1)_staging_prefix_dir: prefix path inside of the package's staging dir - $(1)_extract_dir: path to the package's extracted sources - $(1)_build_dir: path where configure/build/stage commands will be run - $(1)_patch_dir: path where the package's patches (if any) are found + + $(1)_staging_dir: package's destination sysroot path + $(1)_staging_prefix_dir: prefix path inside of the package's staging dir + $(1)_extract_dir: path to the package's extracted sources + $(1)_build_dir: path where configure/build/stage commands will be run + $(1)_patch_dir: path where the package's patches (if any) are found Notes on build commands: @@ -125,4 +143,5 @@ configure step to (usually) correctly configure automatically. Any $($(package)_config_opts) will be appended. Most autotools projects can be properly staged using: - $(MAKE) DESTDIR=$($(package)_staging_dir) install + + $(MAKE) DESTDIR=$($(package)_staging_dir) install diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 951ad4fb29..1c1bcf199a 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -9,6 +9,8 @@ $(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_versi $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_sha256_hash=60d8f69f032d62ef61bf527857ebb933741ec3352d4d328c5516aa520662dab7 +$(package)_extra_sources=$($(package)_clang_file_name) + define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) diff --git a/doc/assets-attribution.md b/doc/assets-attribution.md index c860cdc534..ebba64a61a 100644 --- a/doc/assets-attribution.md +++ b/doc/assets-attribution.md @@ -22,6 +22,7 @@ The following is a list of assets used in the bitcoin source and their proper at src/qt/res/icons/receive.png, src/qt/res/icons/remove.png, src/qt/res/icons/send.png, src/qt/res/icons/synced.png, src/qt/res/icons/transaction*.png, src/qt/res/icons/tx_output.png, + src/qt/res/icons/warning.png Jonas Schnelli ----------------------- diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py index f26cbee1e2..85fd1c982a 100755 --- a/qa/rpc-tests/pruning.py +++ b/qa/rpc-tests/pruning.py @@ -7,7 +7,8 @@ # Test pruning code # ******** # WARNING: -# This test uses 4GB of disk space and takes in excess of 30 mins to run +# This test uses 4GB of disk space. +# This test takes 30 mins or more (up to 2 hours) # ******** from test_framework import BitcoinTestFramework @@ -51,11 +52,11 @@ class PruneTest(BitcoinTestFramework): self.is_network_split = False # Create nodes 0 and 1 to mine - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)) + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) # Create node 2 to test pruning - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=300)) + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900)) self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/" self.address[0] = self.nodes[0].getnewaddress() @@ -108,7 +109,7 @@ class PruneTest(BitcoinTestFramework): # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300) + self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900) # Mine 24 blocks in node 1 self.utxo = self.nodes[1].listunspent() for i in xrange(24): @@ -135,7 +136,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) stop_node(self.nodes[1],1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300) + self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) height = self.nodes[1].getblockcount() print "Current block height:", height @@ -158,7 +159,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node1 to clear those giant tx's from mempool stop_node(self.nodes[1],1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300) + self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) print "Generating new longer chain of 300 more blocks" self.nodes[1].generate(300) @@ -223,7 +224,7 @@ class PruneTest(BitcoinTestFramework): waitstart = time.time() while self.nodes[2].getblockcount() < goalbestheight: time.sleep(0.1) - if time.time() - waitstart > 300: + if time.time() - waitstart > 900: raise AssertionError("Node 2 didn't reorg to proper height") assert(self.nodes[2].getbestblockhash() == goalbesthash) # Verify we can now have the data for a block previously pruned @@ -256,7 +257,7 @@ class PruneTest(BitcoinTestFramework): def run_test(self): - print "Warning! This test requires 4GB of disk space and takes over 30 mins" + print "Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)" print "Mining a big blockchain of 995 blocks" self.create_big_chain() # Chain diagram key: diff --git a/qa/rpc-tests/smartfees.py b/qa/rpc-tests/smartfees.py index 4eb8bb4842..69f3c22c17 100755 --- a/qa/rpc-tests/smartfees.py +++ b/qa/rpc-tests/smartfees.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -# Copyright (c) 2014 The Bitcoin Core developers +# Copyright (c) 2014-2015 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,82 +11,249 @@ from test_framework import BitcoinTestFramework from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException from util import * +# Construct 2 trivial P2SH's and the ScriptSigs that spend them +# So we can create many many transactions without needing to spend +# time signing. +P2SH_1 = "2MySexEGVzZpRgNQ1JdjdP5bRETznm3roQ2" # P2SH of "OP_1 OP_DROP" +P2SH_2 = "2NBdpwq8Aoo1EEKEXPNrKvr5xQr3M9UfcZA" # P2SH of "OP_2 OP_DROP" +# Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2 +# 4 bytes of OP_TRUE and push 2-byte redeem script of "OP_1 OP_DROP" or "OP_2 OP_DROP" +SCRIPT_SIG = ["0451025175", "0451025275"] + +def satoshi_round(amount): + return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) + +def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment): + ''' + Create and send a transaction with a random fee. + The transaction pays to a trival P2SH script, and assumes that its inputs + are of the same form. + The function takes a list of confirmed outputs and unconfirmed outputs + and attempts to use the confirmed list first for its inputs. + It adds the newly created outputs to the unconfirmed list. + Returns (raw transaction, fee) + ''' + # It's best to exponentially distribute our random fees + # because the buckets are exponentially spaced. + # Exponentially distributed from 1-128 * fee_increment + rand_fee = float(fee_increment)*(1.1892**random.randint(0,28)) + # Total fee ranges from min_fee to min_fee + 127*fee_increment + fee = min_fee - fee_increment + satoshi_round(rand_fee) + inputs = [] + total_in = Decimal("0.00000000") + while total_in <= (amount + fee) and len(conflist) > 0: + t = conflist.pop(0) + total_in += t["amount"] + inputs.append({ "txid" : t["txid"], "vout" : t["vout"]} ) + if total_in <= amount + fee: + while total_in <= (amount + fee) and len(unconflist) > 0: + t = unconflist.pop(0) + total_in += t["amount"] + inputs.append({ "txid" : t["txid"], "vout" : t["vout"]} ) + if total_in <= amount + fee: + raise RuntimeError("Insufficient funds: need %d, have %d"%(amount+fee, total_in)) + outputs = {} + outputs[P2SH_1] = total_in - amount - fee + outputs[P2SH_2] = amount + rawtx = from_node.createrawtransaction(inputs, outputs) + # Createrawtransaction constructions a transaction that is ready to be signed + # These transactions don't need to be signed, but we still have to insert the ScriptSig + # that will satisfy the ScriptPubKey. + completetx = rawtx[0:10] + inputnum = 0 + for inp in inputs: + completetx += rawtx[10+82*inputnum:82+82*inputnum] + completetx += SCRIPT_SIG[inp["vout"]] + completetx += rawtx[84+82*inputnum:92+82*inputnum] + inputnum += 1 + completetx += rawtx[10+82*inputnum:] + txid = from_node.sendrawtransaction(completetx, True) + unconflist.append({ "txid" : txid, "vout" : 0 , "amount" : total_in - amount - fee}) + unconflist.append({ "txid" : txid, "vout" : 1 , "amount" : amount}) + + return (completetx, fee) + +def split_inputs(from_node, txins, txouts, initial_split = False): + ''' + We need to generate a lot of very small inputs so we can generate a ton of transactions + and they will have low priority. + This function takes an input from txins, and creates and sends a transaction + which splits the value into 2 outputs which are appended to txouts. + ''' + prevtxout = txins.pop() + inputs = [] + outputs = {} + inputs.append({ "txid" : prevtxout["txid"], "vout" : prevtxout["vout"] }) + half_change = satoshi_round(prevtxout["amount"]/2) + rem_change = prevtxout["amount"] - half_change - Decimal("0.00001000") + outputs[P2SH_1] = half_change + outputs[P2SH_2] = rem_change + rawtx = from_node.createrawtransaction(inputs, outputs) + # If this is the initial split we actually need to sign the transaction + # Otherwise we just need to insert the property ScriptSig + if (initial_split) : + completetx = from_node.signrawtransaction(rawtx)["hex"] + else : + completetx = rawtx[0:82] + SCRIPT_SIG[prevtxout["vout"]] + rawtx[84:] + txid = from_node.sendrawtransaction(completetx, True) + txouts.append({ "txid" : txid, "vout" : 0 , "amount" : half_change}) + txouts.append({ "txid" : txid, "vout" : 1 , "amount" : rem_change}) + +def check_estimates(node, fees_seen, max_invalid, print_estimates = True): + ''' + This function calls estimatefee and verifies that the estimates + meet certain invariants. + ''' + all_estimates = [ node.estimatefee(i) for i in range(1,26) ] + if print_estimates: + print([str(all_estimates[e-1]) for e in [1,2,3,6,15,25]]) + delta = 1.0e-6 # account for rounding error + last_e = max(fees_seen) + for e in filter(lambda x: x >= 0, all_estimates): + # Estimates should be within the bounds of what transactions fees actually were: + if float(e)+delta < min(fees_seen) or float(e)-delta > max(fees_seen): + raise AssertionError("Estimated fee (%f) out of range (%f,%f)" + %(float(e), min(fees_seen), max(fees_seen))) + # Estimates should be monotonically decreasing + if float(e)-delta > last_e: + raise AssertionError("Estimated fee (%f) larger than last fee (%f) for lower number of confirms" + %(float(e),float(last_e))) + last_e = e + valid_estimate = False + invalid_estimates = 0 + for e in all_estimates: + if e >= 0: + valid_estimate = True + else: + invalid_estimates += 1 + # Once we're at a high enough confirmation count that we can give an estimate + # We should have estimates for all higher confirmation counts + if valid_estimate and e < 0: + raise AssertionError("Invalid estimate appears at higher confirm count than valid estimate") + # Check on the expected number of different confirmation counts + # that we might not have valid estimates for + if invalid_estimates > max_invalid: + raise AssertionError("More than (%d) invalid estimates"%(max_invalid)) + return all_estimates + + class EstimateFeeTest(BitcoinTestFramework): def setup_network(self): + ''' + We'll setup the network to have 3 nodes that all mine with different parameters. + But first we need to use one node to create a lot of small low priority outputs + which we will use to generate our transactions. + ''' self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, - ["-debug=mempool", "-debug=estimatefee", "-relaypriority=0"])) - # Node1 mines small-but-not-tiny blocks, and allows free transactions. + # Use node0 to mine blocks for input splitting + self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", + "-relaypriority=0", "-whitelist=127.0.0.1"])) + + print("This test is time consuming, please be patient") + print("Splitting inputs to small size so we can generate low priority tx's") + self.txouts = [] + self.txouts2 = [] + # Split a coinbase into two transaction puzzle outputs + split_inputs(self.nodes[0], self.nodes[0].listunspent(0), self.txouts, True) + + # Mine + while (len(self.nodes[0].getrawmempool()) > 0): + self.nodes[0].generate(1) + + # Repeatedly split those 2 outputs, doubling twice for each rep + # Use txouts to monitor the available utxo, since these won't be tracked in wallet + reps = 0 + while (reps < 5): + #Double txouts to txouts2 + while (len(self.txouts)>0): + split_inputs(self.nodes[0], self.txouts, self.txouts2) + while (len(self.nodes[0].getrawmempool()) > 0): + self.nodes[0].generate(1) + #Double txouts2 to txouts + while (len(self.txouts2)>0): + split_inputs(self.nodes[0], self.txouts2, self.txouts) + while (len(self.nodes[0].getrawmempool()) > 0): + self.nodes[0].generate(1) + reps += 1 + print("Finished splitting") + + # Now we can connect the other nodes, didn't want to connect them earlier + # so the estimates would not be affected by the splitting transactions + # Node1 mines small blocks but that are bigger than the expected transaction rate, + # and allows free transactions. # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes, - # so blockmaxsize of 2,000 is really just 1,000 bytes (room enough for - # 6 or 7 transactions) + # (17k is room enough for 110 or so transactions) self.nodes.append(start_node(1, self.options.tmpdir, - ["-blockprioritysize=1500", "-blockmaxsize=2000", - "-debug=mempool", "-debug=estimatefee", "-relaypriority=0"])) + ["-blockprioritysize=1500", "-blockmaxsize=18000", + "-maxorphantx=1000", "-relaypriority=0", "-debug=estimatefee"])) connect_nodes(self.nodes[1], 0) # Node2 is a stingy miner, that - # produces very small blocks (room for only 3 or so transactions) - node2args = [ "-blockprioritysize=0", "-blockmaxsize=1500", - "-debug=mempool", "-debug=estimatefee", "-relaypriority=0"] + # produces too small blocks (room for only 70 or so transactions) + node2args = ["-blockprioritysize=0", "-blockmaxsize=12000", "-maxorphantx=1000", "-relaypriority=0"] + self.nodes.append(start_node(2, self.options.tmpdir, node2args)) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[2], 1) self.is_network_split = False self.sync_all() - + + def transact_and_mine(self, numblocks, mining_node): + min_fee = Decimal("0.00001") + # We will now mine numblocks blocks generating on average 100 transactions between each block + # We shuffle our confirmed txout set before each set of transactions + # small_txpuzzle_randfee will use the transactions that have inputs already in the chain when possible + # resorting to tx's that depend on the mempool when those run out + for i in range(numblocks): + random.shuffle(self.confutxo) + for j in range(random.randrange(100-50,100+50)): + from_index = random.randint(1,2) + (txhex, fee) = small_txpuzzle_randfee(self.nodes[from_index], self.confutxo, + self.memutxo, Decimal("0.005"), min_fee, min_fee) + tx_kbytes = (len(txhex)/2)/1000.0 + self.fees_per_kb.append(float(fee)/tx_kbytes) + sync_mempools(self.nodes[0:3],.1) + mined = mining_node.getblock(mining_node.generate(1)[0],True)["tx"] + sync_blocks(self.nodes[0:3],.1) + #update which txouts are confirmed + newmem = [] + for utx in self.memutxo: + if utx["txid"] in mined: + self.confutxo.append(utx) + else: + newmem.append(utx) + self.memutxo = newmem def run_test(self): - # Prime the memory pool with pairs of transactions - # (high-priority, random fee and zero-priority, random fee) - min_fee = Decimal("0.001") - fees_per_kb = []; - for i in range(12): - (txid, txhex, fee) = random_zeropri_transaction(self.nodes, Decimal("1.1"), - min_fee, min_fee, 20) - tx_kbytes = (len(txhex)/2)/1000.0 - fees_per_kb.append(float(fee)/tx_kbytes) - - # Mine blocks with node2 until the memory pool clears: - count_start = self.nodes[2].getblockcount() - while len(self.nodes[2].getrawmempool()) > 0: - self.nodes[2].generate(1) - self.sync_all() - - all_estimates = [ self.nodes[0].estimatefee(i) for i in range(1,20) ] - print("Fee estimates, super-stingy miner: "+str([str(e) for e in all_estimates])) + self.fees_per_kb = [] + self.memutxo = [] + self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting + print("Checking estimates for 1/2/3/6/15/25 blocks") + print("Creating transactions and mining them with a huge block size") + # Create transactions and mine 20 big blocks with node 0 such that the mempool is always emptied + self.transact_and_mine(30, self.nodes[0]) + check_estimates(self.nodes[1], self.fees_per_kb, 1) - # Estimates should be within the bounds of what transactions fees actually were: - delta = 1.0e-6 # account for rounding error - for e in filter(lambda x: x >= 0, all_estimates): - if float(e)+delta < min(fees_per_kb) or float(e)-delta > max(fees_per_kb): - raise AssertionError("Estimated fee (%f) out of range (%f,%f)"%(float(e), min_fee_kb, max_fee_kb)) - - # Generate transactions while mining 30 more blocks, this time with node1: - for i in range(30): - for j in range(random.randrange(6-4,6+4)): - (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), - Decimal("0.0"), min_fee, 20) - tx_kbytes = (len(txhex)/2)/1000.0 - fees_per_kb.append(float(fee)/tx_kbytes) - self.nodes[1].generate(1) - self.sync_all() + print("Creating transactions and mining them with a block size that can't keep up") + # Create transactions and mine 30 small blocks with node 2, but create txs faster than we can mine + self.transact_and_mine(20, self.nodes[2]) + check_estimates(self.nodes[1], self.fees_per_kb, 3) - all_estimates = [ self.nodes[0].estimatefee(i) for i in range(1,20) ] - print("Fee estimates, more generous miner: "+str([ str(e) for e in all_estimates])) - for e in filter(lambda x: x >= 0, all_estimates): - if float(e)+delta < min(fees_per_kb) or float(e)-delta > max(fees_per_kb): - raise AssertionError("Estimated fee (%f) out of range (%f,%f)"%(float(e), min_fee_kb, max_fee_kb)) + print("Creating transactions and mining them at a block size that is just big enough") + # Generate transactions while mining 40 more blocks, this time with node1 + # which mines blocks with capacity just above the rate that transactions are being created + self.transact_and_mine(40, self.nodes[1]) + check_estimates(self.nodes[1], self.fees_per_kb, 2) # Finish by mining a normal-sized block: - while len(self.nodes[0].getrawmempool()) > 0: - self.nodes[0].generate(1) - self.sync_all() - - final_estimates = [ self.nodes[0].estimatefee(i) for i in range(1,20) ] - print("Final fee estimates: "+str([ str(e) for e in final_estimates])) + while len(self.nodes[1].getrawmempool()) > 0: + self.nodes[1].generate(1) + sync_blocks(self.nodes[0:3],.1) + print("Final estimates after emptying mempools") + check_estimates(self.nodes[1], self.fees_per_kb, 2) if __name__ == '__main__': EstimateFeeTest().main() diff --git a/qa/rpc-tests/util.py b/qa/rpc-tests/util.py index 3b4a10e46b..997bbcc373 100644 --- a/qa/rpc-tests/util.py +++ b/qa/rpc-tests/util.py @@ -33,7 +33,7 @@ def check_json_precision(): if satoshis != 2000000000000003: raise RuntimeError("JSON encode/decode loses precision") -def sync_blocks(rpc_connections): +def sync_blocks(rpc_connections, wait=1): """ Wait until everybody has the same block count """ @@ -41,9 +41,9 @@ def sync_blocks(rpc_connections): counts = [ x.getblockcount() for x in rpc_connections ] if counts == [ counts[0] ]*len(counts): break - time.sleep(1) + time.sleep(wait) -def sync_mempools(rpc_connections): +def sync_mempools(rpc_connections, wait=1): """ Wait until everybody has the same transactions in their memory pools @@ -56,7 +56,7 @@ def sync_mempools(rpc_connections): num_match = num_match+1 if num_match == len(rpc_connections): break - time.sleep(1) + time.sleep(wait) bitcoind_processes = {} diff --git a/src/Makefile.am b/src/Makefile.am index d85ce0f081..91d530222f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -107,6 +107,7 @@ BITCOIN_CORE_H = \ netbase.h \ net.h \ noui.h \ + policy/fees.h \ pow.h \ primitives/block.h \ primitives/transaction.h \ @@ -116,6 +117,7 @@ BITCOIN_CORE_H = \ rpcclient.h \ rpcprotocol.h \ rpcserver.h \ + scheduler.h \ script/interpreter.h \ script/script_error.h \ script/script.h \ @@ -183,6 +185,7 @@ libbitcoin_server_a_SOURCES = \ miner.cpp \ net.cpp \ noui.cpp \ + policy/fees.cpp \ pow.cpp \ rest.cpp \ rpcblockchain.cpp \ @@ -258,6 +261,7 @@ libbitcoin_common_a_SOURCES = \ netbase.cpp \ protocol.cpp \ pubkey.cpp \ + scheduler.cpp \ script/interpreter.cpp \ script/script.cpp \ script/sign.cpp \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 31fe3a9f69..6b7c42285d 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -256,6 +256,7 @@ RES_ICONS = \ qt/res/icons/tx_input.png \ qt/res/icons/tx_output.png \ qt/res/icons/tx_mined.png \ + qt/res/icons/warning.png \ qt/res/icons/verify.png BITCOIN_QT_CPP = \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9aec13ccef..0997148117 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -57,9 +57,11 @@ BITCOIN_TESTS =\ test/multisig_tests.cpp \ test/netbase_tests.cpp \ test/pmt_tests.cpp \ + test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ + test/scheduler_tests.cpp \ test/script_P2SH_tests.cpp \ test/script_tests.cpp \ test/scriptnum_tests.cpp \ diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index eeca8655c9..cce687ac98 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -8,6 +8,7 @@ #include "init.h" #include "main.h" #include "noui.h" +#include "scheduler.h" #include "util.h" #include <boost/algorithm/string/predicate.hpp> @@ -55,6 +56,7 @@ void WaitForShutdown(boost::thread_group* threadGroup) bool AppInit(int argc, char* argv[]) { boost::thread_group threadGroup; + CScheduler scheduler; bool fRet = false; @@ -142,7 +144,7 @@ bool AppInit(int argc, char* argv[]) #endif SoftSetBoolArg("-server", true); - fRet = AppInit2(threadGroup); + fRet = AppInit2(threadGroup, scheduler); } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInit()"); diff --git a/src/init.cpp b/src/init.cpp index 500206b70c..2d75850253 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -19,6 +19,7 @@ #include "net.h" #include "rpcserver.h" #include "script/standard.h" +#include "scheduler.h" #include "txdb.h" #include "ui_interface.h" #include "util.h" @@ -564,7 +565,7 @@ bool InitSanityCheck(void) /** Initialize bitcoin. * @pre Parameters should be parsed and config file should be read. */ -bool AppInit2(boost::thread_group& threadGroup) +bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -834,13 +835,13 @@ bool AppInit2(boost::thread_group& threadGroup) } } nTxConfirmTarget = GetArg("-txconfirmtarget", 1); - bSpendZeroConfChange = GetArg("-spendzeroconfchange", true); - fSendFreeTransactions = GetArg("-sendfreetransactions", false); + bSpendZeroConfChange = GetBoolArg("-spendzeroconfchange", true); + fSendFreeTransactions = GetBoolArg("-sendfreetransactions", false); std::string strWalletFile = GetArg("-wallet", "wallet.dat"); #endif // ENABLE_WALLET - fIsBareMultisigStd = GetArg("-permitbaremultisig", true) != 0; + fIsBareMultisigStd = GetBoolArg("-permitbaremultisig", true); nMaxDatacarrierBytes = GetArg("-datacarriersize", nMaxDatacarrierBytes); // ********************************************************* Step 4: application initialization: dir lock, daemonize, pidfile, debug log @@ -890,6 +891,10 @@ bool AppInit2(boost::thread_group& threadGroup) threadGroup.create_thread(&ThreadScriptCheck); } + // Start the lightweight task scheduler thread + CScheduler::Function serviceLoop = boost::bind(&CScheduler::serviceQueue, &scheduler); + threadGroup.create_thread(boost::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections * that the server is there and will be ready later). Warmup mode will @@ -955,7 +960,7 @@ bool AppInit2(boost::thread_group& threadGroup) proxyType addrProxy; bool fProxy = false; if (mapArgs.count("-proxy")) { - addrProxy = proxyType(CService(mapArgs["-proxy"], 9050), GetArg("-proxyrandomize", true)); + addrProxy = proxyType(CService(mapArgs["-proxy"], 9050), GetBoolArg("-proxyrandomize", true)); if (!addrProxy.IsValid()) return InitError(strprintf(_("Invalid -proxy address: '%s'"), mapArgs["-proxy"])); @@ -972,7 +977,7 @@ bool AppInit2(boost::thread_group& threadGroup) if (!mapArgs.count("-onion")) addrOnion = addrProxy; else - addrOnion = proxyType(CService(mapArgs["-onion"], 9050), GetArg("-proxyrandomize", true)); + addrOnion = proxyType(CService(mapArgs["-onion"], 9050), GetBoolArg("-proxyrandomize", true)); if (!addrOnion.IsValid()) return InitError(strprintf(_("Invalid -onion address: '%s'"), mapArgs["-onion"])); SetProxy(NET_TOR, addrOnion); @@ -1375,7 +1380,7 @@ bool AppInit2(boost::thread_group& threadGroup) LogPrintf("mapAddressBook.size() = %u\n", pwalletMain ? pwalletMain->mapAddressBook.size() : 0); #endif - StartNode(threadGroup); + StartNode(threadGroup, scheduler); #ifdef ENABLE_WALLET // Generate coins in the background diff --git a/src/init.h b/src/init.h index d451f65be9..dcb2b29360 100644 --- a/src/init.h +++ b/src/init.h @@ -8,6 +8,7 @@ #include <string> +class CScheduler; class CWallet; namespace boost @@ -20,7 +21,7 @@ extern CWallet* pwalletMain; void StartShutdown(); bool ShutdownRequested(); void Shutdown(); -bool AppInit2(boost::thread_group& threadGroup); +bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler); /** The help message mode determines what help message to show */ enum HelpMessageMode { diff --git a/src/main.cpp b/src/main.cpp index 94ab7ff7bb..794edc30fb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -987,7 +987,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa CAmount nFees = nValueIn-nValueOut; double dPriority = view.GetPriority(tx, chainActive.Height()); - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height()); + CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), mempool.HasNoInputsOf(tx)); unsigned int nSize = entry.GetTxSize(); // Don't accept it if it can't get into a block @@ -1053,7 +1053,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa } // Store transaction in memory - pool.addUnchecked(hash, entry); + pool.addUnchecked(hash, entry, !IsInitialBlockDownload()); } SyncWithWallets(tx, NULL); @@ -2115,7 +2115,7 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock * LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool. list<CTransaction> txConflicted; - mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted); + mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); mempool.check(pcoinsTip); // Update chainActive & related variables. UpdateTip(pindexNew); @@ -2960,7 +2960,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) return; } - unsigned int nLastBlockWeMustKeep = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + unsigned int nLastBlockWeCanPrune = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files // So we should leave a buffer under our target to account for another allocation @@ -2980,7 +2980,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) break; // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip - if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeMustKeep) + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) break; PruneOneBlockFile(fileNumber); @@ -2991,10 +2991,10 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) } } - LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB min_must_keep=%d removed %d blk/rev pairs\n", + LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", nPruneTarget/1024/1024, nCurrentUsage/1024/1024, ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, - nLastBlockWeMustKeep, count); + nLastBlockWeCanPrune, count); } bool CheckDiskSpace(uint64_t nAdditionalBytes) diff --git a/src/miner.cpp b/src/miner.cpp index 56a2c5828b..4bceb7d7b4 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -453,8 +453,16 @@ void static BitcoinMiner(CWallet *pwallet) if (chainparams.MiningRequiresPeers()) { // Busy-wait for the network to come online so we don't waste time mining // on an obsolete chain. In regtest mode we expect to fly solo. - while (vNodes.empty()) + do { + bool fvNodesEmpty; + { + LOCK(cs_vNodes); + fvNodesEmpty = vNodes.empty(); + } + if (!fvNodesEmpty && !IsInitialBlockDownload()) + break; MilliSleep(1000); + } while (true); } // @@ -533,6 +541,11 @@ void static BitcoinMiner(CWallet *pwallet) LogPrintf("BitcoinMiner terminated\n"); throw; } + catch (const std::runtime_error &e) + { + LogPrintf("BitcoinMiner runtime error: %s\n", e.what()); + return; + } } void GenerateBitcoins(bool fGenerate, CWallet* pwallet, int nThreads) diff --git a/src/net.cpp b/src/net.cpp index 2de04fc574..6849d79263 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -13,6 +13,7 @@ #include "chainparams.h" #include "clientversion.h" #include "primitives/transaction.h" +#include "scheduler.h" #include "ui_interface.h" #include "crypto/common.h" @@ -1590,7 +1591,7 @@ void static Discover(boost::thread_group& threadGroup) #endif } -void StartNode(boost::thread_group& threadGroup) +void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler) { uiInterface.InitMessage(_("Loading addresses...")); // Load addresses for peers.dat @@ -1640,7 +1641,7 @@ void StartNode(boost::thread_group& threadGroup) threadGroup.create_thread(boost::bind(&TraceThread<void (*)()>, "msghand", &ThreadMessageHandler)); // Dump network addresses - threadGroup.create_thread(boost::bind(&LoopForever<void (*)()>, "dumpaddr", &DumpAddresses, DUMP_ADDRESSES_INTERVAL * 1000)); + scheduler.scheduleEvery(&DumpAddresses, DUMP_ADDRESSES_INTERVAL); } bool StopNode() @@ -32,6 +32,7 @@ class CAddrMan; class CBlockIndex; +class CScheduler; class CNode; namespace boost { @@ -72,7 +73,7 @@ bool OpenNetworkConnection(const CAddress& addrConnect, CSemaphoreGrant *grantOu void MapPort(bool fUseUPnP); unsigned short GetListenPort(); bool BindListenPort(const CService &bindAddr, std::string& strError, bool fWhitelisted = false); -void StartNode(boost::thread_group& threadGroup); +void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler); bool StopNode(); void SocketSendData(CNode *pnode); diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp new file mode 100644 index 0000000000..b1491cec01 --- /dev/null +++ b/src/policy/fees.cpp @@ -0,0 +1,529 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "policy/fees.h" + +#include "amount.h" +#include "primitives/transaction.h" +#include "streams.h" +#include "txmempool.h" +#include "util.h" + +void TxConfirmStats::Initialize(std::vector<double>& defaultBuckets, + unsigned int maxConfirms, double _decay, std::string _dataTypeString) +{ + decay = _decay; + dataTypeString = _dataTypeString; + for (unsigned int i = 0; i < defaultBuckets.size(); i++) { + buckets.push_back(defaultBuckets[i]); + bucketMap[defaultBuckets[i]] = i; + } + confAvg.resize(maxConfirms); + curBlockConf.resize(maxConfirms); + unconfTxs.resize(maxConfirms); + for (unsigned int i = 0; i < maxConfirms; i++) { + confAvg[i].resize(buckets.size()); + curBlockConf[i].resize(buckets.size()); + unconfTxs[i].resize(buckets.size()); + } + + oldUnconfTxs.resize(buckets.size()); + curBlockTxCt.resize(buckets.size()); + txCtAvg.resize(buckets.size()); + curBlockVal.resize(buckets.size()); + avg.resize(buckets.size()); +} + +// Zero out the data for the current block +void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) +{ + for (unsigned int j = 0; j < buckets.size(); j++) { + oldUnconfTxs[j] += unconfTxs[nBlockHeight%unconfTxs.size()][j]; + unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0; + for (unsigned int i = 0; i < curBlockConf.size(); i++) + curBlockConf[i][j] = 0; + curBlockTxCt[j] = 0; + curBlockVal[j] = 0; + } +} + + +void TxConfirmStats::Record(int blocksToConfirm, double val) +{ + // blocksToConfirm is 1-based + if (blocksToConfirm < 1) + return; + unsigned int bucketindex = bucketMap.lower_bound(val)->second; + for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { + curBlockConf[i - 1][bucketindex]++; + } + curBlockTxCt[bucketindex]++; + curBlockVal[bucketindex] += val; +} + +void TxConfirmStats::UpdateMovingAverages() +{ + for (unsigned int j = 0; j < buckets.size(); j++) { + for (unsigned int i = 0; i < confAvg.size(); i++) + confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; + avg[j] = avg[j] * decay + curBlockVal[j]; + txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; + } +} + +// returns -1 on error conditions +double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, + double successBreakPoint, bool requireGreater, + unsigned int nBlockHeight) +{ + // Counters for a bucket (or range of buckets) + double nConf = 0; // Number of tx's confirmed within the confTarget + double totalNum = 0; // Total number of tx's that were ever confirmed + int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + + int maxbucketindex = buckets.size() - 1; + + // requireGreater means we are looking for the lowest fee/priority such that all higher + // values pass, so we start at maxbucketindex (highest fee) and look at succesively + // smaller buckets until we reach failure. Otherwise, we are looking for the highest + // fee/priority such that all lower values fail, and we go in the opposite direction. + unsigned int startbucket = requireGreater ? maxbucketindex : 0; + int step = requireGreater ? -1 : 1; + + // We'll combine buckets until we have enough samples. + // The near and far variables will define the range we've combined + // The best variables are the last range we saw which still had a high + // enough confirmation rate to count as success. + // The cur variables are the current range we're counting. + unsigned int curNearBucket = startbucket; + unsigned int bestNearBucket = startbucket; + unsigned int curFarBucket = startbucket; + unsigned int bestFarBucket = startbucket; + + bool foundAnswer = false; + unsigned int bins = unconfTxs.size(); + + // Start counting from highest(default) or lowest fee/pri transactions + for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + curFarBucket = bucket; + nConf += confAvg[confTarget - 1][bucket]; + totalNum += txCtAvg[bucket]; + for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) + extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; + extraNum += oldUnconfTxs[bucket]; + // If we have enough transaction data points in this range of buckets, + // we can test for success + // (Only count the confirmed data points, so that each confirmation count + // will be looking at the same amount of data and same bucket breaks) + if (totalNum >= sufficientTxVal / (1 - decay)) { + double curPct = nConf / (totalNum + extraNum); + + // Check to see if we are no longer getting confirmed at the success rate + if (requireGreater && curPct < successBreakPoint) + break; + if (!requireGreater && curPct > successBreakPoint) + break; + + // Otherwise update the cumulative stats, and the bucket variables + // and reset the counters + else { + foundAnswer = true; + nConf = 0; + totalNum = 0; + extraNum = 0; + bestNearBucket = curNearBucket; + bestFarBucket = curFarBucket; + curNearBucket = bucket + step; + } + } + } + + double median = -1; + double txSum = 0; + + // Calculate the "average" fee of the best bucket range that met success conditions + // Find the bucket with the median transaction and then report the average fee from that bucket + // This is a compromise between finding the median which we can't since we don't save all tx's + // and reporting the average which is less accurate + unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; + unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + for (unsigned int j = minBucket; j <= maxBucket; j++) { + txSum += txCtAvg[j]; + } + if (foundAnswer && txSum != 0) { + txSum = txSum / 2; + for (unsigned int j = minBucket; j <= maxBucket; j++) { + if (txCtAvg[j] < txSum) + txSum -= txCtAvg[j]; + else { // we're in the right bucket + median = avg[j] / txCtAvg[j]; + break; + } + } + } + + LogPrint("estimatefee", "%3d: For conf success %s %4.2f need %s %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", + confTarget, requireGreater ? ">" : "<", successBreakPoint, dataTypeString, + requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], + 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); + + return median; +} + +void TxConfirmStats::Write(CAutoFile& fileout) +{ + fileout << decay; + fileout << buckets; + fileout << avg; + fileout << txCtAvg; + fileout << confAvg; +} + +void TxConfirmStats::Read(CAutoFile& filein) +{ + // Read data file into temporary variables and do some very basic sanity checking + std::vector<double> fileBuckets; + std::vector<double> fileAvg; + std::vector<std::vector<double> > fileConfAvg; + std::vector<double> fileTxCtAvg; + double fileDecay; + size_t maxConfirms; + size_t numBuckets; + + filein >> fileDecay; + if (fileDecay <= 0 || fileDecay >= 1) + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + filein >> fileBuckets; + numBuckets = fileBuckets.size(); + if (numBuckets <= 1 || numBuckets > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 fee/pri buckets"); + filein >> fileAvg; + if (fileAvg.size() != numBuckets) + throw std::runtime_error("Corrupt estimates file. Mismatch in fee/pri average bucket count"); + filein >> fileTxCtAvg; + if (fileTxCtAvg.size() != numBuckets) + throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); + filein >> fileConfAvg; + maxConfirms = fileConfAvg.size(); + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) // one week + throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + for (unsigned int i = 0; i < maxConfirms; i++) { + if (fileConfAvg[i].size() != numBuckets) + throw std::runtime_error("Corrupt estimates file. Mismatch in fee/pri conf average bucket count"); + } + // Now that we've processed the entire fee estimate data file and not + // thrown any errors, we can copy it to our data structures + decay = fileDecay; + buckets = fileBuckets; + avg = fileAvg; + confAvg = fileConfAvg; + txCtAvg = fileTxCtAvg; + bucketMap.clear(); + + // Resize the current block variables which aren't stored in the data file + // to match the number of confirms and buckets + curBlockConf.resize(maxConfirms); + for (unsigned int i = 0; i < maxConfirms; i++) { + curBlockConf[i].resize(buckets.size()); + } + curBlockTxCt.resize(buckets.size()); + curBlockVal.resize(buckets.size()); + + unconfTxs.resize(maxConfirms); + for (unsigned int i = 0; i < maxConfirms; i++) { + unconfTxs[i].resize(buckets.size()); + } + oldUnconfTxs.resize(buckets.size()); + + for (unsigned int i = 0; i < buckets.size(); i++) + bucketMap[buckets[i]] = i; + + LogPrint("estimatefee", "Reading estimates: %u %s buckets counting confirms up to %u blocks\n", + numBuckets, dataTypeString, maxConfirms); +} + +unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) +{ + unsigned int bucketindex = bucketMap.lower_bound(val)->second; + unsigned int blockIndex = nBlockHeight % unconfTxs.size(); + unconfTxs[blockIndex][bucketindex]++; + LogPrint("estimatefee", "adding to %s\n", dataTypeString); + return bucketindex; +} + +void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) +{ + //nBestSeenHeight is not updated yet for the new block + int blocksAgo = nBestSeenHeight - entryHeight; + if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet + blocksAgo = 0; + if (blocksAgo < 0) { + LogPrint("estimatefee", "Blockpolicy error, blocks ago is negative for mempool tx\n"); + return; //This can't happen becasue we call this with our best seen height, no entries can have higher + } + + if (blocksAgo >= (int)unconfTxs.size()) { + if (oldUnconfTxs[bucketindex] > 0) + oldUnconfTxs[bucketindex]--; + else + LogPrint("estimatefee", "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n", + bucketindex); + } + else { + unsigned int blockIndex = entryHeight % unconfTxs.size(); + if (unconfTxs[blockIndex][bucketindex] > 0) + unconfTxs[blockIndex][bucketindex]--; + else + LogPrint("estimatefee", "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n", + blockIndex, bucketindex); + } +} + +void CBlockPolicyEstimator::removeTx(uint256 hash) +{ + std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash); + if (pos == mapMemPoolTxs.end()) { + LogPrint("estimatefee", "Blockpolicy error mempool tx %s not found for removeTx\n", + hash.ToString().c_str()); + return; + } + TxConfirmStats *stats = pos->second.stats; + unsigned int entryHeight = pos->second.blockHeight; + unsigned int bucketIndex = pos->second.bucketIndex; + + if (stats != NULL) + stats->removeTx(entryHeight, nBestSeenHeight, bucketIndex); + mapMemPoolTxs.erase(hash); +} + +CBlockPolicyEstimator::CBlockPolicyEstimator(const CFeeRate& _minRelayFee) + : nBestSeenHeight(0) +{ + minTrackedFee = _minRelayFee < CFeeRate(MIN_FEERATE) ? CFeeRate(MIN_FEERATE) : _minRelayFee; + std::vector<double> vfeelist; + for (double bucketBoundary = minTrackedFee.GetFeePerK(); bucketBoundary <= MAX_FEERATE; bucketBoundary *= FEE_SPACING) { + vfeelist.push_back(bucketBoundary); + } + vfeelist.push_back(INF_FEERATE); + feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, "FeeRate"); + + minTrackedPriority = AllowFreeThreshold() < MIN_PRIORITY ? MIN_PRIORITY : AllowFreeThreshold(); + std::vector<double> vprilist; + for (double bucketBoundary = minTrackedPriority; bucketBoundary <= MAX_PRIORITY; bucketBoundary *= PRI_SPACING) { + vprilist.push_back(bucketBoundary); + } + vprilist.push_back(INF_PRIORITY); + priStats.Initialize(vprilist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, "Priority"); + + feeUnlikely = CFeeRate(0); + feeLikely = CFeeRate(INF_FEERATE); + priUnlikely = 0; + priLikely = INF_PRIORITY; +} + +bool CBlockPolicyEstimator::isFeeDataPoint(const CFeeRate &fee, double pri) +{ + if ((pri < minTrackedPriority && fee >= minTrackedFee) || + (pri < priUnlikely && fee > feeLikely)) { + return true; + } + return false; +} + +bool CBlockPolicyEstimator::isPriDataPoint(const CFeeRate &fee, double pri) +{ + if ((fee < minTrackedFee && pri >= minTrackedPriority) || + (fee < feeUnlikely && pri > priLikely)) { + return true; + } + return false; +} + +void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool fCurrentEstimate) +{ + unsigned int txHeight = entry.GetHeight(); + uint256 hash = entry.GetTx().GetHash(); + if (mapMemPoolTxs[hash].stats != NULL) { + LogPrint("estimatefee", "Blockpolicy error mempool tx %s already being tracked\n", + hash.ToString().c_str()); + return; + } + + if (txHeight < nBestSeenHeight) { + // Ignore side chains and re-orgs; assuming they are random they don't + // affect the estimate. We'll potentially double count transactions in 1-block reorgs. + return; + } + + // Only want to be updating estimates when our blockchain is synced, + // otherwise we'll miscalculate how many blocks its taking to get included. + if (!fCurrentEstimate) + return; + + if (!entry.WasClearAtEntry()) { + // This transaction depends on other transactions in the mempool to + // be included in a block before it will be able to be included, so + // we shouldn't include it in our calculations + return; + } + + // Fees are stored and reported as BTC-per-kb: + CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); + + // Want the priority of the tx at confirmation. However we don't know + // what that will be and its too hard to continue updating it + // so use starting priority as a proxy + double curPri = entry.GetPriority(txHeight); + mapMemPoolTxs[hash].blockHeight = txHeight; + + LogPrint("estimatefee", "Blockpolicy mempool tx %s ", hash.ToString().substr(0,10)); + // Record this as a priority estimate + if (entry.GetFee() == 0 || isPriDataPoint(feeRate, curPri)) { + mapMemPoolTxs[hash].stats = &priStats; + mapMemPoolTxs[hash].bucketIndex = priStats.NewTx(txHeight, curPri); + } + // Record this as a fee estimate + else if (isFeeDataPoint(feeRate, curPri)) { + mapMemPoolTxs[hash].stats = &feeStats; + mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); + } + else { + LogPrint("estimatefee", "not adding\n"); + } +} + +void CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry& entry) +{ + if (!entry.WasClearAtEntry()) { + // This transaction depended on other transactions in the mempool to + // be included in a block before it was able to be included, so + // we shouldn't include it in our calculations + return; + } + + // How many blocks did it take for miners to include this transaction? + // blocksToConfirm is 1-based, so a transaction included in the earliest + // possible block has confirmation count of 1 + int blocksToConfirm = nBlockHeight - entry.GetHeight(); + if (blocksToConfirm <= 0) { + // This can't happen because we don't process transactions from a block with a height + // lower than our greatest seen height + LogPrint("estimatefee", "Blockpolicy error Transaction had negative blocksToConfirm\n"); + return; + } + + // Fees are stored and reported as BTC-per-kb: + CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); + + // Want the priority of the tx at confirmation. The priority when it + // entered the mempool could easily be very small and change quickly + double curPri = entry.GetPriority(nBlockHeight); + + // Record this as a priority estimate + if (entry.GetFee() == 0 || isPriDataPoint(feeRate, curPri)) { + priStats.Record(blocksToConfirm, curPri); + } + // Record this as a fee estimate + else if (isFeeDataPoint(feeRate, curPri)) { + feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + } +} + +void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, + std::vector<CTxMemPoolEntry>& entries, bool fCurrentEstimate) +{ + if (nBlockHeight <= nBestSeenHeight) { + // Ignore side chains and re-orgs; assuming they are random + // they don't affect the estimate. + // And if an attacker can re-org the chain at will, then + // you've got much bigger problems than "attacker can influence + // transaction fees." + return; + } + nBestSeenHeight = nBlockHeight; + + // Only want to be updating estimates when our blockchain is synced, + // otherwise we'll miscalculate how many blocks its taking to get included. + if (!fCurrentEstimate) + return; + + // Update the dynamic cutoffs + // a fee/priority is "likely" the reason your tx was included in a block if >85% of such tx's + // were confirmed in 2 blocks and is "unlikely" if <50% were confirmed in 10 blocks + LogPrint("estimatefee", "Blockpolicy recalculating dynamic cutoffs:\n"); + priLikely = priStats.EstimateMedianVal(2, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBlockHeight); + if (priLikely == -1) + priLikely = INF_PRIORITY; + + double feeLikelyEst = feeStats.EstimateMedianVal(2, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBlockHeight); + if (feeLikelyEst == -1) + feeLikely = CFeeRate(INF_FEERATE); + else + feeLikely = CFeeRate(feeLikelyEst); + + priUnlikely = priStats.EstimateMedianVal(10, SUFFICIENT_PRITXS, UNLIKELY_PCT, false, nBlockHeight); + if (priUnlikely == -1) + priUnlikely = 0; + + double feeUnlikelyEst = feeStats.EstimateMedianVal(10, SUFFICIENT_FEETXS, UNLIKELY_PCT, false, nBlockHeight); + if (feeUnlikelyEst == -1) + feeUnlikely = CFeeRate(0); + else + feeUnlikely = CFeeRate(feeUnlikelyEst); + + // Clear the current block states + feeStats.ClearCurrent(nBlockHeight); + priStats.ClearCurrent(nBlockHeight); + + // Repopulate the current block states + for (unsigned int i = 0; i < entries.size(); i++) + processBlockTx(nBlockHeight, entries[i]); + + // Update all exponential averages with the current block states + feeStats.UpdateMovingAverages(); + priStats.UpdateMovingAverages(); + + LogPrint("estimatefee", "Blockpolicy after updating estimates for %u confirmed entries, new mempool map size %u\n", + entries.size(), mapMemPoolTxs.size()); +} + +CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) +{ + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) + return CFeeRate(0); + + double median = feeStats.EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + + if (median < 0) + return CFeeRate(0); + + return CFeeRate(median); +} + +double CBlockPolicyEstimator::estimatePriority(int confTarget) +{ + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > priStats.GetMaxConfirms()) + return -1; + + return priStats.EstimateMedianVal(confTarget, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); +} + +void CBlockPolicyEstimator::Write(CAutoFile& fileout) +{ + fileout << nBestSeenHeight; + feeStats.Write(fileout); + priStats.Write(fileout); +} + +void CBlockPolicyEstimator::Read(CAutoFile& filein) +{ + int nFileBestSeenHeight; + filein >> nFileBestSeenHeight; + feeStats.Read(filein); + priStats.Read(filein); + nBestSeenHeight = nFileBestSeenHeight; +} diff --git a/src/policy/fees.h b/src/policy/fees.h new file mode 100644 index 0000000000..ce4d782566 --- /dev/null +++ b/src/policy/fees.h @@ -0,0 +1,276 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef BITCOIN_POLICYESTIMATOR_H +#define BITCOIN_POLICYESTIMATOR_H + +#include "amount.h" +#include "uint256.h" + +#include <map> +#include <string> +#include <vector> + +class CAutoFile; +class CFeeRate; +class CTxMemPoolEntry; + +/** \class CBlockPolicyEstimator + * The BlockPolicyEstimator is used for estimating the fee or priority needed + * for a transaction to be included in a block within a certain number of + * blocks. + * + * At a high level the algorithm works by grouping transactions into buckets + * based on having similar priorities or fees and then tracking how long it + * takes transactions in the various buckets to be mined. It operates under + * the assumption that in general transactions of higher fee/priority will be + * included in blocks before transactions of lower fee/priority. So for + * example if you wanted to know what fee you should put on a transaction to + * be included in a block within the next 5 blocks, you would start by looking + * at the bucket with with the highest fee transactions and verifying that a + * sufficiently high percentage of them were confirmed within 5 blocks and + * then you would look at the next highest fee bucket, and so on, stopping at + * the last bucket to pass the test. The average fee of transactions in this + * bucket will give you an indication of the lowest fee you can put on a + * transaction and still have a sufficiently high chance of being confirmed + * within your desired 5 blocks. + * + * When a transaction enters the mempool or is included within a block we + * decide whether it can be used as a data point for fee estimation, priority + * estimation or neither. If the value of exactly one of those properties was + * below the required minimum it can be used to estimate the other. In + * addition, if a priori our estimation code would indicate that the + * transaction would be much more quickly included in a block because of one + * of the properties compared to the other, we can also decide to use it as + * an estimate for that property. + * + * Here is a brief description of the implementation for fee estimation. + * When a transaction that counts for fee estimation enters the mempool, we + * track the height of the block chain at entry. Whenever a block comes in, + * we count the number of transactions in each bucket and the total amount of fee + * paid in each bucket. Then we calculate how many blocks Y it took each + * transaction to be mined and we track an array of counters in each bucket + * for how long it to took transactions to get confirmed from 1 to a max of 25 + * and we increment all the counters from Y up to 25. This is because for any + * number Z>=Y the transaction was successfully mined within Z blocks. We + * want to save a history of this information, so at any time we have a + * counter of the total number of transactions that happened in a given fee + * bucket and the total number that were confirmed in each number 1-25 blocks + * or less for any bucket. We save this history by keeping an exponentially + * decaying moving average of each one of these stats. Furthermore we also + * keep track of the number unmined (in mempool) transactions in each bucket + * and for how many blocks they have been outstanding and use that to increase + * the number of transactions we've seen in that fee bucket when calculating + * an estimate for any number of confirmations below the number of blocks + * they've been outstanding. + */ + +/** + * We will instantiate two instances of this class, one to track transactions + * that were included in a block due to fee, and one for tx's included due to + * priority. We will lump transactions into a bucket according to their approximate + * fee or priority and then track how long it took for those txs to be included in a block + * + * The tracking of unconfirmed (mempool) transactions is completely independent of the + * historical tracking of transactions that have been confirmed in a block. + */ +class TxConfirmStats +{ +private: + //Define the buckets we will group transactions into (both fee buckets and priority buckets) + std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) + std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + // For each bucket X: + // Count the total # of txs in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> txCtAvg; + // and calcuate the total for the current block to update the moving average + std::vector<int> curBlockTxCt; + + // Count the total # of txs confirmed within Y blocks in each bucket + // Track the historical moving average of theses totals over blocks + std::vector<std::vector<double> > confAvg; // confAvg[Y][X] + // and calcuate the totals for the current block to update the moving averages + std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] + + // Sum the total priority/fee of all tx's in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> avg; + // and calculate the total for the current block to update the moving average + std::vector<double> curBlockVal; + + // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X + // Combine the total value with the tx counts to calculate the avg fee/priority per bucket + + std::string dataTypeString; + double decay; + + // Mempool counts of outstanding transactions + // For each bucket X, track the number of transactions in the mempool + // that are unconfirmed for each possible confirmation value Y + std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] + // transactions still unconfirmed after MAX_CONFIRMS for each bucket + std::vector<int> oldUnconfTxs; + +public: + /** + * Initialize the data structures. This is called by BlockPolicyEstimator's + * constructor with default values. + * @param defaultBuckets contains the upper limits for the bucket boundries + * @param maxConfirms max number of confirms to track + * @param decay how much to decay the historical moving average per block + * @param dataTypeString for logging purposes + */ + void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay, std::string dataTypeString); + + /** Clear the state of the curBlock variables to start counting for the new block */ + void ClearCurrent(unsigned int nBlockHeight); + + /** + * Record a new transaction data point in the current block stats + * @param blocksToConfirm the number of blocks it took this transaction to confirm + * @param val either the fee or the priority when entered of the transaction + * @warning blocksToConfirm is 1-based and has to be >= 1 + */ + void Record(int blocksToConfirm, double val); + + /** Record a new transaction entering the mempool*/ + unsigned int NewTx(unsigned int nBlockHeight, double val); + + /** Remove a transaction from mempool tracking stats*/ + void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, + unsigned int bucketIndex); + + /** Update our estimates by decaying our historical moving average and updating + with the data gathered from the current block */ + void UpdateMovingAverages(); + + /** + * Calculate a fee or priority estimate. Find the lowest value bucket (or range of buckets + * to make sure we have enough data points) whose transactions still have sufficient likelihood + * of being confirmed within the target number of confirmations + * @param confTarget target number of confirmations + * @param sufficientTxVal required average number of transactions per block in a bucket range + * @param minSuccess the success probability we require + * @param requireGreater return the lowest fee/pri such that all higher values pass minSuccess OR + * return the highest fee/pri such that all lower values fail minSuccess + * @param nBlockHeight the current block height + */ + double EstimateMedianVal(int confTarget, double sufficientTxVal, + double minSuccess, bool requireGreater, unsigned int nBlockHeight); + + /** Return the max number of confirms we're tracking */ + unsigned int GetMaxConfirms() { return confAvg.size(); } + + /** Write state of estimation data to a file*/ + void Write(CAutoFile& fileout); + + /** + * Read saved state of estimation data from a file and replace all internal data structures and + * variables with this state. + */ + void Read(CAutoFile& filein); +}; + + + +/** Track confirm delays up to 25 blocks, can't estimate beyond that */ +static const unsigned int MAX_BLOCK_CONFIRMS = 25; + +/** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ +static const double DEFAULT_DECAY = .998; + +/** Require greater than 85% of X fee transactions to be confirmed within Y blocks for X to be big enough */ +static const double MIN_SUCCESS_PCT = .85; +static const double UNLIKELY_PCT = .5; + +/** Require an avg of 1 tx in the combined fee bucket per block to have stat significance */ +static const double SUFFICIENT_FEETXS = 1; + +/** Require only an avg of 1 tx every 5 blocks in the combined pri bucket (way less pri txs) */ +static const double SUFFICIENT_PRITXS = .2; + +// Minimum and Maximum values for tracking fees and priorities +static const double MIN_FEERATE = 10; +static const double MAX_FEERATE = 1e7; +static const double INF_FEERATE = MAX_MONEY; +static const double MIN_PRIORITY = 10; +static const double MAX_PRIORITY = 1e16; +static const double INF_PRIORITY = 1e9 * MAX_MONEY; + +// We have to lump transactions into buckets based on fee or priority, but we want to be able +// to give accurate estimates over a large range of potential fees and priorities +// Therefore it makes sense to exponentially space the buckets +/** Spacing of FeeRate buckets */ +static const double FEE_SPACING = 1.1; + +/** Spacing of Priority buckets */ +static const double PRI_SPACING = 2; + +/** + * We want to be able to estimate fees or priorities that are needed on tx's to be included in + * a certain number of blocks. Every time a block is added to the best chain, this class records + * stats on the transactions included in that block + */ +class CBlockPolicyEstimator +{ +public: + /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ + CBlockPolicyEstimator(const CFeeRate& minRelayFee); + + /** Process all the transactions that have been included in a block */ + void processBlock(unsigned int nBlockHeight, + std::vector<CTxMemPoolEntry>& entries, bool fCurrentEstimate); + + /** Process a transaction confirmed in a block*/ + void processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry& entry); + + /** Process a transaction accepted to the mempool*/ + void processTransaction(const CTxMemPoolEntry& entry, bool fCurrentEstimate); + + /** Remove a transaction from the mempool tracking stats*/ + void removeTx(uint256 hash); + + /** Is this transaction likely included in a block because of its fee?*/ + bool isFeeDataPoint(const CFeeRate &fee, double pri); + + /** Is this transaction likely included in a block because of its priority?*/ + bool isPriDataPoint(const CFeeRate &fee, double pri); + + /** Return a fee estimate */ + CFeeRate estimateFee(int confTarget); + + /** Return a priority estimate */ + double estimatePriority(int confTarget); + + /** Write estimation data to a file */ + void Write(CAutoFile& fileout); + + /** Read estimation data from a file */ + void Read(CAutoFile& filein); + +private: + CFeeRate minTrackedFee; //! Passed to constructor to avoid dependency on main + double minTrackedPriority; //! Set to AllowFreeThreshold + unsigned int nBestSeenHeight; + struct TxStatsInfo + { + TxConfirmStats *stats; + unsigned int blockHeight; + unsigned int bucketIndex; + TxStatsInfo() : stats(NULL), blockHeight(0), bucketIndex(0) {} + }; + + // map of txids to information about that transaction + std::map<uint256, TxStatsInfo> mapMemPoolTxs; + + /** Classes to track historical data on transaction confirmations */ + TxConfirmStats feeStats, priStats; + + /** Breakpoints to help determine whether a transaction was confirmed by priority or Fee */ + CFeeRate feeLikely, feeUnlikely; + double priLikely, priUnlikely; +}; +#endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 018169cfdc..8740b98b70 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -26,6 +26,7 @@ #include "init.h" #include "main.h" #include "rpcserver.h" +#include "scheduler.h" #include "ui_interface.h" #include "util.h" @@ -178,6 +179,7 @@ signals: private: boost::thread_group threadGroup; + CScheduler scheduler; /// Pass fatal exception message to UI thread void handleRunawayException(const std::exception *e); @@ -258,7 +260,7 @@ void BitcoinCore::initialize() try { qDebug() << __func__ << ": Running AppInit2 in thread"; - int rv = AppInit2(threadGroup); + int rv = AppInit2(threadGroup, scheduler); if(rv) { /* Start a dummy RPC thread if no RPC thread is active yet diff --git a/src/qt/bitcoin.qrc b/src/qt/bitcoin.qrc index 63af146fd0..c899e95506 100644 --- a/src/qt/bitcoin.qrc +++ b/src/qt/bitcoin.qrc @@ -45,6 +45,7 @@ <file alias="about">res/icons/about.png</file> <file alias="about_qt">res/icons/about_qt.png</file> <file alias="verify">res/icons/verify.png</file> + <file alias="warning">res/icons/warning.png</file> </qresource> <qresource prefix="/movies"> <file alias="spinner-000">res/movies/spinner-000.png</file> diff --git a/src/qt/forms/overviewpage.ui b/src/qt/forms/overviewpage.ui index 53d416ef38..6d792d1475 100644 --- a/src/qt/forms/overviewpage.ui +++ b/src/qt/forms/overviewpage.ui @@ -59,21 +59,35 @@ </widget> </item> <item> - <widget class="QLabel" name="labelWalletStatus"> - <property name="cursor"> - <cursorShape>WhatsThisCursor</cursorShape> + <widget class="QPushButton" name="labelWalletStatus"> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="maximumSize"> + <size> + <width>30</width> + <height>16777215</height> + </size> </property> <property name="toolTip"> <string>The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet.</string> </property> - <property name="styleSheet"> - <string notr="true">QLabel { color: red; }</string> - </property> <property name="text"> - <string notr="true">(out of sync)</string> + <string/> </property> - <property name="alignment"> - <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set> + <property name="icon"> + <iconset resource="../bitcoin.qrc"> + <normaloff>:/icons/warning</normaloff> + <disabledoff>:/icons/warning</disabledoff>:/icons/warning</iconset> + </property> + <property name="iconSize"> + <size> + <width>24</width> + <height>24</height> + </size> + </property> + <property name="flat"> + <bool>true</bool> </property> </widget> </item> @@ -431,21 +445,35 @@ </widget> </item> <item> - <widget class="QLabel" name="labelTransactionsStatus"> - <property name="cursor"> - <cursorShape>WhatsThisCursor</cursorShape> + <widget class="QPushButton" name="labelTransactionsStatus"> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="maximumSize"> + <size> + <width>30</width> + <height>16777215</height> + </size> </property> <property name="toolTip"> <string>The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet.</string> </property> - <property name="styleSheet"> - <string notr="true">QLabel { color: red; }</string> - </property> <property name="text"> - <string notr="true">(out of sync)</string> + <string/> </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> + <property name="icon"> + <iconset resource="../bitcoin.qrc"> + <normaloff>:/icons/warning</normaloff> + <disabledoff>:/icons/warning</disabledoff>:/icons/warning</iconset> + </property> + <property name="iconSize"> + <size> + <width>24</width> + <height>24</height> + </size> + </property> + <property name="flat"> + <bool>true</bool> </property> </widget> </item> diff --git a/src/qt/forms/sendcoinsentry.ui b/src/qt/forms/sendcoinsentry.ui index 48d0dd093c..df06f36833 100644 --- a/src/qt/forms/sendcoinsentry.ui +++ b/src/qt/forms/sendcoinsentry.ui @@ -21,15 +21,21 @@ <string>This is a normal payment.</string> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout"> - <property name="spacing"> + <property name="topMargin"> + <number>8</number> + </property> + <property name="bottomMargin"> + <number>4</number> + </property> + <property name="horizontalSpacing"> <number>12</number> </property> + <property name="verticalSpacing"> + <number>8</number> + </property> <item row="0" column="0"> <widget class="QLabel" name="payToLabel"> <property name="text"> @@ -193,6 +199,13 @@ </property> </widget> </item> + <item row="4" column="0" colspan="2"> + <widget class="Line" name="line"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + </widget> + </item> </layout> </widget> <widget class="QFrame" name="SendCoins_UnauthenticatedPaymentRequest"> @@ -618,10 +631,7 @@ <bool>true</bool> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout_is"> <property name="spacing"> @@ -1150,10 +1160,7 @@ <bool>true</bool> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout_s"> <property name="spacing"> diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 4fa15db9c6..a422ff9a71 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -18,8 +18,8 @@ #include <QAbstractItemDelegate> #include <QPainter> -#define DECORATION_SIZE 64 -#define NUM_ITEMS 3 +#define DECORATION_SIZE 54 +#define NUM_ITEMS 5 class TxViewDelegate : public QAbstractItemDelegate { @@ -129,10 +129,6 @@ OverviewPage::OverviewPage(QWidget *parent) : connect(ui->listTransactions, SIGNAL(clicked(QModelIndex)), this, SLOT(handleTransactionClicked(QModelIndex))); - // init "out of sync" warning labels - ui->labelWalletStatus->setText("(" + tr("out of sync") + ")"); - ui->labelTransactionsStatus->setText("(" + tr("out of sync") + ")"); - // start with displaying the "out of sync" warnings showOutOfSyncWarning(true); } diff --git a/src/qt/res/icons/warning.png b/src/qt/res/icons/warning.png Binary files differnew file mode 100644 index 0000000000..723a30a658 --- /dev/null +++ b/src/qt/res/icons/warning.png diff --git a/src/qt/scicon.cpp b/src/qt/scicon.cpp index a0ffcd82a9..b2f2399b24 100644 --- a/src/qt/scicon.cpp +++ b/src/qt/scicon.cpp @@ -27,12 +27,17 @@ static void MakeSingleColorImage(QImage& img, const QColor& colorbase) QImage SingleColorImage(const QString& filename, const QColor& colorbase) { QImage img(filename); +#if !defined(WIN32) && !defined(MAC_OSX) MakeSingleColorImage(img, colorbase); +#endif return img; } QIcon SingleColorIcon(const QIcon& ico, const QColor& colorbase) { +#if defined(WIN32) || defined(MAC_OSX) + return ico; +#else QIcon new_ico; QSize sz; Q_FOREACH(sz, ico.availableSizes()) @@ -42,6 +47,7 @@ QIcon SingleColorIcon(const QIcon& ico, const QColor& colorbase) new_ico.addPixmap(QPixmap::fromImage(img)); } return new_ico; +#endif } QIcon SingleColorIcon(const QString& filename, const QColor& colorbase) @@ -51,6 +57,9 @@ QIcon SingleColorIcon(const QString& filename, const QColor& colorbase) QColor SingleColor() { +#if defined(WIN32) || defined(MAC_OSX) + return QColor(0,0,0); +#else const QColor colorHighlightBg(QApplication::palette().color(QPalette::Highlight)); const QColor colorHighlightFg(QApplication::palette().color(QPalette::HighlightedText)); const QColor colorText(QApplication::palette().color(QPalette::WindowText)); @@ -61,6 +70,7 @@ QColor SingleColor() else colorbase = colorHighlightFg; return colorbase; +#endif } QIcon SingleColorIcon(const QString& filename) diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 7a33e3567b..3d57711568 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -590,12 +590,12 @@ void SendCoinsDialog::updateGlobalFeeVariables() { if (ui->radioSmartFee->isChecked()) { - nTxConfirmTarget = (int)25 - (int)std::max(0, std::min(24, ui->sliderSmartFee->value())); + nTxConfirmTarget = defaultConfirmTarget - ui->sliderSmartFee->value(); payTxFee = CFeeRate(0); } else { - nTxConfirmTarget = 25; + nTxConfirmTarget = defaultConfirmTarget; payTxFee = CFeeRate(ui->customFee->value()); fPayAtLeastCustomFee = ui->radioCustomAtLeast->isChecked(); } @@ -629,7 +629,7 @@ void SendCoinsDialog::updateSmartFeeLabel() if(!model || !model->getOptionsModel()) return; - int nBlocksToConfirm = (int)25 - (int)std::max(0, std::min(24, ui->sliderSmartFee->value())); + int nBlocksToConfirm = defaultConfirmTarget - ui->sliderSmartFee->value(); CFeeRate feeRate = mempool.estimateFee(nBlocksToConfirm); if (feeRate <= CFeeRate(0)) // not enough data => minfee { diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 14adb02573..fc513bf2ba 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -23,6 +23,8 @@ QT_BEGIN_NAMESPACE class QUrl; QT_END_NAMESPACE +const int defaultConfirmTarget = 25; + /** Dialog for sending bitcoins */ class SendCoinsDialog : public QDialog { diff --git a/src/scheduler.cpp b/src/scheduler.cpp new file mode 100644 index 0000000000..8b55888ae8 --- /dev/null +++ b/src/scheduler.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "scheduler.h" + +#include <assert.h> +#include <boost/bind.hpp> +#include <utility> + +CScheduler::CScheduler() : nThreadsServicingQueue(0) +{ +} + +CScheduler::~CScheduler() +{ + assert(nThreadsServicingQueue == 0); +} + + +#if BOOST_VERSION < 105000 +static boost::system_time toPosixTime(const boost::chrono::system_clock::time_point& t) +{ + return boost::posix_time::from_time_t(boost::chrono::system_clock::to_time_t(t)); +} +#endif + +void CScheduler::serviceQueue() +{ + boost::unique_lock<boost::mutex> lock(newTaskMutex); + ++nThreadsServicingQueue; + + // newTaskMutex is locked throughout this loop EXCEPT + // when the thread is waiting or when the user's function + // is called. + while (1) { + try { + while (taskQueue.empty()) { + // Wait until there is something to do. + newTaskScheduled.wait(lock); + } +// Wait until either there is a new task, or until +// the time of the first item on the queue: + +// wait_until needs boost 1.50 or later; older versions have timed_wait: +#if BOOST_VERSION < 105000 + while (!taskQueue.empty() && newTaskScheduled.timed_wait(lock, toPosixTime(taskQueue.begin()->first))) { + // Keep waiting until timeout + } +#else + while (!taskQueue.empty() && newTaskScheduled.wait_until(lock, taskQueue.begin()->first) != boost::cv_status::timeout) { + // Keep waiting until timeout + } +#endif + // If there are multiple threads, the queue can empty while we're waiting (another + // thread may service the task we were waiting on). + if (taskQueue.empty()) + continue; + + Function f = taskQueue.begin()->second; + taskQueue.erase(taskQueue.begin()); + + // Unlock before calling f, so it can reschedule itself or another task + // without deadlocking: + lock.unlock(); + f(); + lock.lock(); + } catch (...) { + --nThreadsServicingQueue; + throw; + } + } +} + +void CScheduler::schedule(CScheduler::Function f, boost::chrono::system_clock::time_point t) +{ + { + boost::unique_lock<boost::mutex> lock(newTaskMutex); + taskQueue.insert(std::make_pair(t, f)); + } + newTaskScheduled.notify_one(); +} + +void CScheduler::scheduleFromNow(CScheduler::Function f, int64_t deltaSeconds) +{ + schedule(f, boost::chrono::system_clock::now() + boost::chrono::seconds(deltaSeconds)); +} + +static void Repeat(CScheduler* s, CScheduler::Function f, int64_t deltaSeconds) +{ + f(); + s->scheduleFromNow(boost::bind(&Repeat, s, f, deltaSeconds), deltaSeconds); +} + +void CScheduler::scheduleEvery(CScheduler::Function f, int64_t deltaSeconds) +{ + scheduleFromNow(boost::bind(&Repeat, this, f, deltaSeconds), deltaSeconds); +} diff --git a/src/scheduler.h b/src/scheduler.h new file mode 100644 index 0000000000..bb383ab9f9 --- /dev/null +++ b/src/scheduler.h @@ -0,0 +1,70 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SCHEDULER_H +#define BITCOIN_SCHEDULER_H + +// +// NOTE: +// boost::thread / boost::function / boost::chrono should be ported to +// std::thread / std::function / std::chrono when we support C++11. +// +#include <boost/function.hpp> +#include <boost/chrono/chrono.hpp> +#include <boost/thread.hpp> +#include <map> + +// +// Simple class for background tasks that should be run +// periodically or once "after a while" +// +// Usage: +// +// CScheduler* s = new CScheduler(); +// s->scheduleFromNow(doSomething, 11); // Assuming a: void doSomething() { } +// s->scheduleFromNow(boost::bind(Class::func, this, argument), 3); +// boost::thread* t = new boost::thread(boost::bind(CScheduler::serviceQueue, s)); +// +// ... then at program shutdown, clean up the thread running serviceQueue: +// t->interrupt(); +// t->join(); +// delete t; +// delete s; // Must be done after thread is interrupted/joined. +// + +class CScheduler +{ +public: + CScheduler(); + ~CScheduler(); + + typedef boost::function<void(void)> Function; + + // Call func at/after time t + void schedule(Function f, boost::chrono::system_clock::time_point t); + + // Convenience method: call f once deltaSeconds from now + void scheduleFromNow(Function f, int64_t deltaSeconds); + + // Another convenience method: call f approximately + // every deltaSeconds forever, starting deltaSeconds from now. + // To be more precise: every time f is finished, it + // is rescheduled to run deltaSeconds later. If you + // need more accurate scheduling, don't use this method. + void scheduleEvery(Function f, int64_t deltaSeconds); + + // To keep things as simple as possible, there is no unschedule. + + // Services the queue 'forever'. Should be run in a thread, + // and interrupted using boost::interrupt_thread + void serviceQueue(); + +private: + std::multimap<boost::chrono::system_clock::time_point, Function> taskQueue; + boost::condition_variable newTaskScheduled; + boost::mutex newTaskMutex; + int nThreadsServicingQueue; +}; + +#endif diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp new file mode 100644 index 0000000000..cb64ee7c69 --- /dev/null +++ b/src/test/policyestimator_tests.cpp @@ -0,0 +1,186 @@ +// Copyright (c) 2011-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "policy/fees.h" +#include "txmempool.h" +#include "uint256.h" +#include "util.h" + +#include "test/test_bitcoin.h" + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) +{ + CTxMemPool mpool(CFeeRate(1000)); + CAmount basefee(2000); + double basepri = 1e6; + CAmount deltaFee(100); + double deltaPri=5e5; + std::vector<CAmount> feeV[2]; + std::vector<double> priV[2]; + + // Populate vectors of increasing fees or priorities + for (int j = 0; j < 10; j++) { + //V[0] is for fee transactions + feeV[0].push_back(basefee * (j+1)); + priV[0].push_back(0); + //V[1] is for priority transactions + feeV[1].push_back(CAmount(0)); + priV[1].push_back(basepri * pow(10, j+1)); + } + + // Store the hashes of transactions that have been + // added to the mempool by their associate fee/pri + // txHashes[j] is populated with transactions either of + // fee = basefee * (j+1) OR pri = 10^6 * 10^(j+1) + std::vector<uint256> txHashes[10]; + + // Create a transaction template + CScript garbage; + for (unsigned int i = 0; i < 128; i++) + garbage.push_back('X'); + CMutableTransaction tx; + std::list<CTransaction> dummyConflicted; + tx.vin.resize(1); + tx.vin[0].scriptSig = garbage; + tx.vout.resize(1); + tx.vout[0].nValue=0LL; + CFeeRate baseRate(basefee, ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); + + // Create a fake block + std::vector<CTransaction> block; + int blocknum = 0; + + // Loop through 200 blocks + // At a decay .998 and 4 fee transactions per block + // This makes the tx count about 1.33 per bucket, above the 1 threshold + while (blocknum < 200) { + for (int j = 0; j < 10; j++) { // For each fee/pri multiple + for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + tx.vin[0].prevout.n = 10000*blocknum+100*j+k; // make transaction unique + uint256 hash = tx.GetHash(); + mpool.addUnchecked(hash, CTxMemPoolEntry(tx, feeV[k/4][j], GetTime(), priV[k/4][j], blocknum, mpool.HasNoInputsOf(tx))); + txHashes[j].push_back(hash); + } + } + //Create blocks where higher fee/pri txs are included more often + for (int h = 0; h <= blocknum%10; h++) { + // 10/10 blocks add highest fee/pri transactions + // 9/10 blocks add 2nd highest and so on until ... + // 1/10 blocks add lowest fee/pri transactions + while (txHashes[9-h].size()) { + CTransaction btx; + if (mpool.lookup(txHashes[9-h].back(), btx)) + block.push_back(btx); + txHashes[9-h].pop_back(); + } + } + mpool.removeForBlock(block, ++blocknum, dummyConflicted); + block.clear(); + if (blocknum == 30) { + // At this point we should need to combine 5 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 8*baserate + BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(mpool.estimateFee(2).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(mpool.estimateFee(2).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); + } + } + + std::vector<CAmount> origFeeEst; + std::vector<double> origPriEst; + // Highest feerate is 10*baseRate and gets in all blocks, + // second highest feerate is 9*baseRate and gets in 9/10 blocks = 90%, + // third highest feerate is 8*base rate, and gets in 8/10 blocks = 80%, + // so estimateFee(1) should return 9*baseRate. + // Third highest feerate has 90% chance of being included by 2 blocks, + // so estimateFee(2) should return 8*baseRate etc... + for (int i = 1; i < 10;i++) { + origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); + origPriEst.push_back(mpool.estimatePriority(i)); + if (i > 1) { // Fee estimates should be monotonically decreasing + BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); + BOOST_CHECK(origPriEst[i-1] <= origPriEst[i-2]); + } + BOOST_CHECK(origFeeEst[i-1] < (10-i)*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(origFeeEst[i-1] > (10-i)*baseRate.GetFeePerK() - deltaFee); + BOOST_CHECK(origPriEst[i-1] < pow(10,10-i) * basepri + deltaPri); + BOOST_CHECK(origPriEst[i-1] > pow(10,10-i) * basepri - deltaPri); + } + + // Mine 50 more blocks with no transactions happening, estimates shouldn't change + // We haven't decayed the moving average enough so we still have enough data points in every bucket + while (blocknum < 250) + mpool.removeForBlock(block, ++blocknum, dummyConflicted); + + for (int i = 1; i < 10;i++) { + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i-1] + deltaPri); + BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); + } + + + // Mine 15 more blocks with lots of transactions happening and not getting mined + // Estimates should go up + while (blocknum < 265) { + for (int j = 0; j < 10; j++) { // For each fee/pri multiple + for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + tx.vin[0].prevout.n = 10000*blocknum+100*j+k; + uint256 hash = tx.GetHash(); + mpool.addUnchecked(hash, CTxMemPoolEntry(tx, feeV[k/4][j], GetTime(), priV[k/4][j], blocknum, mpool.HasNoInputsOf(tx))); + txHashes[j].push_back(hash); + } + } + mpool.removeForBlock(block, ++blocknum, dummyConflicted); + } + + for (int i = 1; i < 10;i++) { + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); + } + + // Mine all those transactions + // Estimates should still not be below original + for (int j = 0; j < 10; j++) { + while(txHashes[j].size()) { + CTransaction btx; + if (mpool.lookup(txHashes[j].back(), btx)) + block.push_back(btx); + txHashes[j].pop_back(); + } + } + mpool.removeForBlock(block, 265, dummyConflicted); + block.clear(); + for (int i = 1; i < 10;i++) { + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); + } + + // Mine 100 more blocks where everything is mined every block + // Estimates should be below original estimates (not possible for last estimate) + while (blocknum < 365) { + for (int j = 0; j < 10; j++) { // For each fee/pri multiple + for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + tx.vin[0].prevout.n = 10000*blocknum+100*j+k; + uint256 hash = tx.GetHash(); + mpool.addUnchecked(hash, CTxMemPoolEntry(tx, feeV[k/4][j], GetTime(), priV[k/4][j], blocknum, mpool.HasNoInputsOf(tx))); + CTransaction btx; + if (mpool.lookup(hash, btx)) + block.push_back(btx); + } + } + mpool.removeForBlock(block, ++blocknum, dummyConflicted); + block.clear(); + } + for (int i = 1; i < 9; i++) { + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); + BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i-1] - deltaPri); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp new file mode 100644 index 0000000000..a26d0afaed --- /dev/null +++ b/src/test/scheduler_tests.cpp @@ -0,0 +1,110 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "random.h" +#include "scheduler.h" + +#include "test/test_bitcoin.h" + +#include <boost/bind.hpp> +#include <boost/random/mersenne_twister.hpp> +#include <boost/random/uniform_int_distribution.hpp> +#include <boost/thread.hpp> +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(scheduler_tests) + +static void microTask(CScheduler& s, boost::mutex& mutex, int& counter, int delta, boost::chrono::system_clock::time_point rescheduleTime) +{ + { + boost::unique_lock<boost::mutex> lock(mutex); + counter += delta; + } + boost::chrono::system_clock::time_point noTime = boost::chrono::system_clock::time_point::min(); + if (rescheduleTime != noTime) { + CScheduler::Function f = boost::bind(µTask, boost::ref(s), boost::ref(mutex), boost::ref(counter), -delta + 1, noTime); + s.schedule(f, rescheduleTime); + } +} + +static void MicroSleep(uint64_t n) +{ +#if defined(HAVE_WORKING_BOOST_SLEEP_FOR) + boost::this_thread::sleep_for(boost::chrono::microseconds(n)); +#elif defined(HAVE_WORKING_BOOST_SLEEP) + boost::this_thread::sleep(boost::posix_time::microseconds(n)); +#else + //should never get here + #error missing boost sleep implementation +#endif +} + +BOOST_AUTO_TEST_CASE(manythreads) +{ + // Stress test: hundreds of microsecond-scheduled tasks, + // serviced by 10 threads. + // + // So... ten shared counters, which if all the tasks execute + // properly will sum to the number of tasks done. + // Each task adds or subtracts from one of the counters a + // random amount, and then schedules another task 0-1000 + // microseconds in the future to subtract or add from + // the counter -random_amount+1, so in the end the shared + // counters should sum to the number of initial tasks performed. + CScheduler microTasks; + + boost::thread_group microThreads; + for (int i = 0; i < 5; i++) + microThreads.create_thread(boost::bind(&CScheduler::serviceQueue, µTasks)); + + boost::mutex counterMutex[10]; + int counter[10] = { 0 }; + boost::random::mt19937 rng(insecure_rand()); + boost::random::uniform_int_distribution<> zeroToNine(0, 9); + boost::random::uniform_int_distribution<> randomMsec(-11, 1000); + boost::random::uniform_int_distribution<> randomDelta(-1000, 1000); + + boost::chrono::system_clock::time_point start = boost::chrono::system_clock::now(); + boost::chrono::system_clock::time_point now = start; + + for (int i = 0; i < 100; i++) { + boost::chrono::system_clock::time_point t = now + boost::chrono::microseconds(randomMsec(rng)); + boost::chrono::system_clock::time_point tReschedule = now + boost::chrono::microseconds(500 + randomMsec(rng)); + int whichCounter = zeroToNine(rng); + CScheduler::Function f = boost::bind(µTask, boost::ref(microTasks), + boost::ref(counterMutex[whichCounter]), boost::ref(counter[whichCounter]), + randomDelta(rng), tReschedule); + microTasks.schedule(f, t); + } + + MicroSleep(600); + now = boost::chrono::system_clock::now(); + // More threads and more tasks: + for (int i = 0; i < 5; i++) + microThreads.create_thread(boost::bind(&CScheduler::serviceQueue, µTasks)); + for (int i = 0; i < 100; i++) { + boost::chrono::system_clock::time_point t = now + boost::chrono::microseconds(randomMsec(rng)); + boost::chrono::system_clock::time_point tReschedule = now + boost::chrono::microseconds(500 + randomMsec(rng)); + int whichCounter = zeroToNine(rng); + CScheduler::Function f = boost::bind(µTask, boost::ref(microTasks), + boost::ref(counterMutex[whichCounter]), boost::ref(counter[whichCounter]), + randomDelta(rng), tReschedule); + microTasks.schedule(f, t); + } + + // All 2,000 tasks should be finished within 2 milliseconds. Sleep a bit longer. + MicroSleep(2100); + + microThreads.interrupt_all(); + microThreads.join_all(); + + int counterSum = 0; + for (int i = 0; i < 10; i++) { + BOOST_CHECK(counter[i] != 0); + counterSum += counter[i]; + } + BOOST_CHECK_EQUAL(counterSum, 200); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.cpp b/src/txmempool.cpp index d05e3c6eec..071fa9d52c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -8,28 +8,27 @@ #include "clientversion.h" #include "consensus/consensus.h" #include "main.h" +#include "policy/fees.h" #include "streams.h" #include "util.h" #include "utilmoneystr.h" #include "version.h" -#include <boost/circular_buffer.hpp> - using namespace std; CTxMemPoolEntry::CTxMemPoolEntry(): - nFee(0), nTxSize(0), nModSize(0), nTime(0), dPriority(0.0) + nFee(0), nTxSize(0), nModSize(0), nTime(0), dPriority(0.0), hadNoDependencies(false) { nHeight = MEMPOOL_HEIGHT; } CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _dPriority, - unsigned int _nHeight): - tx(_tx), nFee(_nFee), nTime(_nTime), dPriority(_dPriority), nHeight(_nHeight) + unsigned int _nHeight, bool poolHasNoInputsOf): + tx(_tx), nFee(_nFee), nTime(_nTime), dPriority(_dPriority), nHeight(_nHeight), + hadNoDependencies(poolHasNoInputsOf) { nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); - nModSize = tx.CalculateModifiedSize(nTxSize); } @@ -47,346 +46,15 @@ CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const return dResult; } -/** - * Keep track of fee/priority for transactions confirmed within N blocks - */ -class CBlockAverage -{ -private: - boost::circular_buffer<CFeeRate> feeSamples; - boost::circular_buffer<double> prioritySamples; - - template<typename T> std::vector<T> buf2vec(boost::circular_buffer<T> buf) const - { - std::vector<T> vec(buf.begin(), buf.end()); - return vec; - } - -public: - CBlockAverage() : feeSamples(100), prioritySamples(100) { } - - void RecordFee(const CFeeRate& feeRate) { - feeSamples.push_back(feeRate); - } - - void RecordPriority(double priority) { - prioritySamples.push_back(priority); - } - - size_t FeeSamples() const { return feeSamples.size(); } - size_t GetFeeSamples(std::vector<CFeeRate>& insertInto) const - { - BOOST_FOREACH(const CFeeRate& f, feeSamples) - insertInto.push_back(f); - return feeSamples.size(); - } - size_t PrioritySamples() const { return prioritySamples.size(); } - size_t GetPrioritySamples(std::vector<double>& insertInto) const - { - BOOST_FOREACH(double d, prioritySamples) - insertInto.push_back(d); - return prioritySamples.size(); - } - - /** - * Used as belt-and-suspenders check when reading to detect - * file corruption - */ - static bool AreSane(const CFeeRate fee, const CFeeRate& minRelayFee) - { - if (fee < CFeeRate(0)) - return false; - if (fee.GetFeePerK() > minRelayFee.GetFeePerK() * 10000) - return false; - return true; - } - static bool AreSane(const std::vector<CFeeRate>& vecFee, const CFeeRate& minRelayFee) - { - BOOST_FOREACH(CFeeRate fee, vecFee) - { - if (!AreSane(fee, minRelayFee)) - return false; - } - return true; - } - static bool AreSane(const double priority) - { - return priority >= 0; - } - static bool AreSane(const std::vector<double> vecPriority) - { - BOOST_FOREACH(double priority, vecPriority) - { - if (!AreSane(priority)) - return false; - } - return true; - } - - void Write(CAutoFile& fileout) const - { - std::vector<CFeeRate> vecFee = buf2vec(feeSamples); - fileout << vecFee; - std::vector<double> vecPriority = buf2vec(prioritySamples); - fileout << vecPriority; - } - - void Read(CAutoFile& filein, const CFeeRate& minRelayFee) { - std::vector<CFeeRate> vecFee; - filein >> vecFee; - if (AreSane(vecFee, minRelayFee)) - feeSamples.insert(feeSamples.end(), vecFee.begin(), vecFee.end()); - else - throw runtime_error("Corrupt fee value in estimates file."); - std::vector<double> vecPriority; - filein >> vecPriority; - if (AreSane(vecPriority)) - prioritySamples.insert(prioritySamples.end(), vecPriority.begin(), vecPriority.end()); - else - throw runtime_error("Corrupt priority value in estimates file."); - if (feeSamples.size() + prioritySamples.size() > 0) - LogPrint("estimatefee", "Read %d fee samples and %d priority samples\n", - feeSamples.size(), prioritySamples.size()); - } -}; - -class CMinerPolicyEstimator -{ -private: - /** - * Records observed averages transactions that confirmed within one block, two blocks, - * three blocks etc. - */ - std::vector<CBlockAverage> history; - std::vector<CFeeRate> sortedFeeSamples; - std::vector<double> sortedPrioritySamples; - - int nBestSeenHeight; - - /** - * nBlocksAgo is 0 based, i.e. transactions that confirmed in the highest seen block are - * nBlocksAgo == 0, transactions in the block before that are nBlocksAgo == 1 etc. - */ - void seenTxConfirm(const CFeeRate& feeRate, const CFeeRate& minRelayFee, double dPriority, int nBlocksAgo) - { - // Last entry records "everything else". - int nBlocksTruncated = min(nBlocksAgo, (int) history.size() - 1); - assert(nBlocksTruncated >= 0); - - // We need to guess why the transaction was included in a block-- either - // because it is high-priority or because it has sufficient fees. - bool sufficientFee = (feeRate > minRelayFee); - bool sufficientPriority = AllowFree(dPriority); - const char* assignedTo = "unassigned"; - if (sufficientFee && !sufficientPriority && CBlockAverage::AreSane(feeRate, minRelayFee)) - { - history[nBlocksTruncated].RecordFee(feeRate); - assignedTo = "fee"; - } - else if (sufficientPriority && !sufficientFee && CBlockAverage::AreSane(dPriority)) - { - history[nBlocksTruncated].RecordPriority(dPriority); - assignedTo = "priority"; - } - else - { - // Neither or both fee and priority sufficient to get confirmed: - // don't know why they got confirmed. - } - LogPrint("estimatefee", "Seen TX confirm: %s: %s fee/%g priority, took %d blocks\n", - assignedTo, feeRate.ToString(), dPriority, nBlocksAgo); - } - -public: - CMinerPolicyEstimator(int nEntries) : nBestSeenHeight(0) - { - history.resize(nEntries); - } - - void seenBlock(const std::vector<CTxMemPoolEntry>& entries, int nBlockHeight, const CFeeRate minRelayFee) - { - if (nBlockHeight <= nBestSeenHeight) - { - // Ignore side chains and re-orgs; assuming they are random - // they don't affect the estimate. - // And if an attacker can re-org the chain at will, then - // you've got much bigger problems than "attacker can influence - // transaction fees." - return; - } - nBestSeenHeight = nBlockHeight; - - // Fill up the history buckets based on how long transactions took - // to confirm. - std::vector<std::vector<const CTxMemPoolEntry*> > entriesByConfirmations; - entriesByConfirmations.resize(history.size()); - BOOST_FOREACH(const CTxMemPoolEntry& entry, entries) - { - // How many blocks did it take for miners to include this transaction? - int delta = nBlockHeight - entry.GetHeight(); - if (delta <= 0) - { - // Re-org made us lose height, this should only happen if we happen - // to re-org on a difficulty transition point: very rare! - continue; - } - if ((delta-1) >= (int)history.size()) - delta = history.size(); // Last bucket is catch-all - entriesByConfirmations.at(delta-1).push_back(&entry); - } - for (size_t i = 0; i < entriesByConfirmations.size(); i++) - { - std::vector<const CTxMemPoolEntry*> &e = entriesByConfirmations.at(i); - // Insert at most 10 random entries per bucket, otherwise a single block - // can dominate an estimate: - if (e.size() > 10) { - std::random_shuffle(e.begin(), e.end()); - e.resize(10); - } - BOOST_FOREACH(const CTxMemPoolEntry* entry, e) - { - // Fees are stored and reported as BTC-per-kb: - CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); - double dPriority = entry->GetPriority(entry->GetHeight()); // Want priority when it went IN - seenTxConfirm(feeRate, minRelayFee, dPriority, i); - } - } - - // After new samples are added, we have to clear the sorted lists, - // so they'll be resorted the next time someone asks for an estimate - sortedFeeSamples.clear(); - sortedPrioritySamples.clear(); - - for (size_t i = 0; i < history.size(); i++) { - if (history[i].FeeSamples() + history[i].PrioritySamples() > 0) - LogPrint("estimatefee", "estimates: for confirming within %d blocks based on %d/%d samples, fee=%s, prio=%g\n", - i, - history[i].FeeSamples(), history[i].PrioritySamples(), - estimateFee(i+1).ToString(), estimatePriority(i+1)); - } - } - - /** - * Can return CFeeRate(0) if we don't have any data for that many blocks back. nBlocksToConfirm is 1 based. - */ - CFeeRate estimateFee(int nBlocksToConfirm) - { - nBlocksToConfirm--; - - if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size()) - return CFeeRate(0); - - if (sortedFeeSamples.size() == 0) - { - for (size_t i = 0; i < history.size(); i++) - history.at(i).GetFeeSamples(sortedFeeSamples); - std::sort(sortedFeeSamples.begin(), sortedFeeSamples.end(), - std::greater<CFeeRate>()); - } - if (sortedFeeSamples.size() < 11) - { - // Eleven is Gavin's Favorite Number - // ... but we also take a maximum of 10 samples per block so eleven means - // we're getting samples from at least two different blocks - return CFeeRate(0); - } - - int nBucketSize = history.at(nBlocksToConfirm).FeeSamples(); - - // Estimates should not increase as number of confirmations goes up, - // but the estimates are noisy because confirmations happen discretely - // in blocks. To smooth out the estimates, use all samples in the history - // and use the nth highest where n is (number of samples in previous bucket + - // half the samples in nBlocksToConfirm bucket): - size_t nPrevSize = 0; - for (int i = 0; i < nBlocksToConfirm; i++) - nPrevSize += history.at(i).FeeSamples(); - size_t index = min(nPrevSize + nBucketSize/2, sortedFeeSamples.size()-1); - return sortedFeeSamples[index]; - } - double estimatePriority(int nBlocksToConfirm) - { - nBlocksToConfirm--; - - if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size()) - return -1; - - if (sortedPrioritySamples.size() == 0) - { - for (size_t i = 0; i < history.size(); i++) - history.at(i).GetPrioritySamples(sortedPrioritySamples); - std::sort(sortedPrioritySamples.begin(), sortedPrioritySamples.end(), - std::greater<double>()); - } - if (sortedPrioritySamples.size() < 11) - return -1.0; - - int nBucketSize = history.at(nBlocksToConfirm).PrioritySamples(); - - // Estimates should not increase as number of confirmations needed goes up, - // but the estimates are noisy because confirmations happen discretely - // in blocks. To smooth out the estimates, use all samples in the history - // and use the nth highest where n is (number of samples in previous buckets + - // half the samples in nBlocksToConfirm bucket). - size_t nPrevSize = 0; - for (int i = 0; i < nBlocksToConfirm; i++) - nPrevSize += history.at(i).PrioritySamples(); - size_t index = min(nPrevSize + nBucketSize/2, sortedPrioritySamples.size()-1); - return sortedPrioritySamples[index]; - } - - void Write(CAutoFile& fileout) const - { - fileout << nBestSeenHeight; - fileout << (uint32_t)history.size(); - BOOST_FOREACH(const CBlockAverage& entry, history) - { - entry.Write(fileout); - } - } - - void Read(CAutoFile& filein, const CFeeRate& minRelayFee) - { - int nFileBestSeenHeight; - filein >> nFileBestSeenHeight; - uint32_t numEntries; - filein >> numEntries; - if (numEntries <= 0 || numEntries > 10000) - throw runtime_error("Corrupt estimates file. Must have between 1 and 10k entries."); - - std::vector<CBlockAverage> fileHistory; - - for (size_t i = 0; i < numEntries; i++) - { - CBlockAverage entry; - entry.Read(filein, minRelayFee); - fileHistory.push_back(entry); - } - - // Now that we've processed the entire fee estimate data file and not - // thrown any errors, we can copy it to our history - nBestSeenHeight = nFileBestSeenHeight; - history = fileHistory; - assert(history.size() > 0); - } -}; - - CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) : - nTransactionsUpdated(0), - minRelayFee(_minRelayFee) + nTransactionsUpdated(0) { // Sanity checks off by default for performance, because otherwise // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool fSanityCheck = false; - // 25 blocks is a compromise between using a lot of disk/memory and - // trying to give accurate estimates to people who might be willing - // to wait a day or two to save a fraction of a penny in fees. - // Confirmation times for very-low-fee transactions that take more - // than an hour or three to confirm are highly variable. - minerPolicyEstimator = new CMinerPolicyEstimator(25); + minerPolicyEstimator = new CBlockPolicyEstimator(_minRelayFee); } CTxMemPool::~CTxMemPool() @@ -420,20 +88,20 @@ void CTxMemPool::AddTransactionsUpdated(unsigned int n) } -bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry) +bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate) { // Add to memory pool without checking anything. // Used by main.cpp AcceptToMemoryPool(), which DOES do // all the appropriate checks. LOCK(cs); - { - mapTx[hash] = entry; - const CTransaction& tx = mapTx[hash].GetTx(); - for (unsigned int i = 0; i < tx.vin.size(); i++) - mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i); - nTransactionsUpdated++; - totalTxSize += entry.GetTxSize(); - } + mapTx[hash] = entry; + const CTransaction& tx = mapTx[hash].GetTx(); + for (unsigned int i = 0; i < tx.vin.size(); i++) + mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i); + nTransactionsUpdated++; + totalTxSize += entry.GetTxSize(); + minerPolicyEstimator->processTransaction(entry, fCurrentEstimate); + return true; } @@ -479,6 +147,7 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& rem totalTxSize -= mapTx[hash].GetTxSize(); mapTx.erase(hash); nTransactionsUpdated++; + minerPolicyEstimator->removeTx(hash); } } } @@ -529,7 +198,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction> * Called when a block is connected. Removes from mempool and updates the miner fee estimator. */ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts) + std::list<CTransaction>& conflicts, bool fCurrentEstimate) { LOCK(cs); std::vector<CTxMemPoolEntry> entries; @@ -539,7 +208,6 @@ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned i if (mapTx.count(hash)) entries.push_back(mapTx[hash]); } - minerPolicyEstimator->seenBlock(entries, nBlockHeight, minRelayFee); BOOST_FOREACH(const CTransaction& tx, vtx) { std::list<CTransaction> dummy; @@ -547,9 +215,10 @@ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned i removeConflicts(tx, conflicts); ClearPrioritisation(tx.GetHash()); } + // After the txs in the new block have been removed from the mempool, update policy estimates + minerPolicyEstimator->processBlock(nBlockHeight, entries, fCurrentEstimate); } - void CTxMemPool::clear() { LOCK(cs); @@ -666,7 +335,7 @@ CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const { try { LOCK(cs); - fileout << 99900; // version required to read: 0.9.99 or later + fileout << 109900; // version required to read: 0.10.99 or later fileout << CLIENT_VERSION; // version that wrote the file minerPolicyEstimator->Write(fileout); } @@ -687,7 +356,7 @@ CTxMemPool::ReadFeeEstimates(CAutoFile& filein) return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); LOCK(cs); - minerPolicyEstimator->Read(filein, minRelayFee); + minerPolicyEstimator->Read(filein); } catch (const std::exception&) { LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)"); @@ -724,6 +393,13 @@ void CTxMemPool::ClearPrioritisation(const uint256 hash) mapDeltas.erase(hash); } +bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const +{ + for (unsigned int i = 0; i < tx.vin.size(); i++) + if (exists(tx.vin[i].prevout.hash)) + return false; + return true; +} CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } diff --git a/src/txmempool.h b/src/txmempool.h index 0732af67e6..7271a5f603 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -43,10 +43,11 @@ private: int64_t nTime; //! Local time when entering the mempool double dPriority; //! Priority when entering the mempool unsigned int nHeight; //! Chain height when entering the mempool + bool hadNoDependencies; //! Not dependent on any other txs when it entered the mempool public: CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, - int64_t _nTime, double _dPriority, unsigned int _nHeight); + int64_t _nTime, double _dPriority, unsigned int _nHeight, bool poolHasNoInputsOf = false); CTxMemPoolEntry(); CTxMemPoolEntry(const CTxMemPoolEntry& other); @@ -56,9 +57,10 @@ public: size_t GetTxSize() const { return nTxSize; } int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return nHeight; } + bool WasClearAtEntry() const { return hadNoDependencies; } }; -class CMinerPolicyEstimator; +class CBlockPolicyEstimator; /** An inpoint - a combination of a transaction and an index n into its vin */ class CInPoint @@ -88,9 +90,8 @@ class CTxMemPool private: bool fSanityCheck; //! Normally false, true if -checkmempool or -regtest unsigned int nTransactionsUpdated; - CMinerPolicyEstimator* minerPolicyEstimator; + CBlockPolicyEstimator* minerPolicyEstimator; - CFeeRate minRelayFee; //! Passed to constructor to avoid dependency on main uint64_t totalTxSize; //! sum of all mempool tx' byte sizes public: @@ -111,17 +112,22 @@ public: void check(const CCoinsViewCache *pcoins) const; void setSanityCheck(bool _fSanityCheck) { fSanityCheck = _fSanityCheck; } - bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry); + bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); void remove(const CTransaction &tx, std::list<CTransaction>& removed, bool fRecursive = false); void removeCoinbaseSpends(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight); void removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed); void removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts); + std::list<CTransaction>& conflicts, bool fCurrentEstimate = true); void clear(); void queryHashes(std::vector<uint256>& vtxid); void pruneSpent(const uint256& hash, CCoins &coins); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); + /** + * Check that none of this transactions inputs are in the mempool, and thus + * the tx is not dependent on other mempool transactions to be included in a block. + */ + bool HasNoInputsOf(const CTransaction& tx) const; /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const CAmount& nFeeDelta); @@ -139,7 +145,7 @@ public: return totalTxSize; } - bool exists(uint256 hash) + bool exists(uint256 hash) const { LOCK(cs); return (mapTx.count(hash) != 0); diff --git a/src/util.h b/src/util.h index 483d9d7858..4cc0faf4d7 100644 --- a/src/util.h +++ b/src/util.h @@ -203,43 +203,6 @@ void SetThreadPriority(int nPriority); void RenameThread(const char* name); /** - * Standard wrapper for do-something-forever thread functions. - * "Forever" really means until the thread is interrupted. - * Use it like: - * new boost::thread(boost::bind(&LoopForever<void (*)()>, "dumpaddr", &DumpAddresses, 900000)); - * or maybe: - * boost::function<void()> f = boost::bind(&FunctionWithArg, argument); - * threadGroup.create_thread(boost::bind(&LoopForever<boost::function<void()> >, "nothing", f, milliseconds)); - */ -template <typename Callable> void LoopForever(const char* name, Callable func, int64_t msecs) -{ - std::string s = strprintf("bitcoin-%s", name); - RenameThread(s.c_str()); - LogPrintf("%s thread start\n", name); - try - { - while (1) - { - MilliSleep(msecs); - func(); - } - } - catch (const boost::thread_interrupted&) - { - LogPrintf("%s thread stop\n", name); - throw; - } - catch (const std::exception& e) { - PrintExceptionContinue(&e, name); - throw; - } - catch (...) { - PrintExceptionContinue(NULL, name); - throw; - } -} - -/** * .. and a wrapper that just calls func once */ template <typename Callable> void TraceThread(const char* name, Callable func) |
