diff options
92 files changed, 3891 insertions, 1243 deletions
diff --git a/configure.ac b/configure.ac index 7c0a9f4a94..c5d08a0285 100644 --- a/configure.ac +++ b/configure.ac @@ -590,17 +590,15 @@ fi if test x$use_boost = xyes; then -BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB" +BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB" dnl Boost >= 1.50 uses sleep_for rather than the now-deprecated sleep, however dnl it was broken from 1.50 to 1.52 when backed by nanosleep. Use sleep_for if dnl a working version is available, else fall back to sleep. sleep was removed dnl after 1.56. dnl If neither is available, abort. -dnl If sleep_for is used, boost_chrono becomes a requirement. -if test x$ax_cv_boost_chrono = xyes; then TEMP_LIBS="$LIBS" -LIBS="$BOOST_LIBS $BOOST_CHRONO_LIB $LIBS" +LIBS="$BOOST_LIBS $LIBS" TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -613,12 +611,11 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ choke me #endif ]])], - [boost_sleep=yes; BOOST_LIBS="$BOOST_LIBS $BOOST_CHRONO_LIB"; + [boost_sleep=yes; AC_DEFINE(HAVE_WORKING_BOOST_SLEEP_FOR, 1, [Define this symbol if boost sleep_for works])], [boost_sleep=no]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" -fi if test x$boost_sleep != xyes; then TEMP_LIBS="$LIBS" diff --git a/depends/Makefile b/depends/Makefile index 05ef33f2ee..ef5a20e6c3 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -91,12 +91,12 @@ include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) -$(host_prefix)/.stamp_$(final_build_id): | $(native_packages) $(packages) +$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) - $(AT)echo copying packages: $| + $(AT)echo copying packages: $^ $(AT)echo to: $(@D) - $(AT)cd $(@D); $(foreach package,$|, tar xf $($(package)_cached); ) + $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) @@ -121,8 +121,35 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ $< > $@ $(AT)touch $@ -install: $(host_prefix)/share/config.site -download-one: $(all_sources) + +define check_or_remove_cached + mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \ + $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \ + ( rm -f $($(package)_cached_checksum); \ + if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi ) +endef + +define check_or_remove_sources + mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ + $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ + ( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \ + rm -f $($(package)_all_sources) $($(1)_fetched)) +endef + +check-packages: + @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));) +check-sources: + @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) + +$(host_prefix)/share/config.site: check-packages + +check-packages: check-sources + +install: check-packages $(host_prefix)/share/config.site + + +download-one: check-sources $(all_sources) + download-osx: @$(MAKE) -s HOST=x86_64-apple-darwin11 download-one download-linux: @@ -130,4 +157,5 @@ download-linux: download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win -.PHONY: install cached download-one download-osx download-linux download-win download + +.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources diff --git a/depends/README.md b/depends/README.md new file mode 100644 index 0000000000..2dc0b9e47e --- /dev/null +++ b/depends/README.md @@ -0,0 +1,56 @@ +### Usage + +To build dependencies for the current arch+OS: + + make + +To build for another arch/OS: + + make HOST=host-platform-triplet + +For example: + + make HOST=x86_64-w64-mingw32 -j4 + +A prefix will be generated that's suitable for plugging into Bitcoin's +configure. In the above example, a dir named i686-w64-mingw32 will be +created. To use it for Bitcoin: + + ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32 + +Common `host-platform-triplets` for cross compilation are: + +- `i686-w64-mingw32` for Win32 +- `x86_64-w64-mingw32` for Win64 +- `x86_64-apple-darwin11` for MacOSX +- `arm-linux-gnueabihf` for Linux ARM + +No other options are needed, the paths are automatically configured. + +Dependency Options: +The following can be set when running make: make FOO=bar + + SOURCES_PATH: downloaded sources will be placed here + BASE_CACHE: built packages will be placed here + SDK_PATH: Path where sdk's can be found (used by OSX) + FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up + NO_QT: Don't download/build/cache qt and its dependencies + NO_WALLET: Don't download/build/cache libs needed to enable the wallet + NO_UPNP: Don't download/build/cache packages needed for enabling upnp + DEBUG: disable some optimizations and enable more runtime checking + +If some packages are not built, for example `make NO_WALLET=1`, the appropriate +options will be passed to bitcoin's configure. In this case, `--disable-wallet`. + +Additional targets: + + download: run 'make download' to fetch all sources without building them + download-osx: run 'make download-osx' to fetch all sources needed for osx builds + download-win: run 'make download-win' to fetch all sources needed for win builds + download-linux: run 'make download-linux' to fetch all sources needed for linux builds + +### Other documentation + +- [description.md](description.md): General description of the depends system +- [packages.md](packages.md): Steps for adding packages + diff --git a/depends/README.usage b/depends/README.usage deleted file mode 100644 index 24e1231d82..0000000000 --- a/depends/README.usage +++ /dev/null @@ -1,34 +0,0 @@ -To build dependencies for the current arch+OS: - make -To build for another arch/OS: - make HOST=host-platform-triplet && make HOST=host-platform-triplet - (For example: make HOST=i686-w64-mingw32 -j4) - -A prefix will be generated that's suitable for plugging into Bitcoin's -configure. In the above example, a dir named i686-w64-mingw32 will be -created. To use it for Bitcoin: - -./configure --prefix=`pwd`/depends/i686-w64-mingw32 - -No other options are needed, the paths are automatically configured. - -Dependency Options: -The following can be set when running make: make FOO=bar - -SOURCES_PATH: downloaded sources will be placed here -BASE_CACHE: built packages will be placed here -SDK_PATH: Path where sdk's can be found (used by OSX) -FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up -NO_QT: Don't download/build/cache qt and its dependencies -NO_WALLET: Don't download/build/cache libs needed to enable the wallet -NO_UPNP: Don't download/build/cache packages needed for enabling upnp -DEBUG: disable some optimizations and enable more runtime checking - -If some packages are not built, for example 'make NO_WALLET=1', the appropriate -options will be passed to bitcoin's configure. In this case, --disable-wallet. - -Additional targets: -download: run 'make download' to fetch all sources without building them -download-osx: run 'make download-osx' to fetch all sources needed for osx builds -download-win: run 'make download-win' to fetch all sources needed for win builds -download-linux: run 'make download-linux' to fetch all sources needed for linux builds diff --git a/depends/README b/depends/description.md index 55e7222697..74f9ef3f20 100644 --- a/depends/README +++ b/depends/description.md @@ -1,9 +1,7 @@ -This is a system of building and caching dependencies necessary for building -Bitcoin. - +This is a system of building and caching dependencies necessary for building Bitcoin. There are several features that make it different from most similar systems: -- It is designed to be builder and host agnostic +### It is designed to be builder and host agnostic In theory, binaries for any target OS/architecture can be created, from a builder running any OS/architecture. In practice, build-side tools must be @@ -11,18 +9,18 @@ specified when the defaults don't fit, and packages must be amended to work on new hosts. For now, a build architecture of x86_64 is assumed, either on Linux or OSX. -- No reliance on timestamps +### No reliance on timestamps File presence is used to determine what needs to be built. This makes the results distributable and easily digestable by automated builders. -- Each build only has its specified dependencies available at build-time. +### Each build only has its specified dependencies available at build-time. For each build, the sysroot is wiped and the (recursive) dependencies are installed. This makes each build deterministic, since there will never be any unknown files available to cause side-effects. -- Each package is cached and only rebuilt as needed. +### Each package is cached and only rebuilt as needed. Before building, a unique build-id is generated for each package. This id consists of a hash of all files used to build the package (Makefiles, packages, @@ -32,7 +30,7 @@ any other package that depends on it. If any of the main makefiles (Makefile, funcs.mk, etc) are changed, all packages will be rebuilt. After building, the results are cached into a tarball that can be re-used and distributed. -- Package build results are (relatively) deterministic. +### Package build results are (relatively) deterministic. Each package is configured and patched so that it will yield the same build-results with each consequent build, within a reasonable set of @@ -41,13 +39,13 @@ beyond the scope of this system. Additionally, the toolchain itself must be capable of deterministic results. When revisions are properly bumped, a cached build should represent an exact single payload. -- Sources are fetched and verified automatically +### Sources are fetched and verified automatically Each package must define its source location and checksum. The build will fail if the fetched source does not match. Sources may be pre-seeded and/or cached as desired. -- Self-cleaning +### Self-cleaning Build and staging dirs are wiped after use, and any previous version of a cached result is removed following a successful build. Automated builders diff --git a/depends/funcs.mk b/depends/funcs.mk index b407737f7f..050a9b1321 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -53,12 +53,14 @@ $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix) $(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) $(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) $(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir) +$(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps -$(1)_fetched=$$($(1)_source_dir)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name) +$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned @@ -154,7 +156,10 @@ endef define int_add_cmds $($(1)_fetched): $(AT)mkdir -p $$(@D) $(SOURCES_PATH) + $(AT)rm -f $$@ + $(AT)touch $$@ $(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) $(AT)touch $$@ $($(1)_extracted): | $($(1)_fetched) $(AT)echo Extracting $(1)... @@ -195,10 +200,12 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) $(AT)rm -rf $$(@D) && mkdir -p $$(@D) $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) $(AT)rm -rf $($(1)_staging_dir) +$($(1)_cached_checksum): $($(1)_cached) + $(AT)cd $$(@D); $(build_SHA256SUM) $$(<F) > $$(@) .PHONY: $(1) -$(1): | $($(1)_cached) -.SECONDARY: $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) +$(1): | $($(1)_cached_checksum) +.SECONDARY: $($(1)_cached) $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) endef diff --git a/depends/README.packages b/depends/packages.md index 5ab7ed7dee..7c80362509 100644 --- a/depends/README.packages +++ b/depends/packages.md @@ -4,119 +4,137 @@ variables, and defining build commands. The package "mylib" will be used here as an example General tips: -mylib_foo is written as $(package)_foo in order to make recipes more similar. +- mylib_foo is written as $(package)_foo in order to make recipes more similar. -Identifiers: +## Identifiers Each package is required to define at least these variables: - $(package)_version: + + $(package)_version: Version of the upstream library or program. If there is no version, a placeholder such as 1.0 can be used. - $(package)_download_path: + + $(package)_download_path: Location of the upstream source, without the file-name. Usually http or ftp. - $(package)_file_name: + + $(package)_file_name: The upstream source filename available at the download path. - $(package)_sha256_hash: + + $(package)_sha256_hash: The sha256 hash of the upstream file These variables are optional: - $(package)_build_subdir: + + $(package)_build_subdir: cd to this dir before running configure/build/stage commands. - $(package)_download_file: + + $(package)_download_file: The file-name of the upstream source if it differs from how it should be stored locally. This can be used to avoid storing file-names with strange characters. - $(package)_dependencies: + + $(package)_dependencies: Names of any other packages that this one depends on. - $(package)_patches: + + $(package)_patches: Filenames of any patches needed to build the package + $(package)_extra_sources: + Any extra files that will be fetched via $(package)_fetch_cmds. These are + specified so that they can be fetched and verified via 'make download'. + -Build Variables: +## Build Variables: After defining the main identifiers, build variables may be added or customized before running the build commands. They should be added to a function called $(package)_set_vars. For example: -define $(package)_set_vars -... -endef + define $(package)_set_vars + ... + endef Most variables can be prefixed with the host, architecture, or both, to make the modifications specific to that case. For example: - Universal: $(package)_cc=gcc - Linux only: $(package)_linux_cc=gcc - x86_64 only: $(package)_x86_64_cc = gcc - x86_64 linux only: $(package)_x86_64_linux_cc = gcc + Universal: $(package)_cc=gcc + Linux only: $(package)_linux_cc=gcc + x86_64 only: $(package)_x86_64_cc = gcc + x86_64 linux only: $(package)_x86_64_linux_cc = gcc These variables may be set to override or append their default values. - $(package)_cc - $(package)_cxx - $(package)_objc - $(package)_objcxx - $(package)_ar - $(package)_ranlib - $(package)_libtool - $(package)_nm - $(package)_cflags - $(package)_cxxflags - $(package)_ldflags - $(package)_cppflags - $(package)_config_env - $(package)_build_env - $(package)_stage_env - $(package)_build_opts - $(package)_config_opts + + $(package)_cc + $(package)_cxx + $(package)_objc + $(package)_objcxx + $(package)_ar + $(package)_ranlib + $(package)_libtool + $(package)_nm + $(package)_cflags + $(package)_cxxflags + $(package)_ldflags + $(package)_cppflags + $(package)_config_env + $(package)_build_env + $(package)_stage_env + $(package)_build_opts + $(package)_config_opts The *_env variables are used to add environment variables to the respective commands. Many variables respect a debug/release suffix as well, in order to use them for only the appropriate build config. For example: - $(package)_cflags_release = -O3 - $(package)_cflags_i686_debug = -g - $(package)_config_opts_release = --disable-debug + + $(package)_cflags_release = -O3 + $(package)_cflags_i686_debug = -g + $(package)_config_opts_release = --disable-debug These will be used in addition to the options that do not specify debug/release. All builds are considered to be release unless DEBUG=1 is set by -the user. +the user. Other variables may be defined as needed. -Other variables may be defined as needed. - -Build commands: +## Build commands: For each build, a unique build dir and staging dir are created. For example, - work/build/mylib/1.0-1adac830f6e and work/staging/mylib/1.0-1adac830f6e. + `work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. The following build commands are available for each recipe: - $(package)_fetch_cmds: + $(package)_fetch_cmds: Runs from: build dir Fetch the source file. If undefined, it will be fetched and verified against its hash. - $(package)_extract_cmds: + + $(package)_extract_cmds: Runs from: build dir Verify the source file against its hash and extract it. If undefined, the source is assumed to be a tarball. - $(package)_preprocess_cmds: + + $(package)_preprocess_cmds: Runs from: build dir/$(package)_build_subdir Preprocess the source as necessary. If undefined, does nothing. - $(package)_config_cmds: + + $(package)_config_cmds: Runs from: build dir/$(package)_build_subdir Configure the source. If undefined, does nothing. - $(package)_build_cmds: + + $(package)_build_cmds: Runs from: build dir/$(package)_build_subdir Build the source. If undefined, does nothing. - $(package)_stage_cmds: + + $(package)_stage_cmds: Runs from: build dir/$(package)_build_subdir Stage the build results. If undefined, does nothing. The following variables are available for each recipe: - $(1)_staging_dir: package's destination sysroot path - $(1)_staging_prefix_dir: prefix path inside of the package's staging dir - $(1)_extract_dir: path to the package's extracted sources - $(1)_build_dir: path where configure/build/stage commands will be run - $(1)_patch_dir: path where the package's patches (if any) are found + + $(1)_staging_dir: package's destination sysroot path + $(1)_staging_prefix_dir: prefix path inside of the package's staging dir + $(1)_extract_dir: path to the package's extracted sources + $(1)_build_dir: path where configure/build/stage commands will be run + $(1)_patch_dir: path where the package's patches (if any) are found Notes on build commands: @@ -125,4 +143,5 @@ configure step to (usually) correctly configure automatically. Any $($(package)_config_opts) will be appended. Most autotools projects can be properly staged using: - $(MAKE) DESTDIR=$($(package)_staging_dir) install + + $(MAKE) DESTDIR=$($(package)_staging_dir) install diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 951ad4fb29..1c1bcf199a 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -9,6 +9,8 @@ $(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_versi $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_sha256_hash=60d8f69f032d62ef61bf527857ebb933741ec3352d4d328c5516aa520662dab7 +$(package)_extra_sources=$($(package)_clang_file_name) + define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 68bf7187fa..11af040ad2 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -46,6 +46,13 @@ Only supports JSON as output format. * verificationprogress : (numeric) estimate of verification progress [0..1] * chainwork : (string) total amount of work in active chain, in hexadecimal +`GET /rest/getutxos` + +The getutxo command allows querying of the UTXO set given a set of of outpoints. +See BIP64 for input and output serialisation: +https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki + + Risks ------------- Running a webbrowser on the same node with a REST enabled bitcoind can be a risk. Accessing prepared XSS websites could read out tx/block data of your node by placing links like `<script src="http://127.0.0.1:8332/rest/tx/1234567890.json">` which might break the nodes privacy. diff --git a/doc/assets-attribution.md b/doc/assets-attribution.md index c860cdc534..ebba64a61a 100644 --- a/doc/assets-attribution.md +++ b/doc/assets-attribution.md @@ -22,6 +22,7 @@ The following is a list of assets used in the bitcoin source and their proper at src/qt/res/icons/receive.png, src/qt/res/icons/remove.png, src/qt/res/icons/send.png, src/qt/res/icons/synced.png, src/qt/res/icons/transaction*.png, src/qt/res/icons/tx_output.png, + src/qt/res/icons/warning.png Jonas Schnelli ----------------------- diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index 4f82263a7a..ecde45059b 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -31,9 +31,10 @@ testScripts=( 'proxy_test.py' 'merkle_blocks.py' 'signrawtransactions.py' -# 'forknotify.py' 'maxblocksinflight.py' 'invalidblockrequest.py' + 'rawtransactions.py' +# 'forknotify.py' ); if [ "x${ENABLE_BITCOIND}${ENABLE_UTILS}${ENABLE_WALLET}" = "x111" ]; then for (( i = 0; i < ${#testScripts[@]}; i++ )) diff --git a/qa/rpc-tests/conflictedbalance.sh b/qa/rpc-tests/conflictedbalance.sh index 3b6c8dc318..7e44097374 100755 --- a/qa/rpc-tests/conflictedbalance.sh +++ b/qa/rpc-tests/conflictedbalance.sh @@ -49,7 +49,7 @@ B2ARGS="-datadir=$D2 -debug=mempool" $BITCOIND $B2ARGS & B2PID=$! -# Wait until all four nodes are at the same block number +# Wait until both nodes are at the same block number function WaitBlocks { while : do diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py index f26cbee1e2..85fd1c982a 100755 --- a/qa/rpc-tests/pruning.py +++ b/qa/rpc-tests/pruning.py @@ -7,7 +7,8 @@ # Test pruning code # ******** # WARNING: -# This test uses 4GB of disk space and takes in excess of 30 mins to run +# This test uses 4GB of disk space. +# This test takes 30 mins or more (up to 2 hours) # ******** from test_framework import BitcoinTestFramework @@ -51,11 +52,11 @@ class PruneTest(BitcoinTestFramework): self.is_network_split = False # Create nodes 0 and 1 to mine - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)) + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) # Create node 2 to test pruning - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=300)) + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900)) self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/" self.address[0] = self.nodes[0].getnewaddress() @@ -108,7 +109,7 @@ class PruneTest(BitcoinTestFramework): # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300) + self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900) # Mine 24 blocks in node 1 self.utxo = self.nodes[1].listunspent() for i in xrange(24): @@ -135,7 +136,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) stop_node(self.nodes[1],1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300) + self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) height = self.nodes[1].getblockcount() print "Current block height:", height @@ -158,7 +159,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node1 to clear those giant tx's from mempool stop_node(self.nodes[1],1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300) + self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) print "Generating new longer chain of 300 more blocks" self.nodes[1].generate(300) @@ -223,7 +224,7 @@ class PruneTest(BitcoinTestFramework): waitstart = time.time() while self.nodes[2].getblockcount() < goalbestheight: time.sleep(0.1) - if time.time() - waitstart > 300: + if time.time() - waitstart > 900: raise AssertionError("Node 2 didn't reorg to proper height") assert(self.nodes[2].getbestblockhash() == goalbesthash) # Verify we can now have the data for a block previously pruned @@ -256,7 +257,7 @@ class PruneTest(BitcoinTestFramework): def run_test(self): - print "Warning! This test requires 4GB of disk space and takes over 30 mins" + print "Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)" print "Mining a big blockchain of 995 blocks" self.create_big_chain() # Chain diagram key: diff --git a/qa/rpc-tests/rawtransactions.py b/qa/rpc-tests/rawtransactions.py index c02c929d8d..3d80c97d74 100755 --- a/qa/rpc-tests/rawtransactions.py +++ b/qa/rpc-tests/rawtransactions.py @@ -15,7 +15,7 @@ from time import sleep # Create one-input, one-output, no-fee transaction: class RawTransactionsTest(BitcoinTestFramework): - + def setup_chain(self): print("Initializing test directory "+self.options.tmpdir) initialize_chain_clean(self.options.tmpdir, 3) @@ -28,16 +28,16 @@ class RawTransactionsTest(BitcoinTestFramework): #proxy = AuthServiceProxy(url) #proxy.url = url # store URL on proxy for info #self.nodes.append(proxy) - + connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) - + self.is_network_split=False self.sync_all() - + def run_test(self): - + #prepare some coins for multiple *rawtransaction commands self.nodes[2].generate(1) self.nodes[0].generate(101) @@ -56,14 +56,89 @@ class RawTransactionsTest(BitcoinTestFramework): outputs = { self.nodes[0].getnewaddress() : 4.998 } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) rawtx = self.nodes[2].signrawtransaction(rawtx) - + errorString = "" try: rawtx = self.nodes[2].sendrawtransaction(rawtx['hex']) except JSONRPCException,e: errorString = e.error['message'] + + assert_equal("Missing inputs" in errorString, True); + + ######################### + # RAW TX MULTISIG TESTS # + ######################### + # 2of2 test + addr1 = self.nodes[2].getnewaddress() + addr2 = self.nodes[2].getnewaddress() + + addr1Obj = self.nodes[2].validateaddress(addr1) + addr2Obj = self.nodes[2].validateaddress(addr2) + + mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']]) + mSigObjValid = self.nodes[2].validateaddress(mSigObj) + + #use balance deltas instead of absolute values + bal = self.nodes[2].getbalance() + + # send 1.2 BTC to msig adr + txId = self.nodes[0].sendtoaddress(mSigObj, 1.2); + self.sync_all() + self.nodes[0].generate(1) + self.sync_all() + assert_equal(self.nodes[2].getbalance(), bal+Decimal('1.20000000')) #node2 has both keys of the 2of2 ms addr., tx should affect the balance + + + + + # 2of3 test from different nodes + bal = self.nodes[2].getbalance() + addr1 = self.nodes[1].getnewaddress() + addr2 = self.nodes[2].getnewaddress() + addr3 = self.nodes[2].getnewaddress() + + addr1Obj = self.nodes[1].validateaddress(addr1) + addr2Obj = self.nodes[2].validateaddress(addr2) + addr3Obj = self.nodes[2].validateaddress(addr3) + + mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey'], addr3Obj['pubkey']]) + mSigObjValid = self.nodes[2].validateaddress(mSigObj) + + txId = self.nodes[0].sendtoaddress(mSigObj, 2.2); + decTx = self.nodes[0].gettransaction(txId) + rawTx = self.nodes[0].decoderawtransaction(decTx['hex']) + sPK = rawTx['vout'][0]['scriptPubKey']['hex'] + self.sync_all() + self.nodes[0].generate(1) + self.sync_all() + + #THIS IS A INCOMPLETE FEATURE + #NODE2 HAS TWO OF THREE KEY AND THE FUNDS SHOULD BE SPENDABLE AND COUNT AT BALANCE CALCULATION + assert_equal(self.nodes[2].getbalance(), bal) #for now, assume the funds of a 2of3 multisig tx are not marked as spendable + + txDetails = self.nodes[0].gettransaction(txId, True) + rawTx = self.nodes[0].decoderawtransaction(txDetails['hex']) + vout = False + for outpoint in rawTx['vout']: + if outpoint['value'] == Decimal('2.20000000'): + vout = outpoint + break; + + bal = self.nodes[0].getbalance() + inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex']}] + outputs = { self.nodes[0].getnewaddress() : 2.19 } + rawTx = self.nodes[2].createrawtransaction(inputs, outputs) + rawTxPartialSigned = self.nodes[1].signrawtransaction(rawTx, inputs) + assert_equal(rawTxPartialSigned['complete'], False) #node1 only has one key, can't comp. sign the tx - assert_equal("Missing inputs" in errorString, True); + rawTxSigned = self.nodes[2].signrawtransaction(rawTx, inputs) + assert_equal(rawTxSigned['complete'], True) #node2 can sign the tx compl., own two of three keys + self.nodes[2].sendrawtransaction(rawTxSigned['hex']) + rawTx = self.nodes[0].decoderawtransaction(rawTxSigned['hex']) + self.sync_all() + self.nodes[0].generate(1) + self.sync_all() + assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx if __name__ == '__main__': RawTransactionsTest().main() diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py index 9b7008531c..9f0d049fe9 100755 --- a/qa/rpc-tests/rest.py +++ b/qa/rpc-tests/rest.py @@ -9,7 +9,10 @@ from test_framework import BitcoinTestFramework from util import * +from struct import * +import binascii import json +import StringIO try: import http.client as httplib @@ -20,45 +23,210 @@ try: except ImportError: import urlparse -def http_get_call(host, port, path, response_object = 0): +def deser_uint256(f): + r = 0 + for i in range(8): + t = unpack(b"<I", f.read(4))[0] + r += t << (i * 32) + return r + +#allows simple http get calls with a request body +def http_get_call(host, port, path, requestdata = '', response_object = 0): conn = httplib.HTTPConnection(host, port) - conn.request('GET', path) + conn.request('GET', path, requestdata) if response_object: return conn.getresponse() return conn.getresponse().read() - class RESTTest (BitcoinTestFramework): FORMAT_SEPARATOR = "." + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 3) + + def setup_network(self, split=False): + self.nodes = start_nodes(3, self.options.tmpdir) + connect_nodes_bi(self.nodes,0,1) + connect_nodes_bi(self.nodes,1,2) + connect_nodes_bi(self.nodes,0,2) + self.is_network_split=False + self.sync_all() + def run_test(self): url = urlparse.urlparse(self.nodes[0].url) + print "Mining blocks..." + + self.nodes[0].generate(1) + self.sync_all() + self.nodes[2].generate(100) + self.sync_all() + + assert_equal(self.nodes[0].getbalance(), 50) + + txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) + self.sync_all() + self.nodes[2].generate(1) + self.sync_all() + bb_hash = self.nodes[0].getbestblockhash() + + assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) #balance now should be 0.1 on node 1 + + # load the latest 0.1 tx over the REST API + json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json") + json_obj = json.loads(json_string) + vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then) + # get n of 0.1 outpoint + n = 0 + for vout in json_obj['vout']: + if vout['value'] == 0.1: + n = vout['n'] + + + ###################################### + # GETUTXOS: query a unspent outpoint # + ###################################### + json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + + #check chainTip response + assert_equal(json_obj['chaintipHash'], bb_hash) + + #make sure there is one utxo + assert_equal(len(json_obj['utxos']), 1) + assert_equal(json_obj['utxos'][0]['value'], 0.1) + + + ################################################ + # GETUTXOS: now query a already spent outpoint # + ################################################ + json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+vintx+'","n":0}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + + #check chainTip response + assert_equal(json_obj['chaintipHash'], bb_hash) + + #make sure there is no utox in the response because this oupoint has been spent + assert_equal(len(json_obj['utxos']), 0) + + #check bitmap + assert_equal(json_obj['bitmap'], "0") + + + ################################################## + # GETUTXOS: now check both with the same request # + ################################################## + json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'},{"txid":"'+vintx+'","n":0}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + assert_equal(len(json_obj['utxos']), 1) + assert_equal(json_obj['bitmap'], "10") + + #test binary response bb_hash = self.nodes[0].getbestblockhash() + binaryRequest = b'\x01\x02' + binaryRequest += binascii.unhexlify(txid) + binaryRequest += pack("i", n); + binaryRequest += binascii.unhexlify(vintx); + binaryRequest += pack("i", 0); + + bin_response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest) + + output = StringIO.StringIO() + output.write(bin_response) + output.seek(0) + chainHeight = unpack("i", output.read(4))[0] + hashFromBinResponse = hex(deser_uint256(output))[2:].zfill(65).rstrip("L") + + assert_equal(bb_hash, hashFromBinResponse) #check if getutxo's chaintip during calculation was fine + assert_equal(chainHeight, 102) #chain height must be 102 + + + ############################ + # GETUTXOS: mempool checks # + ############################ + + # do a tx and don't sync + txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) + json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json") + json_obj = json.loads(json_string) + vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then) + # get n of 0.1 outpoint + n = 0 + for vout in json_obj['vout']: + if vout['value'] == 0.1: + n = vout['n'] + + json_request = '{"checkmempool":false,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + assert_equal(len(json_obj['utxos']), 0) #there should be a outpoint because it has just added to the mempool + + json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + assert_equal(len(json_obj['utxos']), 1) #there should be a outpoint because it has just added to the mempool + + #do some invalid requests + json_request = '{"checkmempool' + response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 500) #must be a 500 because we send a invalid json request + + json_request = '{"checkmempool' + response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True) + assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request + + #test limits + json_request = '{"checkmempool":true,"outpoints":[' + for x in range(0, 200): + json_request += '{"txid":"'+txid+'","n":'+str(n)+'},' + json_request = json_request.rstrip(",") + json_request+="]}"; + response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 500) #must be a 500 because we exceeding the limits + + json_request = '{"checkmempool":true,"outpoints":[' + for x in range(0, 90): + json_request += '{"txid":"'+txid+'","n":'+str(n)+'},' + json_request = json_request.rstrip(",") + json_request+="]}"; + response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 200) #must be a 500 because we exceeding the limits + + self.nodes[0].generate(1) #generate block to not affect upcomming tests + self.sync_all() + + ################ + # /rest/block/ # + ################ + # check binary format - response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True) + response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True) assert_equal(response.status, 200) assert_greater_than(int(response.getheader('content-length')), 80) response_str = response.read() # compare with block header - response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True) + response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True) assert_equal(response_header.status, 200) assert_equal(int(response_header.getheader('content-length')), 80) response_header_str = response_header.read() assert_equal(response_str[0:80], response_header_str) # check block hex format - response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) + response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True) assert_equal(response_hex.status, 200) assert_greater_than(int(response_hex.getheader('content-length')), 160) response_hex_str = response_hex.read() assert_equal(response_str.encode("hex")[0:160], response_hex_str[0:160]) # compare with hex block header - response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) + response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True) assert_equal(response_header_hex.status, 200) assert_greater_than(int(response_header_hex.getheader('content-length')), 160) response_header_hex_str = response_header_hex.read() @@ -77,9 +245,11 @@ class RESTTest (BitcoinTestFramework): assert_equal(json_obj['txid'], tx_hash) # check hex format response - hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", True) + hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", "", True) assert_equal(hex_string.status, 200) assert_greater_than(int(response.getheader('content-length')), 10) + + # check block tx details # let's make 3 tx and mine them on node 1 diff --git a/qa/rpc-tests/smartfees.py b/qa/rpc-tests/smartfees.py index 4eb8bb4842..69f3c22c17 100755 --- a/qa/rpc-tests/smartfees.py +++ b/qa/rpc-tests/smartfees.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -# Copyright (c) 2014 The Bitcoin Core developers +# Copyright (c) 2014-2015 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,82 +11,249 @@ from test_framework import BitcoinTestFramework from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException from util import * +# Construct 2 trivial P2SH's and the ScriptSigs that spend them +# So we can create many many transactions without needing to spend +# time signing. +P2SH_1 = "2MySexEGVzZpRgNQ1JdjdP5bRETznm3roQ2" # P2SH of "OP_1 OP_DROP" +P2SH_2 = "2NBdpwq8Aoo1EEKEXPNrKvr5xQr3M9UfcZA" # P2SH of "OP_2 OP_DROP" +# Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2 +# 4 bytes of OP_TRUE and push 2-byte redeem script of "OP_1 OP_DROP" or "OP_2 OP_DROP" +SCRIPT_SIG = ["0451025175", "0451025275"] + +def satoshi_round(amount): + return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) + +def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment): + ''' + Create and send a transaction with a random fee. + The transaction pays to a trival P2SH script, and assumes that its inputs + are of the same form. + The function takes a list of confirmed outputs and unconfirmed outputs + and attempts to use the confirmed list first for its inputs. + It adds the newly created outputs to the unconfirmed list. + Returns (raw transaction, fee) + ''' + # It's best to exponentially distribute our random fees + # because the buckets are exponentially spaced. + # Exponentially distributed from 1-128 * fee_increment + rand_fee = float(fee_increment)*(1.1892**random.randint(0,28)) + # Total fee ranges from min_fee to min_fee + 127*fee_increment + fee = min_fee - fee_increment + satoshi_round(rand_fee) + inputs = [] + total_in = Decimal("0.00000000") + while total_in <= (amount + fee) and len(conflist) > 0: + t = conflist.pop(0) + total_in += t["amount"] + inputs.append({ "txid" : t["txid"], "vout" : t["vout"]} ) + if total_in <= amount + fee: + while total_in <= (amount + fee) and len(unconflist) > 0: + t = unconflist.pop(0) + total_in += t["amount"] + inputs.append({ "txid" : t["txid"], "vout" : t["vout"]} ) + if total_in <= amount + fee: + raise RuntimeError("Insufficient funds: need %d, have %d"%(amount+fee, total_in)) + outputs = {} + outputs[P2SH_1] = total_in - amount - fee + outputs[P2SH_2] = amount + rawtx = from_node.createrawtransaction(inputs, outputs) + # Createrawtransaction constructions a transaction that is ready to be signed + # These transactions don't need to be signed, but we still have to insert the ScriptSig + # that will satisfy the ScriptPubKey. + completetx = rawtx[0:10] + inputnum = 0 + for inp in inputs: + completetx += rawtx[10+82*inputnum:82+82*inputnum] + completetx += SCRIPT_SIG[inp["vout"]] + completetx += rawtx[84+82*inputnum:92+82*inputnum] + inputnum += 1 + completetx += rawtx[10+82*inputnum:] + txid = from_node.sendrawtransaction(completetx, True) + unconflist.append({ "txid" : txid, "vout" : 0 , "amount" : total_in - amount - fee}) + unconflist.append({ "txid" : txid, "vout" : 1 , "amount" : amount}) + + return (completetx, fee) + +def split_inputs(from_node, txins, txouts, initial_split = False): + ''' + We need to generate a lot of very small inputs so we can generate a ton of transactions + and they will have low priority. + This function takes an input from txins, and creates and sends a transaction + which splits the value into 2 outputs which are appended to txouts. + ''' + prevtxout = txins.pop() + inputs = [] + outputs = {} + inputs.append({ "txid" : prevtxout["txid"], "vout" : prevtxout["vout"] }) + half_change = satoshi_round(prevtxout["amount"]/2) + rem_change = prevtxout["amount"] - half_change - Decimal("0.00001000") + outputs[P2SH_1] = half_change + outputs[P2SH_2] = rem_change + rawtx = from_node.createrawtransaction(inputs, outputs) + # If this is the initial split we actually need to sign the transaction + # Otherwise we just need to insert the property ScriptSig + if (initial_split) : + completetx = from_node.signrawtransaction(rawtx)["hex"] + else : + completetx = rawtx[0:82] + SCRIPT_SIG[prevtxout["vout"]] + rawtx[84:] + txid = from_node.sendrawtransaction(completetx, True) + txouts.append({ "txid" : txid, "vout" : 0 , "amount" : half_change}) + txouts.append({ "txid" : txid, "vout" : 1 , "amount" : rem_change}) + +def check_estimates(node, fees_seen, max_invalid, print_estimates = True): + ''' + This function calls estimatefee and verifies that the estimates + meet certain invariants. + ''' + all_estimates = [ node.estimatefee(i) for i in range(1,26) ] + if print_estimates: + print([str(all_estimates[e-1]) for e in [1,2,3,6,15,25]]) + delta = 1.0e-6 # account for rounding error + last_e = max(fees_seen) + for e in filter(lambda x: x >= 0, all_estimates): + # Estimates should be within the bounds of what transactions fees actually were: + if float(e)+delta < min(fees_seen) or float(e)-delta > max(fees_seen): + raise AssertionError("Estimated fee (%f) out of range (%f,%f)" + %(float(e), min(fees_seen), max(fees_seen))) + # Estimates should be monotonically decreasing + if float(e)-delta > last_e: + raise AssertionError("Estimated fee (%f) larger than last fee (%f) for lower number of confirms" + %(float(e),float(last_e))) + last_e = e + valid_estimate = False + invalid_estimates = 0 + for e in all_estimates: + if e >= 0: + valid_estimate = True + else: + invalid_estimates += 1 + # Once we're at a high enough confirmation count that we can give an estimate + # We should have estimates for all higher confirmation counts + if valid_estimate and e < 0: + raise AssertionError("Invalid estimate appears at higher confirm count than valid estimate") + # Check on the expected number of different confirmation counts + # that we might not have valid estimates for + if invalid_estimates > max_invalid: + raise AssertionError("More than (%d) invalid estimates"%(max_invalid)) + return all_estimates + + class EstimateFeeTest(BitcoinTestFramework): def setup_network(self): + ''' + We'll setup the network to have 3 nodes that all mine with different parameters. + But first we need to use one node to create a lot of small low priority outputs + which we will use to generate our transactions. + ''' self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, - ["-debug=mempool", "-debug=estimatefee", "-relaypriority=0"])) - # Node1 mines small-but-not-tiny blocks, and allows free transactions. + # Use node0 to mine blocks for input splitting + self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", + "-relaypriority=0", "-whitelist=127.0.0.1"])) + + print("This test is time consuming, please be patient") + print("Splitting inputs to small size so we can generate low priority tx's") + self.txouts = [] + self.txouts2 = [] + # Split a coinbase into two transaction puzzle outputs + split_inputs(self.nodes[0], self.nodes[0].listunspent(0), self.txouts, True) + + # Mine + while (len(self.nodes[0].getrawmempool()) > 0): + self.nodes[0].generate(1) + + # Repeatedly split those 2 outputs, doubling twice for each rep + # Use txouts to monitor the available utxo, since these won't be tracked in wallet + reps = 0 + while (reps < 5): + #Double txouts to txouts2 + while (len(self.txouts)>0): + split_inputs(self.nodes[0], self.txouts, self.txouts2) + while (len(self.nodes[0].getrawmempool()) > 0): + self.nodes[0].generate(1) + #Double txouts2 to txouts + while (len(self.txouts2)>0): + split_inputs(self.nodes[0], self.txouts2, self.txouts) + while (len(self.nodes[0].getrawmempool()) > 0): + self.nodes[0].generate(1) + reps += 1 + print("Finished splitting") + + # Now we can connect the other nodes, didn't want to connect them earlier + # so the estimates would not be affected by the splitting transactions + # Node1 mines small blocks but that are bigger than the expected transaction rate, + # and allows free transactions. # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes, - # so blockmaxsize of 2,000 is really just 1,000 bytes (room enough for - # 6 or 7 transactions) + # (17k is room enough for 110 or so transactions) self.nodes.append(start_node(1, self.options.tmpdir, - ["-blockprioritysize=1500", "-blockmaxsize=2000", - "-debug=mempool", "-debug=estimatefee", "-relaypriority=0"])) + ["-blockprioritysize=1500", "-blockmaxsize=18000", + "-maxorphantx=1000", "-relaypriority=0", "-debug=estimatefee"])) connect_nodes(self.nodes[1], 0) # Node2 is a stingy miner, that - # produces very small blocks (room for only 3 or so transactions) - node2args = [ "-blockprioritysize=0", "-blockmaxsize=1500", - "-debug=mempool", "-debug=estimatefee", "-relaypriority=0"] + # produces too small blocks (room for only 70 or so transactions) + node2args = ["-blockprioritysize=0", "-blockmaxsize=12000", "-maxorphantx=1000", "-relaypriority=0"] + self.nodes.append(start_node(2, self.options.tmpdir, node2args)) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[2], 1) self.is_network_split = False self.sync_all() - + + def transact_and_mine(self, numblocks, mining_node): + min_fee = Decimal("0.00001") + # We will now mine numblocks blocks generating on average 100 transactions between each block + # We shuffle our confirmed txout set before each set of transactions + # small_txpuzzle_randfee will use the transactions that have inputs already in the chain when possible + # resorting to tx's that depend on the mempool when those run out + for i in range(numblocks): + random.shuffle(self.confutxo) + for j in range(random.randrange(100-50,100+50)): + from_index = random.randint(1,2) + (txhex, fee) = small_txpuzzle_randfee(self.nodes[from_index], self.confutxo, + self.memutxo, Decimal("0.005"), min_fee, min_fee) + tx_kbytes = (len(txhex)/2)/1000.0 + self.fees_per_kb.append(float(fee)/tx_kbytes) + sync_mempools(self.nodes[0:3],.1) + mined = mining_node.getblock(mining_node.generate(1)[0],True)["tx"] + sync_blocks(self.nodes[0:3],.1) + #update which txouts are confirmed + newmem = [] + for utx in self.memutxo: + if utx["txid"] in mined: + self.confutxo.append(utx) + else: + newmem.append(utx) + self.memutxo = newmem def run_test(self): - # Prime the memory pool with pairs of transactions - # (high-priority, random fee and zero-priority, random fee) - min_fee = Decimal("0.001") - fees_per_kb = []; - for i in range(12): - (txid, txhex, fee) = random_zeropri_transaction(self.nodes, Decimal("1.1"), - min_fee, min_fee, 20) - tx_kbytes = (len(txhex)/2)/1000.0 - fees_per_kb.append(float(fee)/tx_kbytes) - - # Mine blocks with node2 until the memory pool clears: - count_start = self.nodes[2].getblockcount() - while len(self.nodes[2].getrawmempool()) > 0: - self.nodes[2].generate(1) - self.sync_all() - - all_estimates = [ self.nodes[0].estimatefee(i) for i in range(1,20) ] - print("Fee estimates, super-stingy miner: "+str([str(e) for e in all_estimates])) + self.fees_per_kb = [] + self.memutxo = [] + self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting + print("Checking estimates for 1/2/3/6/15/25 blocks") + print("Creating transactions and mining them with a huge block size") + # Create transactions and mine 20 big blocks with node 0 such that the mempool is always emptied + self.transact_and_mine(30, self.nodes[0]) + check_estimates(self.nodes[1], self.fees_per_kb, 1) - # Estimates should be within the bounds of what transactions fees actually were: - delta = 1.0e-6 # account for rounding error - for e in filter(lambda x: x >= 0, all_estimates): - if float(e)+delta < min(fees_per_kb) or float(e)-delta > max(fees_per_kb): - raise AssertionError("Estimated fee (%f) out of range (%f,%f)"%(float(e), min_fee_kb, max_fee_kb)) - - # Generate transactions while mining 30 more blocks, this time with node1: - for i in range(30): - for j in range(random.randrange(6-4,6+4)): - (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), - Decimal("0.0"), min_fee, 20) - tx_kbytes = (len(txhex)/2)/1000.0 - fees_per_kb.append(float(fee)/tx_kbytes) - self.nodes[1].generate(1) - self.sync_all() + print("Creating transactions and mining them with a block size that can't keep up") + # Create transactions and mine 30 small blocks with node 2, but create txs faster than we can mine + self.transact_and_mine(20, self.nodes[2]) + check_estimates(self.nodes[1], self.fees_per_kb, 3) - all_estimates = [ self.nodes[0].estimatefee(i) for i in range(1,20) ] - print("Fee estimates, more generous miner: "+str([ str(e) for e in all_estimates])) - for e in filter(lambda x: x >= 0, all_estimates): - if float(e)+delta < min(fees_per_kb) or float(e)-delta > max(fees_per_kb): - raise AssertionError("Estimated fee (%f) out of range (%f,%f)"%(float(e), min_fee_kb, max_fee_kb)) + print("Creating transactions and mining them at a block size that is just big enough") + # Generate transactions while mining 40 more blocks, this time with node1 + # which mines blocks with capacity just above the rate that transactions are being created + self.transact_and_mine(40, self.nodes[1]) + check_estimates(self.nodes[1], self.fees_per_kb, 2) # Finish by mining a normal-sized block: - while len(self.nodes[0].getrawmempool()) > 0: - self.nodes[0].generate(1) - self.sync_all() - - final_estimates = [ self.nodes[0].estimatefee(i) for i in range(1,20) ] - print("Final fee estimates: "+str([ str(e) for e in final_estimates])) + while len(self.nodes[1].getrawmempool()) > 0: + self.nodes[1].generate(1) + sync_blocks(self.nodes[0:3],.1) + print("Final estimates after emptying mempools") + check_estimates(self.nodes[1], self.fees_per_kb, 2) if __name__ == '__main__': EstimateFeeTest().main() diff --git a/qa/rpc-tests/util.py b/qa/rpc-tests/util.py index 3b4a10e46b..997bbcc373 100644 --- a/qa/rpc-tests/util.py +++ b/qa/rpc-tests/util.py @@ -33,7 +33,7 @@ def check_json_precision(): if satoshis != 2000000000000003: raise RuntimeError("JSON encode/decode loses precision") -def sync_blocks(rpc_connections): +def sync_blocks(rpc_connections, wait=1): """ Wait until everybody has the same block count """ @@ -41,9 +41,9 @@ def sync_blocks(rpc_connections): counts = [ x.getblockcount() for x in rpc_connections ] if counts == [ counts[0] ]*len(counts): break - time.sleep(1) + time.sleep(wait) -def sync_mempools(rpc_connections): +def sync_mempools(rpc_connections, wait=1): """ Wait until everybody has the same transactions in their memory pools @@ -56,7 +56,7 @@ def sync_mempools(rpc_connections): num_match = num_match+1 if num_match == len(rpc_connections): break - time.sleep(1) + time.sleep(wait) bitcoind_processes = {} diff --git a/src/Makefile.am b/src/Makefile.am index 72d79619b2..91d530222f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -100,12 +100,14 @@ BITCOIN_CORE_H = \ leveldbwrapper.h \ limitedmap.h \ main.h \ + memusage.h \ merkleblock.h \ miner.h \ mruset.h \ netbase.h \ net.h \ noui.h \ + policy/fees.h \ pow.h \ primitives/block.h \ primitives/transaction.h \ @@ -115,6 +117,7 @@ BITCOIN_CORE_H = \ rpcclient.h \ rpcprotocol.h \ rpcserver.h \ + scheduler.h \ script/interpreter.h \ script/script_error.h \ script/script.h \ @@ -182,6 +185,7 @@ libbitcoin_server_a_SOURCES = \ miner.cpp \ net.cpp \ noui.cpp \ + policy/fees.cpp \ pow.cpp \ rest.cpp \ rpcblockchain.cpp \ @@ -257,6 +261,7 @@ libbitcoin_common_a_SOURCES = \ netbase.cpp \ protocol.cpp \ pubkey.cpp \ + scheduler.cpp \ script/interpreter.cpp \ script/script.cpp \ script/sign.cpp \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 31fe3a9f69..6b7c42285d 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -256,6 +256,7 @@ RES_ICONS = \ qt/res/icons/tx_input.png \ qt/res/icons/tx_output.png \ qt/res/icons/tx_mined.png \ + qt/res/icons/warning.png \ qt/res/icons/verify.png BITCOIN_QT_CPP = \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9aec13ccef..0997148117 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -57,9 +57,11 @@ BITCOIN_TESTS =\ test/multisig_tests.cpp \ test/netbase_tests.cpp \ test/pmt_tests.cpp \ + test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ + test/scheduler_tests.cpp \ test/script_P2SH_tests.cpp \ test/script_tests.cpp \ test/scriptnum_tests.cpp \ diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index d024b48020..c82d4f93a8 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -442,9 +442,18 @@ static void MutateTxSign(CMutableTransaction& tx, const string& flagStr) tx = mergedTx; } +class Secp256k1Init +{ +public: + Secp256k1Init() { ECC_Start(); } + ~Secp256k1Init() { ECC_Stop(); } +}; + static void MutateTx(CMutableTransaction& tx, const string& command, const string& commandVal) { + boost::scoped_ptr<Secp256k1Init> ecc; + if (command == "nversion") MutateTxVersion(tx, commandVal); else if (command == "locktime") @@ -462,8 +471,10 @@ static void MutateTx(CMutableTransaction& tx, const string& command, else if (command == "outscript") MutateTxAddOutScript(tx, commandVal); - else if (command == "sign") + else if (command == "sign") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxSign(tx, commandVal); + } else if (command == "load") RegisterLoad(commandVal); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index eeca8655c9..cce687ac98 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -8,6 +8,7 @@ #include "init.h" #include "main.h" #include "noui.h" +#include "scheduler.h" #include "util.h" #include <boost/algorithm/string/predicate.hpp> @@ -55,6 +56,7 @@ void WaitForShutdown(boost::thread_group* threadGroup) bool AppInit(int argc, char* argv[]) { boost::thread_group threadGroup; + CScheduler scheduler; bool fRet = false; @@ -142,7 +144,7 @@ bool AppInit(int argc, char* argv[]) #endif SoftSetBoolArg("-server", true); - fRet = AppInit2(threadGroup); + fRet = AppInit2(threadGroup, scheduler); } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInit()"); diff --git a/src/coins.cpp b/src/coins.cpp index d79e29951b..a41d5a310d 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -4,6 +4,7 @@ #include "coins.h" +#include "memusage.h" #include "random.h" #include <assert.h> @@ -57,13 +58,17 @@ bool CCoinsViewBacked::GetStats(CCoinsStats &stats) const { return base->GetStat CCoinsKeyHasher::CCoinsKeyHasher() : salt(GetRandHash()) {} -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false) { } +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } CCoinsViewCache::~CCoinsViewCache() { assert(!hasModifier); } +size_t CCoinsViewCache::DynamicMemoryUsage() const { + return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; +} + CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { CCoinsMap::iterator it = cacheCoins.find(txid); if (it != cacheCoins.end()) @@ -78,6 +83,7 @@ CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } + cachedCoinsUsage += memusage::DynamicUsage(ret->second.coins); return ret; } @@ -93,6 +99,7 @@ bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { assert(!hasModifier); std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); + size_t cachedCoinUsage = 0; if (ret.second) { if (!base->GetCoins(txid, ret.first->second.coins)) { // The parent view does not have this entry; mark it as fresh. @@ -102,10 +109,12 @@ CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { // The parent view only has a pruned entry for this; mark it as fresh. ret.first->second.flags = CCoinsCacheEntry::FRESH; } + } else { + cachedCoinUsage = memusage::DynamicUsage(ret.first->second.coins); } // Assume that whenever ModifyCoins is called, the entry will be modified. ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first); + return CCoinsModifier(*this, ret.first, cachedCoinUsage); } const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { @@ -150,6 +159,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn assert(it->second.flags & CCoinsCacheEntry::FRESH); CCoinsCacheEntry& entry = cacheCoins[it->first]; entry.coins.swap(it->second.coins); + cachedCoinsUsage += memusage::DynamicUsage(entry.coins); entry.flags = CCoinsCacheEntry::DIRTY | CCoinsCacheEntry::FRESH; } } else { @@ -157,10 +167,13 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. + cachedCoinsUsage -= memusage::DynamicUsage(itUs->second.coins); cacheCoins.erase(itUs); } else { // A normal modification. + cachedCoinsUsage -= memusage::DynamicUsage(itUs->second.coins); itUs->second.coins.swap(it->second.coins); + cachedCoinsUsage += memusage::DynamicUsage(itUs->second.coins); itUs->second.flags |= CCoinsCacheEntry::DIRTY; } } @@ -175,6 +188,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn bool CCoinsViewCache::Flush() { bool fOk = base->BatchWrite(cacheCoins, hashBlock); cacheCoins.clear(); + cachedCoinsUsage = 0; return fOk; } @@ -232,7 +246,7 @@ double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight) const return tx.ComputePriority(dResult); } -CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_) : cache(cache_), it(it_) { +CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage) : cache(cache_), it(it_), cachedCoinUsage(usage) { assert(!cache.hasModifier); cache.hasModifier = true; } @@ -242,7 +256,11 @@ CCoinsModifier::~CCoinsModifier() assert(cache.hasModifier); cache.hasModifier = false; it->second.coins.Cleanup(); + cache.cachedCoinsUsage -= cachedCoinUsage; // Subtract the old usage if ((it->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { cache.cacheCoins.erase(it); + } else { + // If the coin still exists after the modification, add the new usage + cache.cachedCoinsUsage += memusage::DynamicUsage(it->second.coins); } } diff --git a/src/coins.h b/src/coins.h index fe2eaa08e5..a4671645df 100644 --- a/src/coins.h +++ b/src/coins.h @@ -7,6 +7,7 @@ #define BITCOIN_COINS_H #include "compressor.h" +#include "memusage.h" #include "serialize.h" #include "uint256.h" @@ -252,6 +253,15 @@ public: return false; return true; } + + size_t DynamicMemoryUsage() const { + size_t ret = memusage::DynamicUsage(vout); + BOOST_FOREACH(const CTxOut &out, vout) { + const std::vector<unsigned char> *script = &out.scriptPubKey; + ret += memusage::DynamicUsage(*script); + } + return ret; + } }; class CCoinsKeyHasher @@ -356,7 +366,8 @@ class CCoinsModifier private: CCoinsViewCache& cache; CCoinsMap::iterator it; - CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_); + size_t cachedCoinUsage; // Cached memory usage of the CCoins object before modification + CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage); public: CCoins* operator->() { return &it->second.coins; } @@ -372,6 +383,7 @@ protected: /* Whether this cache has an active modifier. */ bool hasModifier; + /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". @@ -379,6 +391,9 @@ protected: mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; + /* Cached dynamic memory usage for the inner CCoins objects. */ + mutable size_t cachedCoinsUsage; + public: CCoinsViewCache(CCoinsView *baseIn); ~CCoinsViewCache(); @@ -414,6 +429,9 @@ public: //! Calculate the size of the cache (in number of transactions) unsigned int GetCacheSize() const; + //! Calculate the size of the cache (in bytes) + size_t DynamicMemoryUsage() const; + /** * Amount of bitcoins coming in to a transaction * Note that lightweight clients may not know anything besides the hash of previous transactions, diff --git a/src/hash.cpp b/src/hash.cpp index 20d5d21777..9711293e38 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -5,6 +5,7 @@ #include "hash.h" #include "crypto/common.h" #include "crypto/hmac_sha512.h" +#include "pubkey.h" inline uint32_t ROTL32(uint32_t x, int8_t r) @@ -71,15 +72,12 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char return h1; } -void BIP32Hash(const unsigned char chainCode[32], unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]) +void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]) { unsigned char num[4]; num[0] = (nChild >> 24) & 0xFF; num[1] = (nChild >> 16) & 0xFF; num[2] = (nChild >> 8) & 0xFF; num[3] = (nChild >> 0) & 0xFF; - CHMAC_SHA512(chainCode, 32).Write(&header, 1) - .Write(data, 32) - .Write(num, 4) - .Finalize(output); + CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output); } diff --git a/src/hash.h b/src/hash.h index e56b784a61..0771555623 100644 --- a/src/hash.h +++ b/src/hash.h @@ -14,6 +14,8 @@ #include <vector> +typedef uint256 ChainCode; + /** A hasher class for Bitcoin's 256-bit hash (double SHA-256). */ class CHash256 { private: @@ -159,6 +161,6 @@ uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char>& vDataToHash); -void BIP32Hash(const unsigned char chainCode[32], unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]); +void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]); #endif // BITCOIN_HASH_H diff --git a/src/init.cpp b/src/init.cpp index 00bf7b695a..2d75850253 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -19,6 +19,7 @@ #include "net.h" #include "rpcserver.h" #include "script/standard.h" +#include "scheduler.h" #include "txdb.h" #include "ui_interface.h" #include "util.h" @@ -194,6 +195,7 @@ void Shutdown() delete pwalletMain; pwalletMain = NULL; #endif + ECC_Stop(); LogPrintf("%s: done\n", __func__); } @@ -278,7 +280,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); - + strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files on startup")); #if !defined(WIN32) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif @@ -563,7 +565,7 @@ bool InitSanityCheck(void) /** Initialize bitcoin. * @pre Parameters should be parsed and config file should be read. */ -bool AppInit2(boost::thread_group& threadGroup) +bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -833,17 +835,20 @@ bool AppInit2(boost::thread_group& threadGroup) } } nTxConfirmTarget = GetArg("-txconfirmtarget", 1); - bSpendZeroConfChange = GetArg("-spendzeroconfchange", true); - fSendFreeTransactions = GetArg("-sendfreetransactions", false); + bSpendZeroConfChange = GetBoolArg("-spendzeroconfchange", true); + fSendFreeTransactions = GetBoolArg("-sendfreetransactions", false); std::string strWalletFile = GetArg("-wallet", "wallet.dat"); #endif // ENABLE_WALLET - fIsBareMultisigStd = GetArg("-permitbaremultisig", true) != 0; + fIsBareMultisigStd = GetBoolArg("-permitbaremultisig", true); nMaxDatacarrierBytes = GetArg("-datacarriersize", nMaxDatacarrierBytes); // ********************************************************* Step 4: application initialization: dir lock, daemonize, pidfile, debug log + // Initialize elliptic curve code + ECC_Start(); + // Sanity check if (!InitSanityCheck()) return InitError(_("Initialization sanity check failed. Bitcoin Core is shutting down.")); @@ -886,6 +891,10 @@ bool AppInit2(boost::thread_group& threadGroup) threadGroup.create_thread(&ThreadScriptCheck); } + // Start the lightweight task scheduler thread + CScheduler::Function serviceLoop = boost::bind(&CScheduler::serviceQueue, &scheduler); + threadGroup.create_thread(boost::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections * that the server is there and will be ready later). Warmup mode will @@ -951,7 +960,7 @@ bool AppInit2(boost::thread_group& threadGroup) proxyType addrProxy; bool fProxy = false; if (mapArgs.count("-proxy")) { - addrProxy = proxyType(CService(mapArgs["-proxy"], 9050), GetArg("-proxyrandomize", true)); + addrProxy = proxyType(CService(mapArgs["-proxy"], 9050), GetBoolArg("-proxyrandomize", true)); if (!addrProxy.IsValid()) return InitError(strprintf(_("Invalid -proxy address: '%s'"), mapArgs["-proxy"])); @@ -968,7 +977,7 @@ bool AppInit2(boost::thread_group& threadGroup) if (!mapArgs.count("-onion")) addrOnion = addrProxy; else - addrOnion = proxyType(CService(mapArgs["-onion"], 9050), GetArg("-proxyrandomize", true)); + addrOnion = proxyType(CService(mapArgs["-onion"], 9050), GetBoolArg("-proxyrandomize", true)); if (!addrOnion.IsValid()) return InitError(strprintf(_("Invalid -onion address: '%s'"), mapArgs["-onion"])); SetProxy(NET_TOR, addrOnion); @@ -1052,18 +1061,20 @@ bool AppInit2(boost::thread_group& threadGroup) } // cache size calculations - size_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); - if (nTotalCache < (nMinDbCache << 20)) - nTotalCache = (nMinDbCache << 20); // total cache cannot be less than nMinDbCache - else if (nTotalCache > (nMaxDbCache << 20)) - nTotalCache = (nMaxDbCache << 20); // total cache cannot be greater than nMaxDbCache - size_t nBlockTreeDBCache = nTotalCache / 8; + int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); + nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache + nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache + int64_t nBlockTreeDBCache = nTotalCache / 8; if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false)) nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB nTotalCache -= nBlockTreeDBCache; - size_t nCoinDBCache = nTotalCache / 2; // use half of the remaining cache for coindb cache + int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nTotalCache -= nCoinDBCache; - nCoinCacheSize = nTotalCache / 300; // coins in memory require around 300 bytes + nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache + LogPrintf("Cache configuration:\n"); + LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); + LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); + LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024)); bool fLoaded = false; while (!fLoaded) { @@ -1369,7 +1380,7 @@ bool AppInit2(boost::thread_group& threadGroup) LogPrintf("mapAddressBook.size() = %u\n", pwalletMain ? pwalletMain->mapAddressBook.size() : 0); #endif - StartNode(threadGroup); + StartNode(threadGroup, scheduler); #ifdef ENABLE_WALLET // Generate coins in the background diff --git a/src/init.h b/src/init.h index d451f65be9..dcb2b29360 100644 --- a/src/init.h +++ b/src/init.h @@ -8,6 +8,7 @@ #include <string> +class CScheduler; class CWallet; namespace boost @@ -20,7 +21,7 @@ extern CWallet* pwalletMain; void StartShutdown(); bool ShutdownRequested(); void Shutdown(); -bool AppInit2(boost::thread_group& threadGroup); +bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler); /** The help message mode determines what help message to show */ enum HelpMessageMode { diff --git a/src/key.cpp b/src/key.cpp index e146e47d0d..b772dff333 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -14,21 +14,7 @@ #include <secp256k1.h> #include "ecwrapper.h" -//! anonymous namespace -namespace { - -class CSecp256k1Init { -public: - CSecp256k1Init() { - secp256k1_start(SECP256K1_START_SIGN); - } - ~CSecp256k1Init() { - secp256k1_stop(); - } -}; -static CSecp256k1Init instance_of_csecp256k1; - -} // anon namespace +static secp256k1_context_t* secp256k1_context = NULL; bool CKey::Check(const unsigned char *vch) { return eccrypto::Check(vch); @@ -44,7 +30,7 @@ void CKey::MakeNewKey(bool fCompressedIn) { } bool CKey::SetPrivKey(const CPrivKey &privkey, bool fCompressedIn) { - if (!secp256k1_ec_privkey_import((unsigned char*)begin(), &privkey[0], privkey.size())) + if (!secp256k1_ec_privkey_import(secp256k1_context, (unsigned char*)begin(), &privkey[0], privkey.size())) return false; fCompressed = fCompressedIn; fValid = true; @@ -57,7 +43,7 @@ CPrivKey CKey::GetPrivKey() const { int privkeylen, ret; privkey.resize(279); privkeylen = 279; - ret = secp256k1_ec_privkey_export(begin(), (unsigned char*)&privkey[0], &privkeylen, fCompressed); + ret = secp256k1_ec_privkey_export(secp256k1_context, begin(), (unsigned char*)&privkey[0], &privkeylen, fCompressed); assert(ret); privkey.resize(privkeylen); return privkey; @@ -67,7 +53,7 @@ CPubKey CKey::GetPubKey() const { assert(fValid); CPubKey result; int clen = 65; - int ret = secp256k1_ec_pubkey_create((unsigned char*)result.begin(), &clen, begin(), fCompressed); + int ret = secp256k1_ec_pubkey_create(secp256k1_context, (unsigned char*)result.begin(), &clen, begin(), fCompressed); assert((int)result.size() == clen); assert(ret); assert(result.IsValid()); @@ -81,7 +67,7 @@ bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig, uint32_ int nSigLen = 72; unsigned char extra_entropy[32] = {0}; WriteLE32(extra_entropy, test_case); - int ret = secp256k1_ecdsa_sign(hash.begin(), (unsigned char*)&vchSig[0], &nSigLen, begin(), secp256k1_nonce_function_rfc6979, test_case ? extra_entropy : NULL); + int ret = secp256k1_ecdsa_sign(secp256k1_context, hash.begin(), (unsigned char*)&vchSig[0], &nSigLen, begin(), secp256k1_nonce_function_rfc6979, test_case ? extra_entropy : NULL); assert(ret); vchSig.resize(nSigLen); return true; @@ -106,7 +92,7 @@ bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) return false; vchSig.resize(65); int rec = -1; - int ret = secp256k1_ecdsa_sign_compact(hash.begin(), &vchSig[1], begin(), secp256k1_nonce_function_rfc6979, NULL, &rec); + int ret = secp256k1_ecdsa_sign_compact(secp256k1_context, hash.begin(), &vchSig[1], begin(), secp256k1_nonce_function_rfc6979, NULL, &rec); assert(ret); assert(rec != -1); vchSig[0] = 27 + rec + (fCompressed ? 4 : 0); @@ -114,7 +100,7 @@ bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) } bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck=false) { - if (!secp256k1_ec_privkey_import((unsigned char*)begin(), &privkey[0], privkey.size())) + if (!secp256k1_ec_privkey_import(secp256k1_context, (unsigned char*)begin(), &privkey[0], privkey.size())) return false; fCompressed = vchPubKey.IsCompressed(); fValid = true; @@ -125,7 +111,7 @@ bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck=false) { return VerifyPubKey(vchPubKey); } -bool CKey::Derive(CKey& keyChild, unsigned char ccChild[32], unsigned int nChild, const unsigned char cc[32]) const { +bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert(IsCompressed()); unsigned char out[64]; @@ -138,9 +124,9 @@ bool CKey::Derive(CKey& keyChild, unsigned char ccChild[32], unsigned int nChild assert(begin() + 32 == end()); BIP32Hash(cc, nChild, 0, begin(), out); } - memcpy(ccChild, out+32, 32); + memcpy(ccChild.begin(), out+32, 32); memcpy((unsigned char*)keyChild.begin(), begin(), 32); - bool ret = secp256k1_ec_privkey_tweak_add((unsigned char*)keyChild.begin(), out); + bool ret = secp256k1_ec_privkey_tweak_add(secp256k1_context, (unsigned char*)keyChild.begin(), out); UnlockObject(out); keyChild.fCompressed = true; keyChild.fValid = ret; @@ -152,7 +138,7 @@ bool CExtKey::Derive(CExtKey &out, unsigned int nChild) const { CKeyID id = key.GetPubKey().GetID(); memcpy(&out.vchFingerprint[0], &id, 4); out.nChild = nChild; - return key.Derive(out.key, out.vchChainCode, nChild, vchChainCode); + return key.Derive(out.key, out.chaincode, nChild, chaincode); } void CExtKey::SetMaster(const unsigned char *seed, unsigned int nSeedLen) { @@ -161,7 +147,7 @@ void CExtKey::SetMaster(const unsigned char *seed, unsigned int nSeedLen) { LockObject(out); CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(out); key.Set(&out[0], &out[32], true); - memcpy(vchChainCode, &out[32], 32); + memcpy(chaincode.begin(), &out[32], 32); UnlockObject(out); nDepth = 0; nChild = 0; @@ -174,7 +160,7 @@ CExtPubKey CExtKey::Neuter() const { memcpy(&ret.vchFingerprint[0], &vchFingerprint[0], 4); ret.nChild = nChild; ret.pubkey = key.GetPubKey(); - memcpy(&ret.vchChainCode[0], &vchChainCode[0], 32); + ret.chaincode = chaincode; return ret; } @@ -183,7 +169,7 @@ void CExtKey::Encode(unsigned char code[74]) const { memcpy(code+1, vchFingerprint, 4); code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; - memcpy(code+9, vchChainCode, 32); + memcpy(code+9, chaincode.begin(), 32); code[41] = 0; assert(key.size() == 32); memcpy(code+42, key.begin(), 32); @@ -193,7 +179,7 @@ void CExtKey::Decode(const unsigned char code[74]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; - memcpy(vchChainCode, code+9, 32); + memcpy(chaincode.begin(), code+9, 32); key.Set(code+42, code+74, true); } @@ -206,3 +192,32 @@ bool ECC_InitSanityCheck() { CPubKey pubkey = key.GetPubKey(); return key.VerifyPubKey(pubkey); } + + +void ECC_Start() { + assert(secp256k1_context == NULL); + + secp256k1_context_t *ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + assert(ctx != NULL); + + { + // Pass in a random blinding seed to the secp256k1 context. + unsigned char seed[32]; + LockObject(seed); + GetRandBytes(seed, 32); + bool ret = secp256k1_context_randomize(ctx, seed); + assert(ret); + UnlockObject(seed); + } + + secp256k1_context = ctx; +} + +void ECC_Stop() { + secp256k1_context_t *ctx = secp256k1_context; + secp256k1_context = NULL; + + if (ctx) { + secp256k1_context_destroy(ctx); + } +} @@ -6,6 +6,7 @@ #ifndef BITCOIN_KEY_H #define BITCOIN_KEY_H +#include "pubkey.h" #include "serialize.h" #include "support/allocators/secure.h" #include "uint256.h" @@ -13,9 +14,6 @@ #include <stdexcept> #include <vector> -class CPubKey; - -struct CExtPubKey; /** * secp256k1: @@ -138,7 +136,7 @@ public: bool SignCompact(const uint256& hash, std::vector<unsigned char>& vchSig) const; //! Derive BIP32 child key. - bool Derive(CKey& keyChild, unsigned char ccChild[32], unsigned int nChild, const unsigned char cc[32]) const; + bool Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const; /** * Verify thoroughly whether a private key and a public key match. @@ -157,13 +155,13 @@ struct CExtKey { unsigned char nDepth; unsigned char vchFingerprint[4]; unsigned int nChild; - unsigned char vchChainCode[32]; + ChainCode chaincode; CKey key; friend bool operator==(const CExtKey& a, const CExtKey& b) { return a.nDepth == b.nDepth && memcmp(&a.vchFingerprint[0], &b.vchFingerprint[0], 4) == 0 && a.nChild == b.nChild && - memcmp(&a.vchChainCode[0], &b.vchChainCode[0], 32) == 0 && a.key == b.key; + a.chaincode == b.chaincode && a.key == b.key; } void Encode(unsigned char code[74]) const; @@ -173,7 +171,13 @@ struct CExtKey { void SetMaster(const unsigned char* seed, unsigned int nSeedLen); }; -/** Check that required EC support is available at runtime */ +/** Initialize the elliptic curve support. May not be called twice without calling ECC_Stop first. */ +void ECC_Start(void); + +/** Deinitialize the elliptic curve support. No-op if ECC_Start wasn't called first. */ +void ECC_Stop(void); + +/** Check that required EC support is available at runtime. */ bool ECC_InitSanityCheck(void); #endif // BITCOIN_KEY_H diff --git a/src/main.cpp b/src/main.cpp index 7bd2bdd447..5fd9c62486 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -57,7 +57,7 @@ bool fPruneMode = false; bool fIsBareMultisigStd = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = true; -unsigned int nCoinCacheSize = 5000; +size_t nCoinCacheUsage = 5000 * 300; uint64_t nPruneTarget = 0; /** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ @@ -986,7 +986,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa CAmount nFees = nValueIn-nValueOut; double dPriority = view.GetPriority(tx, chainActive.Height()); - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height()); + CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), mempool.HasNoInputsOf(tx)); unsigned int nSize = entry.GetTxSize(); // Don't accept it if it can't get into a block @@ -1052,7 +1052,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa } // Store transaction in memory - pool.addUnchecked(hash, entry); + pool.addUnchecked(hash, entry, !IsInitialBlockDownload()); } SyncWithWallets(tx, NULL); @@ -1879,6 +1879,8 @@ enum FlushStateMode { bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { LOCK2(cs_main, cs_LastBlockFile); static int64_t nLastWrite = 0; + static int64_t nLastFlush = 0; + static int64_t nLastSetChain = 0; std::set<int> setFilesToPrune; bool fFlushForPrune = false; try { @@ -1892,16 +1894,32 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { } } } - if ((mode == FLUSH_STATE_ALWAYS) || - ((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) || - (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000) || - fFlushForPrune) { - // Typical CCoins structures on disk are around 100 bytes in size. - // Pushing a new one to the database can cause it to be written - // twice (once in the log, and once in the tables). This is already - // an overestimation, as most will delete an existing entry or - // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(100 * 2 * 2 * pcoinsTip->GetCacheSize())) + int64_t nNow = GetTimeMicros(); + // Avoid writing/flushing immediately after startup. + if (nLastWrite == 0) { + nLastWrite = nNow; + } + if (nLastFlush == 0) { + nLastFlush = nNow; + } + if (nLastSetChain == 0) { + nLastSetChain = nNow; + } + size_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + // The cache is large and close to the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize * (10.0/9) > nCoinCacheUsage; + // The cache is over the limit, we have to write now. + bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nCoinCacheUsage; + // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. + bool fPeriodicWrite = mode == FLUSH_STATE_PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; + // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. + bool fPeriodicFlush = mode == FLUSH_STATE_PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; + // Combine all conditions that result in a full cache flush. + bool fDoFullFlush = (mode == FLUSH_STATE_ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; + // Write blocks and block index to disk. + if (fDoFullFlush || fPeriodicWrite) { + // Depend on nMinDiskSpace to ensure we can write block index + if (!CheckDiskSpace(0)) return state.Error("out of disk space"); // First make sure all block and undo data is flushed to disk. FlushBlockFile(); @@ -1923,21 +1941,31 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { return state.Abort("Files to write to block index database"); } } - // Flush the chainstate (which may refer to block index entries). - if (!pcoinsTip->Flush()) - return state.Abort("Failed to write to coin database"); - // Finally remove any pruned files if (fFlushForPrune) { UnlinkPrunedFiles(setFilesToPrune); fCheckForPruning = false; } - + nLastWrite = nNow; + } + // Flush best chain related state. This can only be done if the blocks / block index write was also done. + if (fDoFullFlush) { + // Typical CCoins structures on disk are around 128 bytes in size. + // Pushing a new one to the database can cause it to be written + // twice (once in the log, and once in the tables). This is already + // an overestimation, as most will delete an existing entry or + // overwrite one. Still, use a conservative safety factor of 2. + if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + return state.Error("out of disk space"); + // Flush the chainstate (which may refer to block index entries). + if (!pcoinsTip->Flush()) + return state.Abort("Failed to write to coin database"); + nLastFlush = nNow; + } + if ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000) { // Update best block in wallet (so we can detect restored wallets). - if (mode != FLUSH_STATE_IF_NEEDED) { - GetMainSignals().SetBestChain(chainActive.GetLocator()); - } - nLastWrite = GetTimeMicros(); + GetMainSignals().SetBestChain(chainActive.GetLocator()); + nLastSetChain = nNow; } } catch (const std::runtime_error& e) { return state.Abort(std::string("System error while flushing: ") + e.what()); @@ -1965,10 +1993,10 @@ void static UpdateTip(CBlockIndex *pindexNew) { nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); - LogPrintf("%s: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f cache=%u\n", __func__, + LogPrintf("%s: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f cache=%.1fMiB(%utx)\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), (unsigned int)pcoinsTip->GetCacheSize()); + Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); cvBlockChange.notify_all(); @@ -2086,7 +2114,7 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock * LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool. list<CTransaction> txConflicted; - mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted); + mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); mempool.check(pcoinsTip); // Update chainActive & related variables. UpdateTip(pindexNew); @@ -2927,7 +2955,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) return; } - unsigned int nLastBlockWeMustKeep = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + unsigned int nLastBlockWeCanPrune = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files // So we should leave a buffer under our target to account for another allocation @@ -2947,7 +2975,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) break; // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip - if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeMustKeep) + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) break; PruneOneBlockFile(fileNumber); @@ -2958,10 +2986,10 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune) } } - LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB min_must_keep=%d removed %d blk/rev pairs\n", + LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", nPruneTarget/1024/1024, nCurrentUsage/1024/1024, ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, - nLastBlockWeMustKeep, count); + nLastBlockWeCanPrune, count); } bool CheckDiskSpace(uint64_t nAdditionalBytes) @@ -3192,7 +3220,7 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && pindex == pindexState && (coins.GetCacheSize() + pcoinsTip->GetCacheSize()) <= nCoinCacheSize) { + if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { bool fClean = true; if (!DisconnectBlock(block, state, pindex, coins, &fClean)) return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); diff --git a/src/main.h b/src/main.h index 90b809e85e..fcbc4075d2 100644 --- a/src/main.h +++ b/src/main.h @@ -82,8 +82,10 @@ static const unsigned int MAX_HEADERS_RESULTS = 2000; * degree of disordering of blocks on disk (which make reindexing and in the future perhaps pruning * harder). We'll probably want to make this a per-peer adaptive value at some point. */ static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; -/** Time to wait (in seconds) between writing blockchain state to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 3600; +/** Time to wait (in seconds) between writing blocks/block index to disk. */ +static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; +/** Time to wait (in seconds) between flushing chainstate to disk. */ +static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; /** Maximum length of reject messages. */ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; @@ -119,7 +121,7 @@ extern bool fTxIndex; extern bool fIsBareMultisigStd; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; -extern unsigned int nCoinCacheSize; +extern size_t nCoinCacheUsage; extern CFeeRate minRelayTxFee; /** Best header we've seen so far (used for getheaders queries' starting points). */ diff --git a/src/memusage.h b/src/memusage.h new file mode 100644 index 0000000000..9f7de9e2e1 --- /dev/null +++ b/src/memusage.h @@ -0,0 +1,111 @@ +// Copyright (c) 2015 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_MEMUSAGE_H +#define BITCOIN_MEMUSAGE_H + +#include <stdlib.h> + +#include <map> +#include <set> +#include <vector> + +#include <boost/unordered_set.hpp> +#include <boost/unordered_map.hpp> + +namespace memusage +{ + +/** Compute the total memory used by allocating alloc bytes. */ +static size_t MallocUsage(size_t alloc); + +/** Compute the memory used for dynamically allocated but owned data structures. + * For generic data types, this is *not* recursive. DynamicUsage(vector<vector<int> >) + * will compute the memory used for the vector<int>'s, but not for the ints inside. + * This is for efficiency reasons, as these functions are intended to be fast. If + * application data structures require more accurate inner accounting, they should + * do the recursion themselves, or use more efficient caching + updating on modification. + */ +template<typename X> static size_t DynamicUsage(const std::vector<X>& v); +template<typename X> static size_t DynamicUsage(const std::set<X>& s); +template<typename X, typename Y> static size_t DynamicUsage(const std::map<X, Y>& m); +template<typename X, typename Y> static size_t DynamicUsage(const boost::unordered_set<X, Y>& s); +template<typename X, typename Y, typename Z> static size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& s); +template<typename X> static size_t DynamicUsage(const X& x); + +static inline size_t MallocUsage(size_t alloc) +{ + // Measured on libc6 2.19 on Linux. + if (sizeof(void*) == 8) { + return ((alloc + 31) >> 4) << 4; + } else if (sizeof(void*) == 4) { + return ((alloc + 15) >> 3) << 3; + } else { + assert(0); + } +} + +// STL data structures + +template<typename X> +struct stl_tree_node +{ +private: + int color; + void* parent; + void* left; + void* right; + X x; +}; + +template<typename X> +static inline size_t DynamicUsage(const std::vector<X>& v) +{ + return MallocUsage(v.capacity() * sizeof(X)); +} + +template<typename X> +static inline size_t DynamicUsage(const std::set<X>& s) +{ + return MallocUsage(sizeof(stl_tree_node<X>)) * s.size(); +} + +template<typename X, typename Y> +static inline size_t DynamicUsage(const std::map<X, Y>& m) +{ + return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size(); +} + +// Boost data structures + +template<typename X> +struct boost_unordered_node : private X +{ +private: + void* ptr; +}; + +template<typename X, typename Y> +static inline size_t DynamicUsage(const boost::unordered_set<X, Y>& s) +{ + return MallocUsage(sizeof(boost_unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); +} + +template<typename X, typename Y, typename Z> +static inline size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& m) +{ + return MallocUsage(sizeof(boost_unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); +} + +// Dispatch to class method as fallback + +template<typename X> +static inline size_t DynamicUsage(const X& x) +{ + return x.DynamicMemoryUsage(); +} + +} + +#endif diff --git a/src/miner.cpp b/src/miner.cpp index 56a2c5828b..4bceb7d7b4 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -453,8 +453,16 @@ void static BitcoinMiner(CWallet *pwallet) if (chainparams.MiningRequiresPeers()) { // Busy-wait for the network to come online so we don't waste time mining // on an obsolete chain. In regtest mode we expect to fly solo. - while (vNodes.empty()) + do { + bool fvNodesEmpty; + { + LOCK(cs_vNodes); + fvNodesEmpty = vNodes.empty(); + } + if (!fvNodesEmpty && !IsInitialBlockDownload()) + break; MilliSleep(1000); + } while (true); } // @@ -533,6 +541,11 @@ void static BitcoinMiner(CWallet *pwallet) LogPrintf("BitcoinMiner terminated\n"); throw; } + catch (const std::runtime_error &e) + { + LogPrintf("BitcoinMiner runtime error: %s\n", e.what()); + return; + } } void GenerateBitcoins(bool fGenerate, CWallet* pwallet, int nThreads) diff --git a/src/net.cpp b/src/net.cpp index 2de04fc574..6849d79263 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -13,6 +13,7 @@ #include "chainparams.h" #include "clientversion.h" #include "primitives/transaction.h" +#include "scheduler.h" #include "ui_interface.h" #include "crypto/common.h" @@ -1590,7 +1591,7 @@ void static Discover(boost::thread_group& threadGroup) #endif } -void StartNode(boost::thread_group& threadGroup) +void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler) { uiInterface.InitMessage(_("Loading addresses...")); // Load addresses for peers.dat @@ -1640,7 +1641,7 @@ void StartNode(boost::thread_group& threadGroup) threadGroup.create_thread(boost::bind(&TraceThread<void (*)()>, "msghand", &ThreadMessageHandler)); // Dump network addresses - threadGroup.create_thread(boost::bind(&LoopForever<void (*)()>, "dumpaddr", &DumpAddresses, DUMP_ADDRESSES_INTERVAL * 1000)); + scheduler.scheduleEvery(&DumpAddresses, DUMP_ADDRESSES_INTERVAL); } bool StopNode() @@ -32,6 +32,7 @@ class CAddrMan; class CBlockIndex; +class CScheduler; class CNode; namespace boost { @@ -72,7 +73,7 @@ bool OpenNetworkConnection(const CAddress& addrConnect, CSemaphoreGrant *grantOu void MapPort(bool fUseUPnP); unsigned short GetListenPort(); bool BindListenPort(const CService &bindAddr, std::string& strError, bool fWhitelisted = false); -void StartNode(boost::thread_group& threadGroup); +void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler); bool StopNode(); void SocketSendData(CNode *pnode); diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp new file mode 100644 index 0000000000..b1491cec01 --- /dev/null +++ b/src/policy/fees.cpp @@ -0,0 +1,529 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "policy/fees.h" + +#include "amount.h" +#include "primitives/transaction.h" +#include "streams.h" +#include "txmempool.h" +#include "util.h" + +void TxConfirmStats::Initialize(std::vector<double>& defaultBuckets, + unsigned int maxConfirms, double _decay, std::string _dataTypeString) +{ + decay = _decay; + dataTypeString = _dataTypeString; + for (unsigned int i = 0; i < defaultBuckets.size(); i++) { + buckets.push_back(defaultBuckets[i]); + bucketMap[defaultBuckets[i]] = i; + } + confAvg.resize(maxConfirms); + curBlockConf.resize(maxConfirms); + unconfTxs.resize(maxConfirms); + for (unsigned int i = 0; i < maxConfirms; i++) { + confAvg[i].resize(buckets.size()); + curBlockConf[i].resize(buckets.size()); + unconfTxs[i].resize(buckets.size()); + } + + oldUnconfTxs.resize(buckets.size()); + curBlockTxCt.resize(buckets.size()); + txCtAvg.resize(buckets.size()); + curBlockVal.resize(buckets.size()); + avg.resize(buckets.size()); +} + +// Zero out the data for the current block +void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) +{ + for (unsigned int j = 0; j < buckets.size(); j++) { + oldUnconfTxs[j] += unconfTxs[nBlockHeight%unconfTxs.size()][j]; + unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0; + for (unsigned int i = 0; i < curBlockConf.size(); i++) + curBlockConf[i][j] = 0; + curBlockTxCt[j] = 0; + curBlockVal[j] = 0; + } +} + + +void TxConfirmStats::Record(int blocksToConfirm, double val) +{ + // blocksToConfirm is 1-based + if (blocksToConfirm < 1) + return; + unsigned int bucketindex = bucketMap.lower_bound(val)->second; + for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { + curBlockConf[i - 1][bucketindex]++; + } + curBlockTxCt[bucketindex]++; + curBlockVal[bucketindex] += val; +} + +void TxConfirmStats::UpdateMovingAverages() +{ + for (unsigned int j = 0; j < buckets.size(); j++) { + for (unsigned int i = 0; i < confAvg.size(); i++) + confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; + avg[j] = avg[j] * decay + curBlockVal[j]; + txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; + } +} + +// returns -1 on error conditions +double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, + double successBreakPoint, bool requireGreater, + unsigned int nBlockHeight) +{ + // Counters for a bucket (or range of buckets) + double nConf = 0; // Number of tx's confirmed within the confTarget + double totalNum = 0; // Total number of tx's that were ever confirmed + int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + + int maxbucketindex = buckets.size() - 1; + + // requireGreater means we are looking for the lowest fee/priority such that all higher + // values pass, so we start at maxbucketindex (highest fee) and look at succesively + // smaller buckets until we reach failure. Otherwise, we are looking for the highest + // fee/priority such that all lower values fail, and we go in the opposite direction. + unsigned int startbucket = requireGreater ? maxbucketindex : 0; + int step = requireGreater ? -1 : 1; + + // We'll combine buckets until we have enough samples. + // The near and far variables will define the range we've combined + // The best variables are the last range we saw which still had a high + // enough confirmation rate to count as success. + // The cur variables are the current range we're counting. + unsigned int curNearBucket = startbucket; + unsigned int bestNearBucket = startbucket; + unsigned int curFarBucket = startbucket; + unsigned int bestFarBucket = startbucket; + + bool foundAnswer = false; + unsigned int bins = unconfTxs.size(); + + // Start counting from highest(default) or lowest fee/pri transactions + for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + curFarBucket = bucket; + nConf += confAvg[confTarget - 1][bucket]; + totalNum += txCtAvg[bucket]; + for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) + extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; + extraNum += oldUnconfTxs[bucket]; + // If we have enough transaction data points in this range of buckets, + // we can test for success + // (Only count the confirmed data points, so that each confirmation count + // will be looking at the same amount of data and same bucket breaks) + if (totalNum >= sufficientTxVal / (1 - decay)) { + double curPct = nConf / (totalNum + extraNum); + + // Check to see if we are no longer getting confirmed at the success rate + if (requireGreater && curPct < successBreakPoint) + break; + if (!requireGreater && curPct > successBreakPoint) + break; + + // Otherwise update the cumulative stats, and the bucket variables + // and reset the counters + else { + foundAnswer = true; + nConf = 0; + totalNum = 0; + extraNum = 0; + bestNearBucket = curNearBucket; + bestFarBucket = curFarBucket; + curNearBucket = bucket + step; + } + } + } + + double median = -1; + double txSum = 0; + + // Calculate the "average" fee of the best bucket range that met success conditions + // Find the bucket with the median transaction and then report the average fee from that bucket + // This is a compromise between finding the median which we can't since we don't save all tx's + // and reporting the average which is less accurate + unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; + unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + for (unsigned int j = minBucket; j <= maxBucket; j++) { + txSum += txCtAvg[j]; + } + if (foundAnswer && txSum != 0) { + txSum = txSum / 2; + for (unsigned int j = minBucket; j <= maxBucket; j++) { + if (txCtAvg[j] < txSum) + txSum -= txCtAvg[j]; + else { // we're in the right bucket + median = avg[j] / txCtAvg[j]; + break; + } + } + } + + LogPrint("estimatefee", "%3d: For conf success %s %4.2f need %s %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", + confTarget, requireGreater ? ">" : "<", successBreakPoint, dataTypeString, + requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], + 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); + + return median; +} + +void TxConfirmStats::Write(CAutoFile& fileout) +{ + fileout << decay; + fileout << buckets; + fileout << avg; + fileout << txCtAvg; + fileout << confAvg; +} + +void TxConfirmStats::Read(CAutoFile& filein) +{ + // Read data file into temporary variables and do some very basic sanity checking + std::vector<double> fileBuckets; + std::vector<double> fileAvg; + std::vector<std::vector<double> > fileConfAvg; + std::vector<double> fileTxCtAvg; + double fileDecay; + size_t maxConfirms; + size_t numBuckets; + + filein >> fileDecay; + if (fileDecay <= 0 || fileDecay >= 1) + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + filein >> fileBuckets; + numBuckets = fileBuckets.size(); + if (numBuckets <= 1 || numBuckets > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 fee/pri buckets"); + filein >> fileAvg; + if (fileAvg.size() != numBuckets) + throw std::runtime_error("Corrupt estimates file. Mismatch in fee/pri average bucket count"); + filein >> fileTxCtAvg; + if (fileTxCtAvg.size() != numBuckets) + throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); + filein >> fileConfAvg; + maxConfirms = fileConfAvg.size(); + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) // one week + throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + for (unsigned int i = 0; i < maxConfirms; i++) { + if (fileConfAvg[i].size() != numBuckets) + throw std::runtime_error("Corrupt estimates file. Mismatch in fee/pri conf average bucket count"); + } + // Now that we've processed the entire fee estimate data file and not + // thrown any errors, we can copy it to our data structures + decay = fileDecay; + buckets = fileBuckets; + avg = fileAvg; + confAvg = fileConfAvg; + txCtAvg = fileTxCtAvg; + bucketMap.clear(); + + // Resize the current block variables which aren't stored in the data file + // to match the number of confirms and buckets + curBlockConf.resize(maxConfirms); + for (unsigned int i = 0; i < maxConfirms; i++) { + curBlockConf[i].resize(buckets.size()); + } + curBlockTxCt.resize(buckets.size()); + curBlockVal.resize(buckets.size()); + + unconfTxs.resize(maxConfirms); + for (unsigned int i = 0; i < maxConfirms; i++) { + unconfTxs[i].resize(buckets.size()); + } + oldUnconfTxs.resize(buckets.size()); + + for (unsigned int i = 0; i < buckets.size(); i++) + bucketMap[buckets[i]] = i; + + LogPrint("estimatefee", "Reading estimates: %u %s buckets counting confirms up to %u blocks\n", + numBuckets, dataTypeString, maxConfirms); +} + +unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) +{ + unsigned int bucketindex = bucketMap.lower_bound(val)->second; + unsigned int blockIndex = nBlockHeight % unconfTxs.size(); + unconfTxs[blockIndex][bucketindex]++; + LogPrint("estimatefee", "adding to %s\n", dataTypeString); + return bucketindex; +} + +void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) +{ + //nBestSeenHeight is not updated yet for the new block + int blocksAgo = nBestSeenHeight - entryHeight; + if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet + blocksAgo = 0; + if (blocksAgo < 0) { + LogPrint("estimatefee", "Blockpolicy error, blocks ago is negative for mempool tx\n"); + return; //This can't happen becasue we call this with our best seen height, no entries can have higher + } + + if (blocksAgo >= (int)unconfTxs.size()) { + if (oldUnconfTxs[bucketindex] > 0) + oldUnconfTxs[bucketindex]--; + else + LogPrint("estimatefee", "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n", + bucketindex); + } + else { + unsigned int blockIndex = entryHeight % unconfTxs.size(); + if (unconfTxs[blockIndex][bucketindex] > 0) + unconfTxs[blockIndex][bucketindex]--; + else + LogPrint("estimatefee", "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n", + blockIndex, bucketindex); + } +} + +void CBlockPolicyEstimator::removeTx(uint256 hash) +{ + std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash); + if (pos == mapMemPoolTxs.end()) { + LogPrint("estimatefee", "Blockpolicy error mempool tx %s not found for removeTx\n", + hash.ToString().c_str()); + return; + } + TxConfirmStats *stats = pos->second.stats; + unsigned int entryHeight = pos->second.blockHeight; + unsigned int bucketIndex = pos->second.bucketIndex; + + if (stats != NULL) + stats->removeTx(entryHeight, nBestSeenHeight, bucketIndex); + mapMemPoolTxs.erase(hash); +} + +CBlockPolicyEstimator::CBlockPolicyEstimator(const CFeeRate& _minRelayFee) + : nBestSeenHeight(0) +{ + minTrackedFee = _minRelayFee < CFeeRate(MIN_FEERATE) ? CFeeRate(MIN_FEERATE) : _minRelayFee; + std::vector<double> vfeelist; + for (double bucketBoundary = minTrackedFee.GetFeePerK(); bucketBoundary <= MAX_FEERATE; bucketBoundary *= FEE_SPACING) { + vfeelist.push_back(bucketBoundary); + } + vfeelist.push_back(INF_FEERATE); + feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, "FeeRate"); + + minTrackedPriority = AllowFreeThreshold() < MIN_PRIORITY ? MIN_PRIORITY : AllowFreeThreshold(); + std::vector<double> vprilist; + for (double bucketBoundary = minTrackedPriority; bucketBoundary <= MAX_PRIORITY; bucketBoundary *= PRI_SPACING) { + vprilist.push_back(bucketBoundary); + } + vprilist.push_back(INF_PRIORITY); + priStats.Initialize(vprilist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, "Priority"); + + feeUnlikely = CFeeRate(0); + feeLikely = CFeeRate(INF_FEERATE); + priUnlikely = 0; + priLikely = INF_PRIORITY; +} + +bool CBlockPolicyEstimator::isFeeDataPoint(const CFeeRate &fee, double pri) +{ + if ((pri < minTrackedPriority && fee >= minTrackedFee) || + (pri < priUnlikely && fee > feeLikely)) { + return true; + } + return false; +} + +bool CBlockPolicyEstimator::isPriDataPoint(const CFeeRate &fee, double pri) +{ + if ((fee < minTrackedFee && pri >= minTrackedPriority) || + (fee < feeUnlikely && pri > priLikely)) { + return true; + } + return false; +} + +void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool fCurrentEstimate) +{ + unsigned int txHeight = entry.GetHeight(); + uint256 hash = entry.GetTx().GetHash(); + if (mapMemPoolTxs[hash].stats != NULL) { + LogPrint("estimatefee", "Blockpolicy error mempool tx %s already being tracked\n", + hash.ToString().c_str()); + return; + } + + if (txHeight < nBestSeenHeight) { + // Ignore side chains and re-orgs; assuming they are random they don't + // affect the estimate. We'll potentially double count transactions in 1-block reorgs. + return; + } + + // Only want to be updating estimates when our blockchain is synced, + // otherwise we'll miscalculate how many blocks its taking to get included. + if (!fCurrentEstimate) + return; + + if (!entry.WasClearAtEntry()) { + // This transaction depends on other transactions in the mempool to + // be included in a block before it will be able to be included, so + // we shouldn't include it in our calculations + return; + } + + // Fees are stored and reported as BTC-per-kb: + CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); + + // Want the priority of the tx at confirmation. However we don't know + // what that will be and its too hard to continue updating it + // so use starting priority as a proxy + double curPri = entry.GetPriority(txHeight); + mapMemPoolTxs[hash].blockHeight = txHeight; + + LogPrint("estimatefee", "Blockpolicy mempool tx %s ", hash.ToString().substr(0,10)); + // Record this as a priority estimate + if (entry.GetFee() == 0 || isPriDataPoint(feeRate, curPri)) { + mapMemPoolTxs[hash].stats = &priStats; + mapMemPoolTxs[hash].bucketIndex = priStats.NewTx(txHeight, curPri); + } + // Record this as a fee estimate + else if (isFeeDataPoint(feeRate, curPri)) { + mapMemPoolTxs[hash].stats = &feeStats; + mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); + } + else { + LogPrint("estimatefee", "not adding\n"); + } +} + +void CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry& entry) +{ + if (!entry.WasClearAtEntry()) { + // This transaction depended on other transactions in the mempool to + // be included in a block before it was able to be included, so + // we shouldn't include it in our calculations + return; + } + + // How many blocks did it take for miners to include this transaction? + // blocksToConfirm is 1-based, so a transaction included in the earliest + // possible block has confirmation count of 1 + int blocksToConfirm = nBlockHeight - entry.GetHeight(); + if (blocksToConfirm <= 0) { + // This can't happen because we don't process transactions from a block with a height + // lower than our greatest seen height + LogPrint("estimatefee", "Blockpolicy error Transaction had negative blocksToConfirm\n"); + return; + } + + // Fees are stored and reported as BTC-per-kb: + CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); + + // Want the priority of the tx at confirmation. The priority when it + // entered the mempool could easily be very small and change quickly + double curPri = entry.GetPriority(nBlockHeight); + + // Record this as a priority estimate + if (entry.GetFee() == 0 || isPriDataPoint(feeRate, curPri)) { + priStats.Record(blocksToConfirm, curPri); + } + // Record this as a fee estimate + else if (isFeeDataPoint(feeRate, curPri)) { + feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + } +} + +void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, + std::vector<CTxMemPoolEntry>& entries, bool fCurrentEstimate) +{ + if (nBlockHeight <= nBestSeenHeight) { + // Ignore side chains and re-orgs; assuming they are random + // they don't affect the estimate. + // And if an attacker can re-org the chain at will, then + // you've got much bigger problems than "attacker can influence + // transaction fees." + return; + } + nBestSeenHeight = nBlockHeight; + + // Only want to be updating estimates when our blockchain is synced, + // otherwise we'll miscalculate how many blocks its taking to get included. + if (!fCurrentEstimate) + return; + + // Update the dynamic cutoffs + // a fee/priority is "likely" the reason your tx was included in a block if >85% of such tx's + // were confirmed in 2 blocks and is "unlikely" if <50% were confirmed in 10 blocks + LogPrint("estimatefee", "Blockpolicy recalculating dynamic cutoffs:\n"); + priLikely = priStats.EstimateMedianVal(2, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBlockHeight); + if (priLikely == -1) + priLikely = INF_PRIORITY; + + double feeLikelyEst = feeStats.EstimateMedianVal(2, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBlockHeight); + if (feeLikelyEst == -1) + feeLikely = CFeeRate(INF_FEERATE); + else + feeLikely = CFeeRate(feeLikelyEst); + + priUnlikely = priStats.EstimateMedianVal(10, SUFFICIENT_PRITXS, UNLIKELY_PCT, false, nBlockHeight); + if (priUnlikely == -1) + priUnlikely = 0; + + double feeUnlikelyEst = feeStats.EstimateMedianVal(10, SUFFICIENT_FEETXS, UNLIKELY_PCT, false, nBlockHeight); + if (feeUnlikelyEst == -1) + feeUnlikely = CFeeRate(0); + else + feeUnlikely = CFeeRate(feeUnlikelyEst); + + // Clear the current block states + feeStats.ClearCurrent(nBlockHeight); + priStats.ClearCurrent(nBlockHeight); + + // Repopulate the current block states + for (unsigned int i = 0; i < entries.size(); i++) + processBlockTx(nBlockHeight, entries[i]); + + // Update all exponential averages with the current block states + feeStats.UpdateMovingAverages(); + priStats.UpdateMovingAverages(); + + LogPrint("estimatefee", "Blockpolicy after updating estimates for %u confirmed entries, new mempool map size %u\n", + entries.size(), mapMemPoolTxs.size()); +} + +CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) +{ + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) + return CFeeRate(0); + + double median = feeStats.EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + + if (median < 0) + return CFeeRate(0); + + return CFeeRate(median); +} + +double CBlockPolicyEstimator::estimatePriority(int confTarget) +{ + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > priStats.GetMaxConfirms()) + return -1; + + return priStats.EstimateMedianVal(confTarget, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); +} + +void CBlockPolicyEstimator::Write(CAutoFile& fileout) +{ + fileout << nBestSeenHeight; + feeStats.Write(fileout); + priStats.Write(fileout); +} + +void CBlockPolicyEstimator::Read(CAutoFile& filein) +{ + int nFileBestSeenHeight; + filein >> nFileBestSeenHeight; + feeStats.Read(filein); + priStats.Read(filein); + nBestSeenHeight = nFileBestSeenHeight; +} diff --git a/src/policy/fees.h b/src/policy/fees.h new file mode 100644 index 0000000000..ce4d782566 --- /dev/null +++ b/src/policy/fees.h @@ -0,0 +1,276 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef BITCOIN_POLICYESTIMATOR_H +#define BITCOIN_POLICYESTIMATOR_H + +#include "amount.h" +#include "uint256.h" + +#include <map> +#include <string> +#include <vector> + +class CAutoFile; +class CFeeRate; +class CTxMemPoolEntry; + +/** \class CBlockPolicyEstimator + * The BlockPolicyEstimator is used for estimating the fee or priority needed + * for a transaction to be included in a block within a certain number of + * blocks. + * + * At a high level the algorithm works by grouping transactions into buckets + * based on having similar priorities or fees and then tracking how long it + * takes transactions in the various buckets to be mined. It operates under + * the assumption that in general transactions of higher fee/priority will be + * included in blocks before transactions of lower fee/priority. So for + * example if you wanted to know what fee you should put on a transaction to + * be included in a block within the next 5 blocks, you would start by looking + * at the bucket with with the highest fee transactions and verifying that a + * sufficiently high percentage of them were confirmed within 5 blocks and + * then you would look at the next highest fee bucket, and so on, stopping at + * the last bucket to pass the test. The average fee of transactions in this + * bucket will give you an indication of the lowest fee you can put on a + * transaction and still have a sufficiently high chance of being confirmed + * within your desired 5 blocks. + * + * When a transaction enters the mempool or is included within a block we + * decide whether it can be used as a data point for fee estimation, priority + * estimation or neither. If the value of exactly one of those properties was + * below the required minimum it can be used to estimate the other. In + * addition, if a priori our estimation code would indicate that the + * transaction would be much more quickly included in a block because of one + * of the properties compared to the other, we can also decide to use it as + * an estimate for that property. + * + * Here is a brief description of the implementation for fee estimation. + * When a transaction that counts for fee estimation enters the mempool, we + * track the height of the block chain at entry. Whenever a block comes in, + * we count the number of transactions in each bucket and the total amount of fee + * paid in each bucket. Then we calculate how many blocks Y it took each + * transaction to be mined and we track an array of counters in each bucket + * for how long it to took transactions to get confirmed from 1 to a max of 25 + * and we increment all the counters from Y up to 25. This is because for any + * number Z>=Y the transaction was successfully mined within Z blocks. We + * want to save a history of this information, so at any time we have a + * counter of the total number of transactions that happened in a given fee + * bucket and the total number that were confirmed in each number 1-25 blocks + * or less for any bucket. We save this history by keeping an exponentially + * decaying moving average of each one of these stats. Furthermore we also + * keep track of the number unmined (in mempool) transactions in each bucket + * and for how many blocks they have been outstanding and use that to increase + * the number of transactions we've seen in that fee bucket when calculating + * an estimate for any number of confirmations below the number of blocks + * they've been outstanding. + */ + +/** + * We will instantiate two instances of this class, one to track transactions + * that were included in a block due to fee, and one for tx's included due to + * priority. We will lump transactions into a bucket according to their approximate + * fee or priority and then track how long it took for those txs to be included in a block + * + * The tracking of unconfirmed (mempool) transactions is completely independent of the + * historical tracking of transactions that have been confirmed in a block. + */ +class TxConfirmStats +{ +private: + //Define the buckets we will group transactions into (both fee buckets and priority buckets) + std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) + std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + // For each bucket X: + // Count the total # of txs in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> txCtAvg; + // and calcuate the total for the current block to update the moving average + std::vector<int> curBlockTxCt; + + // Count the total # of txs confirmed within Y blocks in each bucket + // Track the historical moving average of theses totals over blocks + std::vector<std::vector<double> > confAvg; // confAvg[Y][X] + // and calcuate the totals for the current block to update the moving averages + std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] + + // Sum the total priority/fee of all tx's in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> avg; + // and calculate the total for the current block to update the moving average + std::vector<double> curBlockVal; + + // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X + // Combine the total value with the tx counts to calculate the avg fee/priority per bucket + + std::string dataTypeString; + double decay; + + // Mempool counts of outstanding transactions + // For each bucket X, track the number of transactions in the mempool + // that are unconfirmed for each possible confirmation value Y + std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] + // transactions still unconfirmed after MAX_CONFIRMS for each bucket + std::vector<int> oldUnconfTxs; + +public: + /** + * Initialize the data structures. This is called by BlockPolicyEstimator's + * constructor with default values. + * @param defaultBuckets contains the upper limits for the bucket boundries + * @param maxConfirms max number of confirms to track + * @param decay how much to decay the historical moving average per block + * @param dataTypeString for logging purposes + */ + void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay, std::string dataTypeString); + + /** Clear the state of the curBlock variables to start counting for the new block */ + void ClearCurrent(unsigned int nBlockHeight); + + /** + * Record a new transaction data point in the current block stats + * @param blocksToConfirm the number of blocks it took this transaction to confirm + * @param val either the fee or the priority when entered of the transaction + * @warning blocksToConfirm is 1-based and has to be >= 1 + */ + void Record(int blocksToConfirm, double val); + + /** Record a new transaction entering the mempool*/ + unsigned int NewTx(unsigned int nBlockHeight, double val); + + /** Remove a transaction from mempool tracking stats*/ + void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, + unsigned int bucketIndex); + + /** Update our estimates by decaying our historical moving average and updating + with the data gathered from the current block */ + void UpdateMovingAverages(); + + /** + * Calculate a fee or priority estimate. Find the lowest value bucket (or range of buckets + * to make sure we have enough data points) whose transactions still have sufficient likelihood + * of being confirmed within the target number of confirmations + * @param confTarget target number of confirmations + * @param sufficientTxVal required average number of transactions per block in a bucket range + * @param minSuccess the success probability we require + * @param requireGreater return the lowest fee/pri such that all higher values pass minSuccess OR + * return the highest fee/pri such that all lower values fail minSuccess + * @param nBlockHeight the current block height + */ + double EstimateMedianVal(int confTarget, double sufficientTxVal, + double minSuccess, bool requireGreater, unsigned int nBlockHeight); + + /** Return the max number of confirms we're tracking */ + unsigned int GetMaxConfirms() { return confAvg.size(); } + + /** Write state of estimation data to a file*/ + void Write(CAutoFile& fileout); + + /** + * Read saved state of estimation data from a file and replace all internal data structures and + * variables with this state. + */ + void Read(CAutoFile& filein); +}; + + + +/** Track confirm delays up to 25 blocks, can't estimate beyond that */ +static const unsigned int MAX_BLOCK_CONFIRMS = 25; + +/** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ +static const double DEFAULT_DECAY = .998; + +/** Require greater than 85% of X fee transactions to be confirmed within Y blocks for X to be big enough */ +static const double MIN_SUCCESS_PCT = .85; +static const double UNLIKELY_PCT = .5; + +/** Require an avg of 1 tx in the combined fee bucket per block to have stat significance */ +static const double SUFFICIENT_FEETXS = 1; + +/** Require only an avg of 1 tx every 5 blocks in the combined pri bucket (way less pri txs) */ +static const double SUFFICIENT_PRITXS = .2; + +// Minimum and Maximum values for tracking fees and priorities +static const double MIN_FEERATE = 10; +static const double MAX_FEERATE = 1e7; +static const double INF_FEERATE = MAX_MONEY; +static const double MIN_PRIORITY = 10; +static const double MAX_PRIORITY = 1e16; +static const double INF_PRIORITY = 1e9 * MAX_MONEY; + +// We have to lump transactions into buckets based on fee or priority, but we want to be able +// to give accurate estimates over a large range of potential fees and priorities +// Therefore it makes sense to exponentially space the buckets +/** Spacing of FeeRate buckets */ +static const double FEE_SPACING = 1.1; + +/** Spacing of Priority buckets */ +static const double PRI_SPACING = 2; + +/** + * We want to be able to estimate fees or priorities that are needed on tx's to be included in + * a certain number of blocks. Every time a block is added to the best chain, this class records + * stats on the transactions included in that block + */ +class CBlockPolicyEstimator +{ +public: + /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ + CBlockPolicyEstimator(const CFeeRate& minRelayFee); + + /** Process all the transactions that have been included in a block */ + void processBlock(unsigned int nBlockHeight, + std::vector<CTxMemPoolEntry>& entries, bool fCurrentEstimate); + + /** Process a transaction confirmed in a block*/ + void processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry& entry); + + /** Process a transaction accepted to the mempool*/ + void processTransaction(const CTxMemPoolEntry& entry, bool fCurrentEstimate); + + /** Remove a transaction from the mempool tracking stats*/ + void removeTx(uint256 hash); + + /** Is this transaction likely included in a block because of its fee?*/ + bool isFeeDataPoint(const CFeeRate &fee, double pri); + + /** Is this transaction likely included in a block because of its priority?*/ + bool isPriDataPoint(const CFeeRate &fee, double pri); + + /** Return a fee estimate */ + CFeeRate estimateFee(int confTarget); + + /** Return a priority estimate */ + double estimatePriority(int confTarget); + + /** Write estimation data to a file */ + void Write(CAutoFile& fileout); + + /** Read estimation data from a file */ + void Read(CAutoFile& filein); + +private: + CFeeRate minTrackedFee; //! Passed to constructor to avoid dependency on main + double minTrackedPriority; //! Set to AllowFreeThreshold + unsigned int nBestSeenHeight; + struct TxStatsInfo + { + TxConfirmStats *stats; + unsigned int blockHeight; + unsigned int bucketIndex; + TxStatsInfo() : stats(NULL), blockHeight(0), bucketIndex(0) {} + }; + + // map of txids to information about that transaction + std::map<uint256, TxStatsInfo> mapMemPoolTxs; + + /** Classes to track historical data on transaction confirmations */ + TxConfirmStats feeStats, priStats; + + /** Breakpoints to help determine whether a transaction was confirmed by priority or Fee */ + CFeeRate feeLikely, feeUnlikely; + double priLikely, priUnlikely; +}; +#endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/pubkey.cpp b/src/pubkey.cpp index a4c046bff5..bdab137600 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -54,13 +54,13 @@ bool CPubKey::Decompress() { return true; } -bool CPubKey::Derive(CPubKey& pubkeyChild, unsigned char ccChild[32], unsigned int nChild, const unsigned char cc[32]) const { +bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert((nChild >> 31) == 0); assert(begin() + 33 == end()); unsigned char out[64]; BIP32Hash(cc, nChild, *begin(), begin()+1, out); - memcpy(ccChild, out+32, 32); + memcpy(ccChild.begin(), out+32, 32); CECKey key; bool ret = key.SetPubKey(begin(), size()); ret &= key.TweakPublic(out); @@ -75,7 +75,7 @@ void CExtPubKey::Encode(unsigned char code[74]) const { memcpy(code+1, vchFingerprint, 4); code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; - memcpy(code+9, vchChainCode, 32); + memcpy(code+9, chaincode.begin(), 32); assert(pubkey.size() == 33); memcpy(code+41, pubkey.begin(), 33); } @@ -84,7 +84,7 @@ void CExtPubKey::Decode(const unsigned char code[74]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; - memcpy(vchChainCode, code+9, 32); + memcpy(chaincode.begin(), code+9, 32); pubkey.Set(code+41, code+74); } @@ -93,5 +93,5 @@ bool CExtPubKey::Derive(CExtPubKey &out, unsigned int nChild) const { CKeyID id = pubkey.GetID(); memcpy(&out.vchFingerprint[0], &id, 4); out.nChild = nChild; - return pubkey.Derive(out.pubkey, out.vchChainCode, nChild, vchChainCode); + return pubkey.Derive(out.pubkey, out.chaincode, nChild, chaincode); } diff --git a/src/pubkey.h b/src/pubkey.h index b0768d4f47..cce9c826e5 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -31,6 +31,8 @@ public: CKeyID(const uint160& in) : uint160(in) {} }; +typedef uint256 ChainCode; + /** An encapsulated public key. */ class CPubKey { @@ -182,20 +184,20 @@ public: bool Decompress(); //! Derive BIP32 child pubkey. - bool Derive(CPubKey& pubkeyChild, unsigned char ccChild[32], unsigned int nChild, const unsigned char cc[32]) const; + bool Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const; }; struct CExtPubKey { unsigned char nDepth; unsigned char vchFingerprint[4]; unsigned int nChild; - unsigned char vchChainCode[32]; + ChainCode chaincode; CPubKey pubkey; - friend bool operator==(const CExtPubKey& a, const CExtPubKey& b) + friend bool operator==(const CExtPubKey &a, const CExtPubKey &b) { return a.nDepth == b.nDepth && memcmp(&a.vchFingerprint[0], &b.vchFingerprint[0], 4) == 0 && a.nChild == b.nChild && - memcmp(&a.vchChainCode[0], &b.vchChainCode[0], 32) == 0 && a.pubkey == b.pubkey; + a.chaincode == b.chaincode && a.pubkey == b.pubkey; } void Encode(unsigned char code[74]) const; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 018169cfdc..8740b98b70 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -26,6 +26,7 @@ #include "init.h" #include "main.h" #include "rpcserver.h" +#include "scheduler.h" #include "ui_interface.h" #include "util.h" @@ -178,6 +179,7 @@ signals: private: boost::thread_group threadGroup; + CScheduler scheduler; /// Pass fatal exception message to UI thread void handleRunawayException(const std::exception *e); @@ -258,7 +260,7 @@ void BitcoinCore::initialize() try { qDebug() << __func__ << ": Running AppInit2 in thread"; - int rv = AppInit2(threadGroup); + int rv = AppInit2(threadGroup, scheduler); if(rv) { /* Start a dummy RPC thread if no RPC thread is active yet diff --git a/src/qt/bitcoin.qrc b/src/qt/bitcoin.qrc index 63af146fd0..c899e95506 100644 --- a/src/qt/bitcoin.qrc +++ b/src/qt/bitcoin.qrc @@ -45,6 +45,7 @@ <file alias="about">res/icons/about.png</file> <file alias="about_qt">res/icons/about_qt.png</file> <file alias="verify">res/icons/verify.png</file> + <file alias="warning">res/icons/warning.png</file> </qresource> <qresource prefix="/movies"> <file alias="spinner-000">res/movies/spinner-000.png</file> diff --git a/src/qt/forms/overviewpage.ui b/src/qt/forms/overviewpage.ui index 53d416ef38..6d792d1475 100644 --- a/src/qt/forms/overviewpage.ui +++ b/src/qt/forms/overviewpage.ui @@ -59,21 +59,35 @@ </widget> </item> <item> - <widget class="QLabel" name="labelWalletStatus"> - <property name="cursor"> - <cursorShape>WhatsThisCursor</cursorShape> + <widget class="QPushButton" name="labelWalletStatus"> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="maximumSize"> + <size> + <width>30</width> + <height>16777215</height> + </size> </property> <property name="toolTip"> <string>The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet.</string> </property> - <property name="styleSheet"> - <string notr="true">QLabel { color: red; }</string> - </property> <property name="text"> - <string notr="true">(out of sync)</string> + <string/> </property> - <property name="alignment"> - <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set> + <property name="icon"> + <iconset resource="../bitcoin.qrc"> + <normaloff>:/icons/warning</normaloff> + <disabledoff>:/icons/warning</disabledoff>:/icons/warning</iconset> + </property> + <property name="iconSize"> + <size> + <width>24</width> + <height>24</height> + </size> + </property> + <property name="flat"> + <bool>true</bool> </property> </widget> </item> @@ -431,21 +445,35 @@ </widget> </item> <item> - <widget class="QLabel" name="labelTransactionsStatus"> - <property name="cursor"> - <cursorShape>WhatsThisCursor</cursorShape> + <widget class="QPushButton" name="labelTransactionsStatus"> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="maximumSize"> + <size> + <width>30</width> + <height>16777215</height> + </size> </property> <property name="toolTip"> <string>The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet.</string> </property> - <property name="styleSheet"> - <string notr="true">QLabel { color: red; }</string> - </property> <property name="text"> - <string notr="true">(out of sync)</string> + <string/> </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> + <property name="icon"> + <iconset resource="../bitcoin.qrc"> + <normaloff>:/icons/warning</normaloff> + <disabledoff>:/icons/warning</disabledoff>:/icons/warning</iconset> + </property> + <property name="iconSize"> + <size> + <width>24</width> + <height>24</height> + </size> + </property> + <property name="flat"> + <bool>true</bool> </property> </widget> </item> diff --git a/src/qt/forms/sendcoinsentry.ui b/src/qt/forms/sendcoinsentry.ui index 48d0dd093c..df06f36833 100644 --- a/src/qt/forms/sendcoinsentry.ui +++ b/src/qt/forms/sendcoinsentry.ui @@ -21,15 +21,21 @@ <string>This is a normal payment.</string> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout"> - <property name="spacing"> + <property name="topMargin"> + <number>8</number> + </property> + <property name="bottomMargin"> + <number>4</number> + </property> + <property name="horizontalSpacing"> <number>12</number> </property> + <property name="verticalSpacing"> + <number>8</number> + </property> <item row="0" column="0"> <widget class="QLabel" name="payToLabel"> <property name="text"> @@ -193,6 +199,13 @@ </property> </widget> </item> + <item row="4" column="0" colspan="2"> + <widget class="Line" name="line"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + </widget> + </item> </layout> </widget> <widget class="QFrame" name="SendCoins_UnauthenticatedPaymentRequest"> @@ -618,10 +631,7 @@ <bool>true</bool> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout_is"> <property name="spacing"> @@ -1150,10 +1160,7 @@ <bool>true</bool> </property> <property name="frameShape"> - <enum>QFrame::StyledPanel</enum> - </property> - <property name="frameShadow"> - <enum>QFrame::Sunken</enum> + <enum>QFrame::NoFrame</enum> </property> <layout class="QGridLayout" name="gridLayout_s"> <property name="spacing"> diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 4fa15db9c6..a422ff9a71 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -18,8 +18,8 @@ #include <QAbstractItemDelegate> #include <QPainter> -#define DECORATION_SIZE 64 -#define NUM_ITEMS 3 +#define DECORATION_SIZE 54 +#define NUM_ITEMS 5 class TxViewDelegate : public QAbstractItemDelegate { @@ -129,10 +129,6 @@ OverviewPage::OverviewPage(QWidget *parent) : connect(ui->listTransactions, SIGNAL(clicked(QModelIndex)), this, SLOT(handleTransactionClicked(QModelIndex))); - // init "out of sync" warning labels - ui->labelWalletStatus->setText("(" + tr("out of sync") + ")"); - ui->labelTransactionsStatus->setText("(" + tr("out of sync") + ")"); - // start with displaying the "out of sync" warnings showOutOfSyncWarning(true); } diff --git a/src/qt/res/icons/warning.png b/src/qt/res/icons/warning.png Binary files differnew file mode 100644 index 0000000000..723a30a658 --- /dev/null +++ b/src/qt/res/icons/warning.png diff --git a/src/qt/scicon.cpp b/src/qt/scicon.cpp index a0ffcd82a9..b2f2399b24 100644 --- a/src/qt/scicon.cpp +++ b/src/qt/scicon.cpp @@ -27,12 +27,17 @@ static void MakeSingleColorImage(QImage& img, const QColor& colorbase) QImage SingleColorImage(const QString& filename, const QColor& colorbase) { QImage img(filename); +#if !defined(WIN32) && !defined(MAC_OSX) MakeSingleColorImage(img, colorbase); +#endif return img; } QIcon SingleColorIcon(const QIcon& ico, const QColor& colorbase) { +#if defined(WIN32) || defined(MAC_OSX) + return ico; +#else QIcon new_ico; QSize sz; Q_FOREACH(sz, ico.availableSizes()) @@ -42,6 +47,7 @@ QIcon SingleColorIcon(const QIcon& ico, const QColor& colorbase) new_ico.addPixmap(QPixmap::fromImage(img)); } return new_ico; +#endif } QIcon SingleColorIcon(const QString& filename, const QColor& colorbase) @@ -51,6 +57,9 @@ QIcon SingleColorIcon(const QString& filename, const QColor& colorbase) QColor SingleColor() { +#if defined(WIN32) || defined(MAC_OSX) + return QColor(0,0,0); +#else const QColor colorHighlightBg(QApplication::palette().color(QPalette::Highlight)); const QColor colorHighlightFg(QApplication::palette().color(QPalette::HighlightedText)); const QColor colorText(QApplication::palette().color(QPalette::WindowText)); @@ -61,6 +70,7 @@ QColor SingleColor() else colorbase = colorHighlightFg; return colorbase; +#endif } QIcon SingleColorIcon(const QString& filename) diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 7a33e3567b..3d57711568 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -590,12 +590,12 @@ void SendCoinsDialog::updateGlobalFeeVariables() { if (ui->radioSmartFee->isChecked()) { - nTxConfirmTarget = (int)25 - (int)std::max(0, std::min(24, ui->sliderSmartFee->value())); + nTxConfirmTarget = defaultConfirmTarget - ui->sliderSmartFee->value(); payTxFee = CFeeRate(0); } else { - nTxConfirmTarget = 25; + nTxConfirmTarget = defaultConfirmTarget; payTxFee = CFeeRate(ui->customFee->value()); fPayAtLeastCustomFee = ui->radioCustomAtLeast->isChecked(); } @@ -629,7 +629,7 @@ void SendCoinsDialog::updateSmartFeeLabel() if(!model || !model->getOptionsModel()) return; - int nBlocksToConfirm = (int)25 - (int)std::max(0, std::min(24, ui->sliderSmartFee->value())); + int nBlocksToConfirm = defaultConfirmTarget - ui->sliderSmartFee->value(); CFeeRate feeRate = mempool.estimateFee(nBlocksToConfirm); if (feeRate <= CFeeRate(0)) // not enough data => minfee { diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 14adb02573..fc513bf2ba 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -23,6 +23,8 @@ QT_BEGIN_NAMESPACE class QUrl; QT_END_NAMESPACE +const int defaultConfirmTarget = 25; + /** Dialog for sending bitcoins */ class SendCoinsDialog : public QDialog { diff --git a/src/rest.cpp b/src/rest.cpp index adc2d56284..1b7954bbf6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -9,14 +9,18 @@ #include "rpcserver.h" #include "streams.h" #include "sync.h" +#include "txmempool.h" #include "utilstrencodings.h" #include "version.h" #include <boost/algorithm/string.hpp> +#include <boost/dynamic_bitset.hpp> using namespace std; using namespace json_spirit; +static const int MAX_GETUTXOS_OUTPOINTS = 100; //allow a max of 100 outpoints to be queried at once + enum RetFormat { RF_UNDEF, RF_BINARY, @@ -34,6 +38,22 @@ static const struct { {RF_JSON, "json"}, }; +struct CCoin { + uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE + uint32_t nHeight; + CTxOut out; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + { + READWRITE(nTxVer); + READWRITE(nHeight); + READWRITE(out); + } +}; + class RestErr { public: @@ -43,6 +63,7 @@ public: extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry); extern Object blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); +extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, Object& out, bool fIncludeHex); static RestErr RESTERR(enum HTTPStatusCode status, string message) { @@ -90,12 +111,13 @@ static bool ParseHashStr(const string& strReq, uint256& v) } static bool rest_headers(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { vector<string> params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); vector<string> path; boost::split(path, params[0], boost::is_any_of("/")); @@ -153,13 +175,14 @@ static bool rest_headers(AcceptedConnection* conn, } static bool rest_block(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun, bool showTxDetails) { vector<string> params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; @@ -174,6 +197,9 @@ static bool rest_block(AcceptedConnection* conn, throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); pblockindex = mapBlockIndex[hash]; + if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) + throw RESTERR(HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); + if (!ReadBlockFromDisk(block, pblockindex)) throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); } @@ -211,28 +237,31 @@ static bool rest_block(AcceptedConnection* conn, } static bool rest_block_extended(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { - return rest_block(conn, strReq, mapHeaders, fRun, true); + return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, true); } static bool rest_block_notxdetails(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { - return rest_block(conn, strReq, mapHeaders, fRun, false); + return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, false); } static bool rest_chaininfo(AcceptedConnection* conn, - const std::string& strReq, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) + const std::string& strURIPart, + const std::string& strRequest, + const std::map<std::string, std::string>& mapHeaders, + bool fRun) { vector<string> params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); switch (rf) { case RF_JSON: { @@ -253,12 +282,13 @@ static bool rest_chaininfo(AcceptedConnection* conn, } static bool rest_tx(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { vector<string> params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; @@ -303,10 +333,191 @@ static bool rest_tx(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } +static bool rest_getutxos(AcceptedConnection* conn, + const std::string& strURIPart, + const std::string& strRequest, + const std::map<std::string, std::string>& mapHeaders, + bool fRun) +{ + vector<string> params; + enum RetFormat rf = ParseDataFormat(params, strURIPart); + + // throw exception in case of a empty request + if (strRequest.length() == 0) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + + bool fCheckMemPool = false; + vector<COutPoint> vOutPoints; + + // parse/deserialize input + // input-format = output-format, rest/getutxos/bin requires binary input, gives binary output, ... + + string strRequestMutable = strRequest; //convert const string to string for allowing hex to bin converting + + switch (rf) { + case RF_HEX: { + // convert hex to bin, continue then with bin part + std::vector<unsigned char> strRequestV = ParseHex(strRequest); + strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); + } + + case RF_BINARY: { + try { + //deserialize + CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); + oss << strRequestMutable; + oss >> fCheckMemPool; + oss >> vOutPoints; + } catch (const std::ios_base::failure& e) { + // abort in case of unreadable binary data + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + } + break; + } + + case RF_JSON: { + try { + // parse json request + Value valRequest; + if (!read_string(strRequest, valRequest)) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + + Object jsonObject = valRequest.get_obj(); + const Value& checkMempoolValue = find_value(jsonObject, "checkmempool"); + + if (!checkMempoolValue.is_null()) { + fCheckMemPool = checkMempoolValue.get_bool(); + } + const Value& outpointsValue = find_value(jsonObject, "outpoints"); + if (!outpointsValue.is_null()) { + Array outPoints = outpointsValue.get_array(); + BOOST_FOREACH (const Value& outPoint, outPoints) { + Object outpointObject = outPoint.get_obj(); + uint256 txid = ParseHashO(outpointObject, "txid"); + Value nValue = find_value(outpointObject, "n"); + int nOutput = nValue.get_int(); + vOutPoints.push_back(COutPoint(txid, nOutput)); + } + } + } catch (...) { + // return HTTP 500 if there was a json parsing error + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + } + break; + } + default: { + throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } + + // limit max outpoints + if (vOutPoints.size() > MAX_GETUTXOS_OUTPOINTS) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); + + // check spentness and form a bitmap (as well as a JSON capable human-readble string representation) + vector<unsigned char> bitmap; + vector<CCoin> outs; + std::string bitmapStringRepresentation; + boost::dynamic_bitset<unsigned char> hits(vOutPoints.size()); + { + LOCK2(cs_main, mempool.cs); + + CCoinsView viewDummy; + CCoinsViewCache view(&viewDummy); + + CCoinsViewCache& viewChain = *pcoinsTip; + CCoinsViewMemPool viewMempool(&viewChain, mempool); + + if (fCheckMemPool) + view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool + + for (size_t i = 0; i < vOutPoints.size(); i++) { + CCoins coins; + uint256 hash = vOutPoints[i].hash; + if (view.GetCoins(hash, coins)) { + mempool.pruneSpent(hash, coins); + if (coins.IsAvailable(vOutPoints[i].n)) { + hits[i] = true; + // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if + // n is valid but points to an already spent output (IsNull). + CCoin coin; + coin.nTxVer = coins.nVersion; + coin.nHeight = coins.nHeight; + coin.out = coins.vout.at(vOutPoints[i].n); + assert(!coin.out.IsNull()); + outs.push_back(coin); + } + } + + bitmapStringRepresentation.append(hits[i] ? "1" : "0"); // form a binary string representation (human-readable for json output) + } + } + boost::to_block_range(hits, std::back_inserter(bitmap)); + + switch (rf) { + case RF_BINARY: { + // serialize data + // use exact same output as mentioned in Bip64 + CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); + ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; + string ssGetUTXOResponseString = ssGetUTXOResponse.str(); + + conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, ssGetUTXOResponseString.size(), "application/octet-stream") << ssGetUTXOResponseString << std::flush; + return true; + } + + case RF_HEX: { + CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); + ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; + string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; + + conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + return true; + } + + case RF_JSON: { + Object objGetUTXOResponse; + + // pack in some essentials + // use more or less the same output as mentioned in Bip64 + objGetUTXOResponse.push_back(Pair("chainHeight", chainActive.Height())); + objGetUTXOResponse.push_back(Pair("chaintipHash", chainActive.Tip()->GetBlockHash().GetHex())); + objGetUTXOResponse.push_back(Pair("bitmap", bitmapStringRepresentation)); + + Array utxos; + BOOST_FOREACH (const CCoin& coin, outs) { + Object utxo; + utxo.push_back(Pair("txvers", (int32_t)coin.nTxVer)); + utxo.push_back(Pair("height", (int32_t)coin.nHeight)); + utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); + + // include the script in a json output + Object o; + ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + utxo.push_back(Pair("scriptPubKey", o)); + utxos.push_back(utxo); + } + objGetUTXOResponse.push_back(Pair("utxos", utxos)); + + // return json string + string strJSON = write_string(Value(objGetUTXOResponse), false) + "\n"; + conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + return true; + } + default: { + throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } + + // not reached + return true; // continue to process further HTTP reqs on this cxn +} + static const struct { const char* prefix; bool (*handler)(AcceptedConnection* conn, - const std::string& strURI, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun); } uri_prefixes[] = { @@ -315,10 +526,12 @@ static const struct { {"/rest/block/", rest_block_extended}, {"/rest/chaininfo", rest_chaininfo}, {"/rest/headers/", rest_headers}, + {"/rest/getutxos", rest_getutxos}, }; bool HTTPReq_REST(AcceptedConnection* conn, const std::string& strURI, + const string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { @@ -330,8 +543,8 @@ bool HTTPReq_REST(AcceptedConnection* conn, for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) { unsigned int plen = strlen(uri_prefixes[i].prefix); if (strURI.substr(0, plen) == uri_prefixes[i].prefix) { - string strReq = strURI.substr(plen); - return uri_prefixes[i].handler(conn, strReq, mapHeaders, fRun); + string strURIPart = strURI.substr(plen); + return uri_prefixes[i].handler(conn, strURIPart, strRequest, mapHeaders, fRun); } } } catch (const RestErr& re) { diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index ed32ee7723..ecf8e8bcc6 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -299,6 +299,9 @@ Value getblock(const Array& params, bool fHelp) CBlock block; CBlockIndex* pblockindex = mapBlockIndex[hash]; + if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) + throw JSONRPCError(RPC_INTERNAL_ERROR, "Block not available (pruned data)"); + if(!ReadBlockFromDisk(block, pblockindex)) throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); @@ -492,6 +495,15 @@ Value getblockchaininfo(const Array& params, bool fHelp) obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("verificationprogress", Checkpoints::GuessVerificationProgress(Params().Checkpoints(), chainActive.Tip()))); obj.push_back(Pair("chainwork", chainActive.Tip()->nChainWork.GetHex())); + obj.push_back(Pair("pruned", fPruneMode)); + if (fPruneMode) + { + CBlockIndex *block = chainActive.Tip(); + while (block && block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) + block = block->pprev; + + obj.push_back(Pair("pruneheight", block->nHeight)); + } return obj; } diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 12a5c4aef9..3f74517a67 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -994,7 +994,7 @@ void ServiceConnection(AcceptedConnection *conn) // Process via HTTP REST API } else if (strURI.substr(0, 6) == "/rest/" && GetBoolArg("-rest", false)) { - if (!HTTPReq_REST(conn, strURI, mapHeaders, fRun)) + if (!HTTPReq_REST(conn, strURI, strRequest, mapHeaders, fRun)) break; } else { diff --git a/src/rpcserver.h b/src/rpcserver.h index 790104f8c9..30a5b28db7 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -239,6 +239,7 @@ extern json_spirit::Value reconsiderblock(const json_spirit::Array& params, bool // in rest.cpp extern bool HTTPReq_REST(AcceptedConnection *conn, const std::string& strURI, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun); diff --git a/src/scheduler.cpp b/src/scheduler.cpp new file mode 100644 index 0000000000..8b55888ae8 --- /dev/null +++ b/src/scheduler.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "scheduler.h" + +#include <assert.h> +#include <boost/bind.hpp> +#include <utility> + +CScheduler::CScheduler() : nThreadsServicingQueue(0) +{ +} + +CScheduler::~CScheduler() +{ + assert(nThreadsServicingQueue == 0); +} + + +#if BOOST_VERSION < 105000 +static boost::system_time toPosixTime(const boost::chrono::system_clock::time_point& t) +{ + return boost::posix_time::from_time_t(boost::chrono::system_clock::to_time_t(t)); +} +#endif + +void CScheduler::serviceQueue() +{ + boost::unique_lock<boost::mutex> lock(newTaskMutex); + ++nThreadsServicingQueue; + + // newTaskMutex is locked throughout this loop EXCEPT + // when the thread is waiting or when the user's function + // is called. + while (1) { + try { + while (taskQueue.empty()) { + // Wait until there is something to do. + newTaskScheduled.wait(lock); + } +// Wait until either there is a new task, or until +// the time of the first item on the queue: + +// wait_until needs boost 1.50 or later; older versions have timed_wait: +#if BOOST_VERSION < 105000 + while (!taskQueue.empty() && newTaskScheduled.timed_wait(lock, toPosixTime(taskQueue.begin()->first))) { + // Keep waiting until timeout + } +#else + while (!taskQueue.empty() && newTaskScheduled.wait_until(lock, taskQueue.begin()->first) != boost::cv_status::timeout) { + // Keep waiting until timeout + } +#endif + // If there are multiple threads, the queue can empty while we're waiting (another + // thread may service the task we were waiting on). + if (taskQueue.empty()) + continue; + + Function f = taskQueue.begin()->second; + taskQueue.erase(taskQueue.begin()); + + // Unlock before calling f, so it can reschedule itself or another task + // without deadlocking: + lock.unlock(); + f(); + lock.lock(); + } catch (...) { + --nThreadsServicingQueue; + throw; + } + } +} + +void CScheduler::schedule(CScheduler::Function f, boost::chrono::system_clock::time_point t) +{ + { + boost::unique_lock<boost::mutex> lock(newTaskMutex); + taskQueue.insert(std::make_pair(t, f)); + } + newTaskScheduled.notify_one(); +} + +void CScheduler::scheduleFromNow(CScheduler::Function f, int64_t deltaSeconds) +{ + schedule(f, boost::chrono::system_clock::now() + boost::chrono::seconds(deltaSeconds)); +} + +static void Repeat(CScheduler* s, CScheduler::Function f, int64_t deltaSeconds) +{ + f(); + s->scheduleFromNow(boost::bind(&Repeat, s, f, deltaSeconds), deltaSeconds); +} + +void CScheduler::scheduleEvery(CScheduler::Function f, int64_t deltaSeconds) +{ + scheduleFromNow(boost::bind(&Repeat, this, f, deltaSeconds), deltaSeconds); +} diff --git a/src/scheduler.h b/src/scheduler.h new file mode 100644 index 0000000000..bb383ab9f9 --- /dev/null +++ b/src/scheduler.h @@ -0,0 +1,70 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SCHEDULER_H +#define BITCOIN_SCHEDULER_H + +// +// NOTE: +// boost::thread / boost::function / boost::chrono should be ported to +// std::thread / std::function / std::chrono when we support C++11. +// +#include <boost/function.hpp> +#include <boost/chrono/chrono.hpp> +#include <boost/thread.hpp> +#include <map> + +// +// Simple class for background tasks that should be run +// periodically or once "after a while" +// +// Usage: +// +// CScheduler* s = new CScheduler(); +// s->scheduleFromNow(doSomething, 11); // Assuming a: void doSomething() { } +// s->scheduleFromNow(boost::bind(Class::func, this, argument), 3); +// boost::thread* t = new boost::thread(boost::bind(CScheduler::serviceQueue, s)); +// +// ... then at program shutdown, clean up the thread running serviceQueue: +// t->interrupt(); +// t->join(); +// delete t; +// delete s; // Must be done after thread is interrupted/joined. +// + +class CScheduler +{ +public: + CScheduler(); + ~CScheduler(); + + typedef boost::function<void(void)> Function; + + // Call func at/after time t + void schedule(Function f, boost::chrono::system_clock::time_point t); + + // Convenience method: call f once deltaSeconds from now + void scheduleFromNow(Function f, int64_t deltaSeconds); + + // Another convenience method: call f approximately + // every deltaSeconds forever, starting deltaSeconds from now. + // To be more precise: every time f is finished, it + // is rescheduled to run deltaSeconds later. If you + // need more accurate scheduling, don't use this method. + void scheduleEvery(Function f, int64_t deltaSeconds); + + // To keep things as simple as possible, there is no unschedule. + + // Services the queue 'forever'. Should be run in a thread, + // and interrupted using boost::interrupt_thread + void serviceQueue(); + +private: + std::multimap<boost::chrono::system_clock::time_point, Function> taskQueue; + boost::condition_variable newTaskScheduled; + boost::mutex newTaskMutex; + int nThreadsServicingQueue; +}; + +#endif diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index b9f7d243ec..076ff1295f 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -2,6 +2,7 @@ bench_inv bench_sign bench_verify bench_recover +bench_internal tests *.exe *.so diff --git a/src/secp256k1/.travis.yml b/src/secp256k1/.travis.yml index 40f8dae23f..0d8089cfe4 100644 --- a/src/secp256k1/.travis.yml +++ b/src/secp256k1/.travis.yml @@ -1,14 +1,14 @@ language: c +sudo: false +addons: + apt: + packages: libgmp-dev compiler: - clang - gcc -install: - - sudo apt-get install -qq libssl-dev - - if [ "$BIGNUM" = "gmp" -o "$BIGNUM" = "auto" ]; then sudo apt-get install --no-install-recommends --no-upgrade -qq libgmp-dev; fi - - if [ -n "$EXTRAPACKAGES" ]; then sudo apt-get update && sudo apt-get install --no-install-recommends --no-upgrade $EXTRAPACKAGES; fi env: global: - - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no ASM=no BUILD=check EXTRAFLAGS= HOST= EXTRAPACKAGES= + - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no ASM=no BUILD=check EXTRAFLAGS= HOST= matrix: - SCALAR=32bit - SCALAR=64bit @@ -22,8 +22,35 @@ env: - BIGNUM=no ENDOMORPHISM=yes - BUILD=distcheck - EXTRAFLAGS=CFLAGS=-DDETERMINISTIC - - HOST=i686-linux-gnu EXTRAPACKAGES="gcc-multilib" - - HOST=i686-linux-gnu EXTRAPACKAGES="gcc-multilib" ENDOMORPHISM=yes +matrix: + fast_finish: true + include: + - compiler: clang + env: HOST=i686-linux-gnu ENDOMORPHISM=yes + addons: + apt: + packages: + - gcc-multilib + - libgmp-dev:i386 + - compiler: clang + env: HOST=i686-linux-gnu + addons: + apt: + packages: + - gcc-multilib + - compiler: gcc + env: HOST=i686-linux-gnu ENDOMORPHISM=yes + addons: + apt: + packages: + - gcc-multilib + - compiler: gcc + env: HOST=i686-linux-gnu + addons: + apt: + packages: + - gcc-multilib + - libgmp-dev:i386 before_script: ./autogen.sh script: - if [ -n "$HOST" ]; then export USE_HOST="--host=$HOST"; fi diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index a6e39d13db..06afd4c65b 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -40,42 +40,60 @@ extern "C" { # define SECP256K1_ARG_NONNULL(_x) # endif +/** Opaque data structure that holds context information (precomputed tables etc.). + * Only functions that take a pointer to a non-const context require exclusive + * access to it. Multiple functions that take a pointer to a const context may + * run simultaneously. + */ +typedef struct secp256k1_context_struct secp256k1_context_t; + +/** Flags to pass to secp256k1_context_create. */ +# define SECP256K1_CONTEXT_VERIFY (1 << 0) +# define SECP256K1_CONTEXT_SIGN (1 << 1) -/** Flags to pass to secp256k1_start. */ -# define SECP256K1_START_VERIFY (1 << 0) -# define SECP256K1_START_SIGN (1 << 1) +/** Create a secp256k1 context object. + * Returns: a newly created context object. + * In: flags: which parts of the context to initialize. + */ +secp256k1_context_t* secp256k1_context_create( + int flags +) SECP256K1_WARN_UNUSED_RESULT; -/** Initialize the library. This may take some time (10-100 ms). - * You need to call this before calling any other function. - * It cannot run in parallel with any other functions, but once - * secp256k1_start() returns, all other functions are thread-safe. +/** Copies a secp256k1 context object. + * Returns: a newly created context object. + * In: ctx: an existing context to copy */ -void secp256k1_start(unsigned int flags); +secp256k1_context_t* secp256k1_context_clone( + const secp256k1_context_t* ctx +) SECP256K1_WARN_UNUSED_RESULT; -/** Free all memory associated with this library. After this, no - * functions can be called anymore, except secp256k1_start() +/** Destroy a secp256k1 context object. + * The context pointer may not be used afterwards. */ -void secp256k1_stop(void); +void secp256k1_context_destroy( + secp256k1_context_t* ctx +) SECP256K1_ARG_NONNULL(1); /** Verify an ECDSA signature. * Returns: 1: correct signature * 0: incorrect signature * -1: invalid public key * -2: invalid signature - * In: msg32: the 32-byte message hash being verified (cannot be NULL) + * In: ctx: a secp256k1 context object, initialized for verification. + * msg32: the 32-byte message hash being verified (cannot be NULL) * sig: the signature being verified (cannot be NULL) * siglen: the length of the signature * pubkey: the public key to verify with (cannot be NULL) * pubkeylen: the length of pubkey - * Requires starting using SECP256K1_START_VERIFY. */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_verify( + const secp256k1_context_t* ctx, const unsigned char *msg32, const unsigned char *sig, int siglen, const unsigned char *pubkey, int pubkeylen -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(5); /** A pointer to a function to deterministically generate a nonce. * Returns: 1 if a nonce was successfully generated. 0 will cause signing to fail. @@ -111,15 +129,14 @@ extern const secp256k1_nonce_function_t secp256k1_nonce_function_default; * Returns: 1: signature created * 0: the nonce generation function failed, the private key was invalid, or there is not * enough space in the signature (as indicated by siglen). - * In: msg32: the 32-byte message hash being signed (cannot be NULL) + * In: ctx: pointer to a context object, initialized for signing (cannot be NULL) + * msg32: the 32-byte message hash being signed (cannot be NULL) * seckey: pointer to a 32-byte secret key (cannot be NULL) * noncefp:pointer to a nonce generation function. If NULL, secp256k1_nonce_function_default is used * ndata: pointer to arbitrary data used by the nonce generation function (can be NULL) * Out: sig: pointer to an array where the signature will be placed (cannot be NULL) * In/Out: siglen: pointer to an int with the length of sig, which will be updated - * to contain the actual signature length (<=72). If 0 is returned, this will be - * set to zero. - * Requires starting using SECP256K1_START_SIGN. + * to contain the actual signature length (<=72). * * The sig always has an s value in the lower half of the range (From 0x1 * to 0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF5D576E7357A4501DDFE92F46681B20A0, @@ -148,145 +165,180 @@ extern const secp256k1_nonce_function_t secp256k1_nonce_function_default; * be taken when this property is required for an application. */ int secp256k1_ecdsa_sign( + const secp256k1_context_t* ctx, const unsigned char *msg32, unsigned char *sig, int *siglen, const unsigned char *seckey, secp256k1_nonce_function_t noncefp, const void *ndata -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); /** Create a compact ECDSA signature (64 byte + recovery id). * Returns: 1: signature created * 0: the nonce generation function failed, or the secret key was invalid. - * In: msg32: the 32-byte message hash being signed (cannot be NULL) + * In: ctx: pointer to a context object, initialized for signing (cannot be NULL) + * msg32: the 32-byte message hash being signed (cannot be NULL) * seckey: pointer to a 32-byte secret key (cannot be NULL) * noncefp:pointer to a nonce generation function. If NULL, secp256k1_nonce_function_default is used * ndata: pointer to arbitrary data used by the nonce generation function (can be NULL) * Out: sig: pointer to a 64-byte array where the signature will be placed (cannot be NULL) * In case 0 is returned, the returned signature length will be zero. * recid: pointer to an int, which will be updated to contain the recovery id (can be NULL) - * Requires starting using SECP256K1_START_SIGN. */ int secp256k1_ecdsa_sign_compact( + const secp256k1_context_t* ctx, const unsigned char *msg32, unsigned char *sig64, const unsigned char *seckey, secp256k1_nonce_function_t noncefp, const void *ndata, int *recid -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); /** Recover an ECDSA public key from a compact signature. * Returns: 1: public key successfully recovered (which guarantees a correct signature). * 0: otherwise. - * In: msg32: the 32-byte message hash assumed to be signed (cannot be NULL) + * In: ctx: pointer to a context object, initialized for verification (cannot be NULL) + * msg32: the 32-byte message hash assumed to be signed (cannot be NULL) * sig64: signature as 64 byte array (cannot be NULL) * compressed: whether to recover a compressed or uncompressed pubkey * recid: the recovery id (0-3, as returned by ecdsa_sign_compact) * Out: pubkey: pointer to a 33 or 65 byte array to put the pubkey (cannot be NULL) * pubkeylen: pointer to an int that will contain the pubkey length (cannot be NULL) - * Requires starting using SECP256K1_START_VERIFY. */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_recover_compact( + const secp256k1_context_t* ctx, const unsigned char *msg32, const unsigned char *sig64, unsigned char *pubkey, int *pubkeylen, int compressed, int recid -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); /** Verify an ECDSA secret key. * Returns: 1: secret key is valid * 0: secret key is invalid - * In: seckey: pointer to a 32-byte secret key (cannot be NULL) + * In: ctx: pointer to a context object (cannot be NULL) + * seckey: pointer to a 32-byte secret key (cannot be NULL) */ -SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_verify(const unsigned char *seckey) SECP256K1_ARG_NONNULL(1); +SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_verify( + const secp256k1_context_t* ctx, + const unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); /** Just validate a public key. - * Returns: 1: valid public key - * 0: invalid public key - * In: pubkey: pointer to a 33-byte or 65-byte public key (cannot be NULL). + * Returns: 1: public key is valid + * 0: public key is invalid + * In: ctx: pointer to a context object (cannot be NULL) + * pubkey: pointer to a 33-byte or 65-byte public key (cannot be NULL). * pubkeylen: length of pubkey */ -SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_verify(const unsigned char *pubkey, int pubkeylen) SECP256K1_ARG_NONNULL(1); +SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_verify( + const secp256k1_context_t* ctx, + const unsigned char *pubkey, + int pubkeylen +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); /** Compute the public key for a secret key. - * In: compressed: whether the computed public key should be compressed + * In: ctx: pointer to a context object, initialized for signing (cannot be NULL) + * compressed: whether the computed public key should be compressed * seckey: pointer to a 32-byte private key (cannot be NULL) * Out: pubkey: pointer to a 33-byte (if compressed) or 65-byte (if uncompressed) * area to store the public key (cannot be NULL) * pubkeylen: pointer to int that will be updated to contains the pubkey's * length (cannot be NULL) * Returns: 1: secret was valid, public key stores - * 0: secret was invalid, try again. - * Requires starting using SECP256K1_START_SIGN. + * 0: secret was invalid, try again */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_create( + const secp256k1_context_t* ctx, unsigned char *pubkey, int *pubkeylen, const unsigned char *seckey, int compressed -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); /** Decompress a public key. + * In: ctx: pointer to a context object (cannot be NULL) * In/Out: pubkey: pointer to a 65-byte array to put the decompressed public key. - It must contain a 33-byte or 65-byte public key already (cannot be NULL) + * It must contain a 33-byte or 65-byte public key already (cannot be NULL) * pubkeylen: pointer to the size of the public key pointed to by pubkey (cannot be NULL) - It will be updated to reflect the new size. - * Returns: 0 if the passed public key was invalid, 1 otherwise. If 1 is returned, the - pubkey is replaced with its decompressed version. + * It will be updated to reflect the new size. + * Returns: 0: pubkey was invalid + * 1: pubkey was valid, and was replaced with its decompressed version */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_decompress( + const secp256k1_context_t* ctx, unsigned char *pubkey, int *pubkeylen -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Export a private key in DER format. */ +/** Export a private key in DER format. + * In: ctx: pointer to a context object, initialized for signing (cannot be NULL) + */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_export( + const secp256k1_context_t* ctx, const unsigned char *seckey, unsigned char *privkey, int *privkeylen, int compressed -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); /** Import a private key in DER format. */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_import( + const secp256k1_context_t* ctx, unsigned char *seckey, const unsigned char *privkey, int privkeylen -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); /** Tweak a private key by adding tweak to it. */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_add( + const secp256k1_context_t* ctx, unsigned char *seckey, const unsigned char *tweak -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); /** Tweak a public key by adding tweak times the generator to it. - * Requires starting with SECP256K1_START_VERIFY. + * In: ctx: pointer to a context object, initialized for verification (cannot be NULL) */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_add( + const secp256k1_context_t* ctx, unsigned char *pubkey, int pubkeylen, const unsigned char *tweak -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); /** Tweak a private key by multiplying it with tweak. */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_mul( + const secp256k1_context_t* ctx, unsigned char *seckey, const unsigned char *tweak -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); /** Tweak a public key by multiplying it with tweak. - * Requires starting with SECP256K1_START_VERIFY. + * In: ctx: pointer to a context object, initialized for verification (cannot be NULL) */ SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( + const secp256k1_context_t* ctx, unsigned char *pubkey, int pubkeylen, const unsigned char *tweak -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); + +/** Updates the context randomization. + * Returns: 1: randomization successfully updated + * 0: error + * In: ctx: pointer to a context object (cannot be NULL) + * seed32: pointer to a 32-byte random seed (NULL resets to initial state) + */ +SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( + secp256k1_context_t* ctx, + const unsigned char *seed32 +) SECP256K1_ARG_NONNULL(1); + # ifdef __cplusplus } diff --git a/src/secp256k1/src/bench.h b/src/secp256k1/src/bench.h index 0559b3e853..db5f68cee1 100644 --- a/src/secp256k1/src/bench.h +++ b/src/secp256k1/src/bench.h @@ -48,7 +48,7 @@ void run_benchmark(char *name, void (*benchmark)(void*), void (*setup)(void*), v print_number(min * 1000000.0 / iter); printf("us / avg "); print_number((sum / count) * 1000000.0 / iter); - printf("us / avg "); + printf("us / max "); print_number(max * 1000000.0 / iter); printf("us\n"); } diff --git a/src/secp256k1/src/bench_recover.c b/src/secp256k1/src/bench_recover.c index 6991cc9d6c..56faed11a0 100644 --- a/src/secp256k1/src/bench_recover.c +++ b/src/secp256k1/src/bench_recover.c @@ -9,6 +9,7 @@ #include "bench.h" typedef struct { + secp256k1_context_t *ctx; unsigned char msg[32]; unsigned char sig[64]; } bench_recover_t; @@ -21,7 +22,7 @@ void bench_recover(void* arg) { for (i = 0; i < 20000; i++) { int j; int pubkeylen = 33; - CHECK(secp256k1_ecdsa_recover_compact(data->msg, data->sig, pubkey, &pubkeylen, 1, i % 2)); + CHECK(secp256k1_ecdsa_recover_compact(data->ctx, data->msg, data->sig, pubkey, &pubkeylen, 1, i % 2)); for (j = 0; j < 32; j++) { data->sig[j + 32] = data->msg[j]; /* Move former message to S. */ data->msg[j] = data->sig[j]; /* Move former R to message. */ @@ -40,10 +41,11 @@ void bench_recover_setup(void* arg) { int main(void) { bench_recover_t data; - secp256k1_start(SECP256K1_START_VERIFY); + + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); run_benchmark("ecdsa_recover", bench_recover, bench_recover_setup, NULL, &data, 10, 20000); - secp256k1_stop(); + secp256k1_context_destroy(data.ctx); return 0; } diff --git a/src/secp256k1/src/bench_sign.c b/src/secp256k1/src/bench_sign.c index c5b6829a84..072a37af51 100644 --- a/src/secp256k1/src/bench_sign.c +++ b/src/secp256k1/src/bench_sign.c @@ -9,6 +9,7 @@ #include "bench.h" typedef struct { + secp256k1_context_t* ctx; unsigned char msg[32]; unsigned char key[32]; } bench_sign_t; @@ -29,7 +30,7 @@ static void bench_sign(void* arg) { for (i = 0; i < 20000; i++) { int j; int recid = 0; - CHECK(secp256k1_ecdsa_sign_compact(data->msg, sig, data->key, NULL, NULL, &recid)); + CHECK(secp256k1_ecdsa_sign_compact(data->ctx, data->msg, sig, data->key, NULL, NULL, &recid)); for (j = 0; j < 32; j++) { data->msg[j] = sig[j]; /* Move former R to message. */ data->key[j] = sig[j + 32]; /* Move former S to key. */ @@ -39,10 +40,11 @@ static void bench_sign(void* arg) { int main(void) { bench_sign_t data; - secp256k1_start(SECP256K1_START_SIGN); + + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); run_benchmark("ecdsa_sign", bench_sign, bench_sign_setup, NULL, &data, 10, 20000); - secp256k1_stop(); + secp256k1_context_destroy(data.ctx); return 0; } diff --git a/src/secp256k1/src/bench_verify.c b/src/secp256k1/src/bench_verify.c index c279305a0d..c8c82752ce 100644 --- a/src/secp256k1/src/bench_verify.c +++ b/src/secp256k1/src/bench_verify.c @@ -12,6 +12,7 @@ #include "bench.h" typedef struct { + secp256k1_context_t *ctx; unsigned char msg[32]; unsigned char key[32]; unsigned char sig[72]; @@ -28,7 +29,7 @@ static void benchmark_verify(void* arg) { data->sig[data->siglen - 1] ^= (i & 0xFF); data->sig[data->siglen - 2] ^= ((i >> 8) & 0xFF); data->sig[data->siglen - 3] ^= ((i >> 16) & 0xFF); - CHECK(secp256k1_ecdsa_verify(data->msg, data->sig, data->siglen, data->pubkey, data->pubkeylen) == (i == 0)); + CHECK(secp256k1_ecdsa_verify(data->ctx, data->msg, data->sig, data->siglen, data->pubkey, data->pubkeylen) == (i == 0)); data->sig[data->siglen - 1] ^= (i & 0xFF); data->sig[data->siglen - 2] ^= ((i >> 8) & 0xFF); data->sig[data->siglen - 3] ^= ((i >> 16) & 0xFF); @@ -39,17 +40,17 @@ int main(void) { int i; benchmark_verify_t data; - secp256k1_start(SECP256K1_START_VERIFY | SECP256K1_START_SIGN); + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); for (i = 0; i < 32; i++) data.msg[i] = 1 + i; for (i = 0; i < 32; i++) data.key[i] = 33 + i; data.siglen = 72; - secp256k1_ecdsa_sign(data.msg, data.sig, &data.siglen, data.key, NULL, NULL); + secp256k1_ecdsa_sign(data.ctx, data.msg, data.sig, &data.siglen, data.key, NULL, NULL); data.pubkeylen = 33; - CHECK(secp256k1_ec_pubkey_create(data.pubkey, &data.pubkeylen, data.key, 1)); + CHECK(secp256k1_ec_pubkey_create(data.ctx, data.pubkey, &data.pubkeylen, data.key, 1)); run_benchmark("ecdsa_verify", benchmark_verify, NULL, NULL, &data, 10, 20000); - secp256k1_stop(); + secp256k1_context_destroy(data.ctx); return 0; } diff --git a/src/secp256k1/src/ecdsa.h b/src/secp256k1/src/ecdsa.h index c195e7afcb..4ef78e8afb 100644 --- a/src/secp256k1/src/ecdsa.h +++ b/src/secp256k1/src/ecdsa.h @@ -9,6 +9,7 @@ #include "scalar.h" #include "group.h" +#include "ecmult.h" typedef struct { secp256k1_scalar_t r, s; @@ -16,8 +17,8 @@ typedef struct { static int secp256k1_ecdsa_sig_parse(secp256k1_ecdsa_sig_t *r, const unsigned char *sig, int size); static int secp256k1_ecdsa_sig_serialize(unsigned char *sig, int *size, const secp256k1_ecdsa_sig_t *a); -static int secp256k1_ecdsa_sig_verify(const secp256k1_ecdsa_sig_t *sig, const secp256k1_ge_t *pubkey, const secp256k1_scalar_t *message); -static int secp256k1_ecdsa_sig_sign(secp256k1_ecdsa_sig_t *sig, const secp256k1_scalar_t *seckey, const secp256k1_scalar_t *message, const secp256k1_scalar_t *nonce, int *recid); -static int secp256k1_ecdsa_sig_recover(const secp256k1_ecdsa_sig_t *sig, secp256k1_ge_t *pubkey, const secp256k1_scalar_t *message, int recid); +static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context_t *ctx, const secp256k1_ecdsa_sig_t *sig, const secp256k1_ge_t *pubkey, const secp256k1_scalar_t *message); +static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context_t *ctx, secp256k1_ecdsa_sig_t *sig, const secp256k1_scalar_t *seckey, const secp256k1_scalar_t *message, const secp256k1_scalar_t *nonce, int *recid); +static int secp256k1_ecdsa_sig_recover(const secp256k1_ecmult_context_t *ctx, const secp256k1_ecdsa_sig_t *sig, secp256k1_ge_t *pubkey, const secp256k1_scalar_t *message, int recid); #endif diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index 1a77649390..ed1d228189 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -53,35 +53,59 @@ static int secp256k1_ecdsa_sig_parse(secp256k1_ecdsa_sig_t *r, const unsigned ch int lenr; int lens; int overflow; - if (sig[0] != 0x30) return 0; + if (sig[0] != 0x30) { + return 0; + } lenr = sig[3]; - if (5+lenr >= size) return 0; + if (5+lenr >= size) { + return 0; + } lens = sig[lenr+5]; - if (sig[1] != lenr+lens+4) return 0; - if (lenr+lens+6 > size) return 0; - if (sig[2] != 0x02) return 0; - if (lenr == 0) return 0; - if (sig[lenr+4] != 0x02) return 0; - if (lens == 0) return 0; + if (sig[1] != lenr+lens+4) { + return 0; + } + if (lenr+lens+6 > size) { + return 0; + } + if (sig[2] != 0x02) { + return 0; + } + if (lenr == 0) { + return 0; + } + if (sig[lenr+4] != 0x02) { + return 0; + } + if (lens == 0) { + return 0; + } sp = sig + 6 + lenr; while (lens > 0 && sp[0] == 0) { lens--; sp++; } - if (lens > 32) return 0; + if (lens > 32) { + return 0; + } rp = sig + 4; while (lenr > 0 && rp[0] == 0) { lenr--; rp++; } - if (lenr > 32) return 0; + if (lenr > 32) { + return 0; + } memcpy(ra + 32 - lenr, rp, lenr); memcpy(sa + 32 - lens, sp, lens); overflow = 0; secp256k1_scalar_set_b32(&r->r, ra, &overflow); - if (overflow) return 0; + if (overflow) { + return 0; + } secp256k1_scalar_set_b32(&r->s, sa, &overflow); - if (overflow) return 0; + if (overflow) { + return 0; + } return 1; } @@ -93,8 +117,9 @@ static int secp256k1_ecdsa_sig_serialize(unsigned char *sig, int *size, const se secp256k1_scalar_get_b32(&s[1], &a->s); while (lenR > 1 && rp[0] == 0 && rp[1] < 0x80) { lenR--; rp++; } while (lenS > 1 && sp[0] == 0 && sp[1] < 0x80) { lenS--; sp++; } - if (*size < 6+lenS+lenR) + if (*size < 6+lenS+lenR) { return 0; + } *size = 6 + lenS + lenR; sig[0] = 0x30; sig[1] = 4 + lenS + lenR; @@ -107,21 +132,22 @@ static int secp256k1_ecdsa_sig_serialize(unsigned char *sig, int *size, const se return 1; } -static int secp256k1_ecdsa_sig_verify(const secp256k1_ecdsa_sig_t *sig, const secp256k1_ge_t *pubkey, const secp256k1_scalar_t *message) { +static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context_t *ctx, const secp256k1_ecdsa_sig_t *sig, const secp256k1_ge_t *pubkey, const secp256k1_scalar_t *message) { unsigned char c[32]; secp256k1_scalar_t sn, u1, u2; secp256k1_fe_t xr; secp256k1_gej_t pubkeyj; secp256k1_gej_t pr; - if (secp256k1_scalar_is_zero(&sig->r) || secp256k1_scalar_is_zero(&sig->s)) + if (secp256k1_scalar_is_zero(&sig->r) || secp256k1_scalar_is_zero(&sig->s)) { return 0; + } secp256k1_scalar_inverse_var(&sn, &sig->s); secp256k1_scalar_mul(&u1, &sn, message); secp256k1_scalar_mul(&u2, &sn, &sig->r); secp256k1_gej_set_ge(&pubkeyj, pubkey); - secp256k1_ecmult(&pr, &pubkeyj, &u2, &u1); + secp256k1_ecmult(ctx, &pr, &pubkeyj, &u2, &u1); if (secp256k1_gej_is_infinity(&pr)) { return 0; } @@ -160,7 +186,7 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_ecdsa_sig_t *sig, const se return 0; } -static int secp256k1_ecdsa_sig_recover(const secp256k1_ecdsa_sig_t *sig, secp256k1_ge_t *pubkey, const secp256k1_scalar_t *message, int recid) { +static int secp256k1_ecdsa_sig_recover(const secp256k1_ecmult_context_t *ctx, const secp256k1_ecdsa_sig_t *sig, secp256k1_ge_t *pubkey, const secp256k1_scalar_t *message, int recid) { unsigned char brx[32]; secp256k1_fe_t fx; secp256k1_ge_t x; @@ -168,36 +194,39 @@ static int secp256k1_ecdsa_sig_recover(const secp256k1_ecdsa_sig_t *sig, secp256 secp256k1_scalar_t rn, u1, u2; secp256k1_gej_t qj; - if (secp256k1_scalar_is_zero(&sig->r) || secp256k1_scalar_is_zero(&sig->s)) + if (secp256k1_scalar_is_zero(&sig->r) || secp256k1_scalar_is_zero(&sig->s)) { return 0; + } secp256k1_scalar_get_b32(brx, &sig->r); VERIFY_CHECK(secp256k1_fe_set_b32(&fx, brx)); /* brx comes from a scalar, so is less than the order; certainly less than p */ if (recid & 2) { - if (secp256k1_fe_cmp_var(&fx, &secp256k1_ecdsa_const_p_minus_order) >= 0) + if (secp256k1_fe_cmp_var(&fx, &secp256k1_ecdsa_const_p_minus_order) >= 0) { return 0; + } secp256k1_fe_add(&fx, &secp256k1_ecdsa_const_order_as_fe); } - if (!secp256k1_ge_set_xo_var(&x, &fx, recid & 1)) + if (!secp256k1_ge_set_xo_var(&x, &fx, recid & 1)) { return 0; + } secp256k1_gej_set_ge(&xj, &x); secp256k1_scalar_inverse_var(&rn, &sig->r); secp256k1_scalar_mul(&u1, &rn, message); secp256k1_scalar_negate(&u1, &u1); secp256k1_scalar_mul(&u2, &rn, &sig->s); - secp256k1_ecmult(&qj, &xj, &u2, &u1); + secp256k1_ecmult(ctx, &qj, &xj, &u2, &u1); secp256k1_ge_set_gej_var(pubkey, &qj); return !secp256k1_gej_is_infinity(&qj); } -static int secp256k1_ecdsa_sig_sign(secp256k1_ecdsa_sig_t *sig, const secp256k1_scalar_t *seckey, const secp256k1_scalar_t *message, const secp256k1_scalar_t *nonce, int *recid) { +static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context_t *ctx, secp256k1_ecdsa_sig_t *sig, const secp256k1_scalar_t *seckey, const secp256k1_scalar_t *message, const secp256k1_scalar_t *nonce, int *recid) { unsigned char b[32]; secp256k1_gej_t rp; secp256k1_ge_t r; secp256k1_scalar_t n; int overflow = 0; - secp256k1_ecmult_gen(&rp, nonce); + secp256k1_ecmult_gen(ctx, &rp, nonce); secp256k1_ge_set_gej(&r, &rp); secp256k1_fe_normalize(&r.x); secp256k1_fe_normalize(&r.y); @@ -209,8 +238,9 @@ static int secp256k1_ecdsa_sig_sign(secp256k1_ecdsa_sig_t *sig, const secp256k1_ secp256k1_ge_clear(&r); return 0; } - if (recid) + if (recid) { *recid = (overflow ? 2 : 0) | (secp256k1_fe_is_odd(&r.y) ? 1 : 0); + } secp256k1_scalar_mul(&n, &sig->r, seckey); secp256k1_scalar_add(&n, &n, message); secp256k1_scalar_inverse(&sig->s, nonce); @@ -218,12 +248,14 @@ static int secp256k1_ecdsa_sig_sign(secp256k1_ecdsa_sig_t *sig, const secp256k1_ secp256k1_scalar_clear(&n); secp256k1_gej_clear(&rp); secp256k1_ge_clear(&r); - if (secp256k1_scalar_is_zero(&sig->s)) + if (secp256k1_scalar_is_zero(&sig->s)) { return 0; + } if (secp256k1_scalar_is_high(&sig->s)) { secp256k1_scalar_negate(&sig->s, &sig->s); - if (recid) + if (recid) { *recid ^= 1; + } } return 1; } diff --git a/src/secp256k1/src/eckey.h b/src/secp256k1/src/eckey.h index 6de5dc0a59..53b818485e 100644 --- a/src/secp256k1/src/eckey.h +++ b/src/secp256k1/src/eckey.h @@ -9,16 +9,18 @@ #include "group.h" #include "scalar.h" +#include "ecmult.h" +#include "ecmult_gen.h" static int secp256k1_eckey_pubkey_parse(secp256k1_ge_t *elem, const unsigned char *pub, int size); static int secp256k1_eckey_pubkey_serialize(secp256k1_ge_t *elem, unsigned char *pub, int *size, int compressed); static int secp256k1_eckey_privkey_parse(secp256k1_scalar_t *key, const unsigned char *privkey, int privkeylen); -static int secp256k1_eckey_privkey_serialize(unsigned char *privkey, int *privkeylen, const secp256k1_scalar_t *key, int compressed); +static int secp256k1_eckey_privkey_serialize(const secp256k1_ecmult_gen_context_t *ctx, unsigned char *privkey, int *privkeylen, const secp256k1_scalar_t *key, int compressed); static int secp256k1_eckey_privkey_tweak_add(secp256k1_scalar_t *key, const secp256k1_scalar_t *tweak); -static int secp256k1_eckey_pubkey_tweak_add(secp256k1_ge_t *key, const secp256k1_scalar_t *tweak); +static int secp256k1_eckey_pubkey_tweak_add(const secp256k1_ecmult_context_t *ctx, secp256k1_ge_t *key, const secp256k1_scalar_t *tweak); static int secp256k1_eckey_privkey_tweak_mul(secp256k1_scalar_t *key, const secp256k1_scalar_t *tweak); -static int secp256k1_eckey_pubkey_tweak_mul(secp256k1_ge_t *key, const secp256k1_scalar_t *tweak); +static int secp256k1_eckey_pubkey_tweak_mul(const secp256k1_ecmult_context_t *ctx, secp256k1_ge_t *key, const secp256k1_scalar_t *tweak); #endif diff --git a/src/secp256k1/src/eckey_impl.h b/src/secp256k1/src/eckey_impl.h index 3e06d05b47..a332bd34ec 100644 --- a/src/secp256k1/src/eckey_impl.h +++ b/src/secp256k1/src/eckey_impl.h @@ -24,8 +24,9 @@ static int secp256k1_eckey_pubkey_parse(secp256k1_ge_t *elem, const unsigned cha return 0; } secp256k1_ge_set_xy(elem, &x, &y); - if ((pub[0] == 0x06 || pub[0] == 0x07) && secp256k1_fe_is_odd(&y) != (pub[0] == 0x07)) + if ((pub[0] == 0x06 || pub[0] == 0x07) && secp256k1_fe_is_odd(&y) != (pub[0] == 0x07)) { return 0; + } return secp256k1_ge_is_valid_var(elem); } else { return 0; @@ -57,40 +58,47 @@ static int secp256k1_eckey_privkey_parse(secp256k1_scalar_t *key, const unsigned int len = 0; int overflow = 0; /* sequence header */ - if (end < privkey+1 || *privkey != 0x30) + if (end < privkey+1 || *privkey != 0x30) { return 0; + } privkey++; /* sequence length constructor */ - if (end < privkey+1 || !(*privkey & 0x80)) + if (end < privkey+1 || !(*privkey & 0x80)) { return 0; + } lenb = *privkey & ~0x80; privkey++; - if (lenb < 1 || lenb > 2) + if (lenb < 1 || lenb > 2) { return 0; - if (end < privkey+lenb) + } + if (end < privkey+lenb) { return 0; + } /* sequence length */ len = privkey[lenb-1] | (lenb > 1 ? privkey[lenb-2] << 8 : 0); privkey += lenb; - if (end < privkey+len) + if (end < privkey+len) { return 0; + } /* sequence element 0: version number (=1) */ - if (end < privkey+3 || privkey[0] != 0x02 || privkey[1] != 0x01 || privkey[2] != 0x01) + if (end < privkey+3 || privkey[0] != 0x02 || privkey[1] != 0x01 || privkey[2] != 0x01) { return 0; + } privkey += 3; /* sequence element 1: octet string, up to 32 bytes */ - if (end < privkey+2 || privkey[0] != 0x04 || privkey[1] > 0x20 || end < privkey+2+privkey[1]) + if (end < privkey+2 || privkey[0] != 0x04 || privkey[1] > 0x20 || end < privkey+2+privkey[1]) { return 0; + } memcpy(c + 32 - privkey[1], privkey + 2, privkey[1]); secp256k1_scalar_set_b32(key, c, &overflow); memset(c, 0, 32); return !overflow; } -static int secp256k1_eckey_privkey_serialize(unsigned char *privkey, int *privkeylen, const secp256k1_scalar_t *key, int compressed) { +static int secp256k1_eckey_privkey_serialize(const secp256k1_ecmult_gen_context_t *ctx, unsigned char *privkey, int *privkeylen, const secp256k1_scalar_t *key, int compressed) { secp256k1_gej_t rp; secp256k1_ge_t r; int pubkeylen = 0; - secp256k1_ecmult_gen(&rp, key); + secp256k1_ecmult_gen(ctx, &rp, key); secp256k1_ge_set_gej(&r, &rp); if (compressed) { static const unsigned char begin[] = { @@ -148,41 +156,45 @@ static int secp256k1_eckey_privkey_serialize(unsigned char *privkey, int *privke static int secp256k1_eckey_privkey_tweak_add(secp256k1_scalar_t *key, const secp256k1_scalar_t *tweak) { secp256k1_scalar_add(key, key, tweak); - if (secp256k1_scalar_is_zero(key)) + if (secp256k1_scalar_is_zero(key)) { return 0; + } return 1; } -static int secp256k1_eckey_pubkey_tweak_add(secp256k1_ge_t *key, const secp256k1_scalar_t *tweak) { +static int secp256k1_eckey_pubkey_tweak_add(const secp256k1_ecmult_context_t *ctx, secp256k1_ge_t *key, const secp256k1_scalar_t *tweak) { secp256k1_gej_t pt; secp256k1_scalar_t one; secp256k1_gej_set_ge(&pt, key); secp256k1_scalar_set_int(&one, 1); - secp256k1_ecmult(&pt, &pt, &one, tweak); + secp256k1_ecmult(ctx, &pt, &pt, &one, tweak); - if (secp256k1_gej_is_infinity(&pt)) + if (secp256k1_gej_is_infinity(&pt)) { return 0; + } secp256k1_ge_set_gej(key, &pt); return 1; } static int secp256k1_eckey_privkey_tweak_mul(secp256k1_scalar_t *key, const secp256k1_scalar_t *tweak) { - if (secp256k1_scalar_is_zero(tweak)) + if (secp256k1_scalar_is_zero(tweak)) { return 0; + } secp256k1_scalar_mul(key, key, tweak); return 1; } -static int secp256k1_eckey_pubkey_tweak_mul(secp256k1_ge_t *key, const secp256k1_scalar_t *tweak) { +static int secp256k1_eckey_pubkey_tweak_mul(const secp256k1_ecmult_context_t *ctx, secp256k1_ge_t *key, const secp256k1_scalar_t *tweak) { secp256k1_scalar_t zero; secp256k1_gej_t pt; - if (secp256k1_scalar_is_zero(tweak)) + if (secp256k1_scalar_is_zero(tweak)) { return 0; + } secp256k1_scalar_set_int(&zero, 0); secp256k1_gej_set_ge(&pt, key); - secp256k1_ecmult(&pt, &pt, tweak, &zero); + secp256k1_ecmult(ctx, &pt, &pt, tweak, &zero); secp256k1_ge_set_gej(key, &pt); return 1; } diff --git a/src/secp256k1/src/ecmult.h b/src/secp256k1/src/ecmult.h index 15a7100a4a..bab9e4ef52 100644 --- a/src/secp256k1/src/ecmult.h +++ b/src/secp256k1/src/ecmult.h @@ -10,10 +10,22 @@ #include "num.h" #include "group.h" -static void secp256k1_ecmult_start(void); -static void secp256k1_ecmult_stop(void); +typedef struct { + /* For accelerating the computation of a*P + b*G: */ + secp256k1_ge_storage_t (*pre_g)[]; /* odd multiples of the generator */ +#ifdef USE_ENDOMORPHISM + secp256k1_ge_storage_t (*pre_g_128)[]; /* odd multiples of 2^128*generator */ +#endif +} secp256k1_ecmult_context_t; + +static void secp256k1_ecmult_context_init(secp256k1_ecmult_context_t *ctx); +static void secp256k1_ecmult_context_build(secp256k1_ecmult_context_t *ctx); +static void secp256k1_ecmult_context_clone(secp256k1_ecmult_context_t *dst, + const secp256k1_ecmult_context_t *src); +static void secp256k1_ecmult_context_clear(secp256k1_ecmult_context_t *ctx); +static int secp256k1_ecmult_context_is_built(const secp256k1_ecmult_context_t *ctx); /** Double multiply: R = na*A + ng*G */ -static void secp256k1_ecmult(secp256k1_gej_t *r, const secp256k1_gej_t *a, const secp256k1_scalar_t *na, const secp256k1_scalar_t *ng); +static void secp256k1_ecmult(const secp256k1_ecmult_context_t *ctx, secp256k1_gej_t *r, const secp256k1_gej_t *a, const secp256k1_scalar_t *na, const secp256k1_scalar_t *ng); #endif diff --git a/src/secp256k1/src/ecmult_gen.h b/src/secp256k1/src/ecmult_gen.h index 42f822f9ce..3745633c47 100644 --- a/src/secp256k1/src/ecmult_gen.h +++ b/src/secp256k1/src/ecmult_gen.h @@ -10,10 +10,34 @@ #include "scalar.h" #include "group.h" -static void secp256k1_ecmult_gen_start(void); -static void secp256k1_ecmult_gen_stop(void); +typedef struct { + /* For accelerating the computation of a*G: + * To harden against timing attacks, use the following mechanism: + * * Break up the multiplicand into groups of 4 bits, called n_0, n_1, n_2, ..., n_63. + * * Compute sum(n_i * 16^i * G + U_i, i=0..63), where: + * * U_i = U * 2^i (for i=0..62) + * * U_i = U * (1-2^63) (for i=63) + * where U is a point with no known corresponding scalar. Note that sum(U_i, i=0..63) = 0. + * For each i, and each of the 16 possible values of n_i, (n_i * 16^i * G + U_i) is + * precomputed (call it prec(i, n_i)). The formula now becomes sum(prec(i, n_i), i=0..63). + * None of the resulting prec group elements have a known scalar, and neither do any of + * the intermediate sums while computing a*G. + */ + secp256k1_ge_storage_t (*prec)[64][16]; /* prec[j][i] = 16^j * i * G + U_i */ + secp256k1_scalar_t blind; + secp256k1_gej_t initial; +} secp256k1_ecmult_gen_context_t; + +static void secp256k1_ecmult_gen_context_init(secp256k1_ecmult_gen_context_t* ctx); +static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context_t* ctx); +static void secp256k1_ecmult_gen_context_clone(secp256k1_ecmult_gen_context_t *dst, + const secp256k1_ecmult_gen_context_t* src); +static void secp256k1_ecmult_gen_context_clear(secp256k1_ecmult_gen_context_t* ctx); +static int secp256k1_ecmult_gen_context_is_built(const secp256k1_ecmult_gen_context_t* ctx); /** Multiply with the generator: R = a*G */ -static void secp256k1_ecmult_gen(secp256k1_gej_t *r, const secp256k1_scalar_t *a); +static void secp256k1_ecmult_gen(const secp256k1_ecmult_gen_context_t* ctx, secp256k1_gej_t *r, const secp256k1_scalar_t *a); + +static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context_t *ctx, const unsigned char *seed32); #endif diff --git a/src/secp256k1/src/ecmult_gen_impl.h b/src/secp256k1/src/ecmult_gen_impl.h index 849452c7a1..4697753ac8 100644 --- a/src/secp256k1/src/ecmult_gen_impl.h +++ b/src/secp256k1/src/ecmult_gen_impl.h @@ -1,5 +1,5 @@ /********************************************************************** - * Copyright (c) 2013, 2014 Pieter Wuille * + * Copyright (c) 2013, 2014, 2015 Pieter Wuille, Gregory Maxwell * * Distributed under the MIT software license, see the accompanying * * file COPYING or http://www.opensource.org/licenses/mit-license.php.* **********************************************************************/ @@ -10,36 +10,23 @@ #include "scalar.h" #include "group.h" #include "ecmult_gen.h" +#include "hash_impl.h" -typedef struct { - /* For accelerating the computation of a*G: - * To harden against timing attacks, use the following mechanism: - * * Break up the multiplicand into groups of 4 bits, called n_0, n_1, n_2, ..., n_63. - * * Compute sum(n_i * 16^i * G + U_i, i=0..63), where: - * * U_i = U * 2^i (for i=0..62) - * * U_i = U * (1-2^63) (for i=63) - * where U is a point with no known corresponding scalar. Note that sum(U_i, i=0..63) = 0. - * For each i, and each of the 16 possible values of n_i, (n_i * 16^i * G + U_i) is - * precomputed (call it prec(i, n_i)). The formula now becomes sum(prec(i, n_i), i=0..63). - * None of the resulting prec group elements have a known scalar, and neither do any of - * the intermediate sums while computing a*G. - */ - secp256k1_ge_storage_t prec[64][16]; /* prec[j][i] = 16^j * i * G + U_i */ -} secp256k1_ecmult_gen_consts_t; - -static const secp256k1_ecmult_gen_consts_t *secp256k1_ecmult_gen_consts = NULL; +static void secp256k1_ecmult_gen_context_init(secp256k1_ecmult_gen_context_t *ctx) { + ctx->prec = NULL; +} -static void secp256k1_ecmult_gen_start(void) { +static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context_t *ctx) { secp256k1_ge_t prec[1024]; secp256k1_gej_t gj; secp256k1_gej_t nums_gej; - secp256k1_ecmult_gen_consts_t *ret; int i, j; - if (secp256k1_ecmult_gen_consts != NULL) + + if (ctx->prec != NULL) { return; + } - /* Allocate the precomputation table. */ - ret = (secp256k1_ecmult_gen_consts_t*)checked_malloc(sizeof(secp256k1_ecmult_gen_consts_t)); + ctx->prec = (secp256k1_ge_storage_t (*)[64][16])checked_malloc(sizeof(*ctx->prec)); /* get the generator */ secp256k1_gej_set_ge(&gj, &secp256k1_ge_const_g); @@ -85,42 +72,113 @@ static void secp256k1_ecmult_gen_start(void) { } for (j = 0; j < 64; j++) { for (i = 0; i < 16; i++) { - secp256k1_ge_to_storage(&ret->prec[j][i], &prec[j*16 + i]); + secp256k1_ge_to_storage(&(*ctx->prec)[j][i], &prec[j*16 + i]); } } + secp256k1_ecmult_gen_blind(ctx, NULL); +} - /* Set the global pointer to the precomputation table. */ - secp256k1_ecmult_gen_consts = ret; +static int secp256k1_ecmult_gen_context_is_built(const secp256k1_ecmult_gen_context_t* ctx) { + return ctx->prec != NULL; } -static void secp256k1_ecmult_gen_stop(void) { - secp256k1_ecmult_gen_consts_t *c; - if (secp256k1_ecmult_gen_consts == NULL) - return; +static void secp256k1_ecmult_gen_context_clone(secp256k1_ecmult_gen_context_t *dst, + const secp256k1_ecmult_gen_context_t *src) { + if (src->prec == NULL) { + dst->prec = NULL; + } else { + dst->prec = (secp256k1_ge_storage_t (*)[64][16])checked_malloc(sizeof(*dst->prec)); + memcpy(dst->prec, src->prec, sizeof(*dst->prec)); + dst->initial = src->initial; + dst->blind = src->blind; + } +} - c = (secp256k1_ecmult_gen_consts_t*)secp256k1_ecmult_gen_consts; - secp256k1_ecmult_gen_consts = NULL; - free(c); +static void secp256k1_ecmult_gen_context_clear(secp256k1_ecmult_gen_context_t *ctx) { + free(ctx->prec); + secp256k1_scalar_clear(&ctx->blind); + secp256k1_gej_clear(&ctx->initial); + ctx->prec = NULL; } -static void secp256k1_ecmult_gen(secp256k1_gej_t *r, const secp256k1_scalar_t *gn) { - const secp256k1_ecmult_gen_consts_t *c = secp256k1_ecmult_gen_consts; +static void secp256k1_ecmult_gen(const secp256k1_ecmult_gen_context_t *ctx, secp256k1_gej_t *r, const secp256k1_scalar_t *gn) { secp256k1_ge_t add; secp256k1_ge_storage_t adds; + secp256k1_scalar_t gnb; int bits; int i, j; - secp256k1_gej_set_infinity(r); + memset(&adds, 0, sizeof(adds)); + *r = ctx->initial; + /* Blind scalar/point multiplication by computing (n-b)G + bG instead of nG. */ + secp256k1_scalar_add(&gnb, gn, &ctx->blind); add.infinity = 0; for (j = 0; j < 64; j++) { - bits = secp256k1_scalar_get_bits(gn, j * 4, 4); + bits = secp256k1_scalar_get_bits(&gnb, j * 4, 4); for (i = 0; i < 16; i++) { - secp256k1_ge_storage_cmov(&adds, &c->prec[j][i], i == bits); + /** This uses a conditional move to avoid any secret data in array indexes. + * _Any_ use of secret indexes has been demonstrated to result in timing + * sidechannels, even when the cache-line access patterns are uniform. + * See also: + * "A word of warning", CHES 2013 Rump Session, by Daniel J. Bernstein and Peter Schwabe + * (https://cryptojedi.org/peter/data/chesrump-20130822.pdf) and + * "Cache Attacks and Countermeasures: the Case of AES", RSA 2006, + * by Dag Arne Osvik, Adi Shamir, and Eran Tromer + * (http://www.tau.ac.il/~tromer/papers/cache.pdf) + */ + secp256k1_ge_storage_cmov(&adds, &(*ctx->prec)[j][i], i == bits); } secp256k1_ge_from_storage(&add, &adds); secp256k1_gej_add_ge(r, r, &add); } bits = 0; secp256k1_ge_clear(&add); + secp256k1_scalar_clear(&gnb); +} + +/* Setup blinding values for secp256k1_ecmult_gen. */ +static void secp256k1_ecmult_gen_blind(secp256k1_ecmult_gen_context_t *ctx, const unsigned char *seed32) { + secp256k1_scalar_t b; + secp256k1_gej_t gb; + secp256k1_fe_t s; + unsigned char nonce32[32]; + secp256k1_rfc6979_hmac_sha256_t rng; + int retry; + if (!seed32) { + /* When seed is NULL, reset the initial point and blinding value. */ + secp256k1_gej_set_ge(&ctx->initial, &secp256k1_ge_const_g); + secp256k1_gej_neg(&ctx->initial, &ctx->initial); + secp256k1_scalar_set_int(&ctx->blind, 1); + } + /* The prior blinding value (if not reset) is chained forward by including it in the hash. */ + secp256k1_scalar_get_b32(nonce32, &ctx->blind); + /** Using a CSPRNG allows a failure free interface, avoids needing large amounts of random data, + * and guards against weak or adversarial seeds. This is a simpler and safer interface than + * asking the caller for blinding values directly and expecting them to retry on failure. + */ + secp256k1_rfc6979_hmac_sha256_initialize(&rng, seed32 ? seed32 : nonce32, 32, nonce32, 32, NULL, 0); + /* Retry for out of range results to achieve uniformity. */ + do { + secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); + retry = !secp256k1_fe_set_b32(&s, nonce32); + retry |= secp256k1_fe_is_zero(&s); + } while (retry); + /* Randomize the projection to defend against multiplier sidechannels. */ + secp256k1_gej_rescale(&ctx->initial, &s); + secp256k1_fe_clear(&s); + do { + secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); + secp256k1_scalar_set_b32(&b, nonce32, &retry); + /* A blinding value of 0 works, but would undermine the projection hardening. */ + retry |= secp256k1_scalar_is_zero(&b); + } while (retry); + secp256k1_rfc6979_hmac_sha256_finalize(&rng); + memset(nonce32, 0, 32); + secp256k1_ecmult_gen(ctx, &gb, &b); + secp256k1_scalar_negate(&b, &b); + ctx->blind = b; + ctx->initial = gb; + secp256k1_scalar_clear(&b); + secp256k1_gej_clear(&gb); } #endif diff --git a/src/secp256k1/src/ecmult_impl.h b/src/secp256k1/src/ecmult_impl.h index ece0b0a459..1b2856f83d 100644 --- a/src/secp256k1/src/ecmult_impl.h +++ b/src/secp256k1/src/ecmult_impl.h @@ -41,16 +41,17 @@ static void secp256k1_ecmult_table_precomp_gej_var(secp256k1_gej_t *pre, const s int i; pre[0] = *a; secp256k1_gej_double_var(&d, &pre[0]); - for (i = 1; i < (1 << (w-2)); i++) + for (i = 1; i < (1 << (w-2)); i++) { secp256k1_gej_add_var(&pre[i], &d, &pre[i-1]); + } } static void secp256k1_ecmult_table_precomp_ge_storage_var(secp256k1_ge_storage_t *pre, const secp256k1_gej_t *a, int w) { secp256k1_gej_t d; int i; const int table_size = 1 << (w-2); - secp256k1_gej_t *prej = checked_malloc(sizeof(secp256k1_gej_t) * table_size); - secp256k1_ge_t *prea = checked_malloc(sizeof(secp256k1_ge_t) * table_size); + secp256k1_gej_t *prej = (secp256k1_gej_t *)checked_malloc(sizeof(secp256k1_gej_t) * table_size); + secp256k1_ge_t *prea = (secp256k1_ge_t *)checked_malloc(sizeof(secp256k1_ge_t) * table_size); prej[0] = *a; secp256k1_gej_double_var(&d, a); for (i = 1; i < table_size; i++) { @@ -73,73 +74,93 @@ static void secp256k1_ecmult_table_precomp_ge_storage_var(secp256k1_ge_storage_t VERIFY_CHECK(((n) & 1) == 1); \ VERIFY_CHECK((n) >= -((1 << ((w)-1)) - 1)); \ VERIFY_CHECK((n) <= ((1 << ((w)-1)) - 1)); \ - if ((n) > 0) \ + if ((n) > 0) { \ *(r) = (pre)[((n)-1)/2]; \ - else \ + } else { \ secp256k1_gej_neg((r), &(pre)[(-(n)-1)/2]); \ + } \ } while(0) #define ECMULT_TABLE_GET_GE_STORAGE(r,pre,n,w) do { \ VERIFY_CHECK(((n) & 1) == 1); \ VERIFY_CHECK((n) >= -((1 << ((w)-1)) - 1)); \ VERIFY_CHECK((n) <= ((1 << ((w)-1)) - 1)); \ - if ((n) > 0) \ + if ((n) > 0) { \ secp256k1_ge_from_storage((r), &(pre)[((n)-1)/2]); \ - else {\ + } else { \ secp256k1_ge_from_storage((r), &(pre)[(-(n)-1)/2]); \ secp256k1_ge_neg((r), (r)); \ } \ } while(0) -typedef struct { - /* For accelerating the computation of a*P + b*G: */ - secp256k1_ge_storage_t pre_g[ECMULT_TABLE_SIZE(WINDOW_G)]; /* odd multiples of the generator */ +static void secp256k1_ecmult_context_init(secp256k1_ecmult_context_t *ctx) { + ctx->pre_g = NULL; #ifdef USE_ENDOMORPHISM - secp256k1_ge_storage_t pre_g_128[ECMULT_TABLE_SIZE(WINDOW_G)]; /* odd multiples of 2^128*generator */ + ctx->pre_g_128 = NULL; #endif -} secp256k1_ecmult_consts_t; - -static const secp256k1_ecmult_consts_t *secp256k1_ecmult_consts = NULL; +} -static void secp256k1_ecmult_start(void) { +static void secp256k1_ecmult_context_build(secp256k1_ecmult_context_t *ctx) { secp256k1_gej_t gj; - secp256k1_ecmult_consts_t *ret; - if (secp256k1_ecmult_consts != NULL) - return; - /* Allocate the precomputation table. */ - ret = (secp256k1_ecmult_consts_t*)checked_malloc(sizeof(secp256k1_ecmult_consts_t)); + if (ctx->pre_g != NULL) { + return; + } /* get the generator */ secp256k1_gej_set_ge(&gj, &secp256k1_ge_const_g); + ctx->pre_g = (secp256k1_ge_storage_t (*)[])checked_malloc(sizeof((*ctx->pre_g)[0]) * ECMULT_TABLE_SIZE(WINDOW_G)); /* precompute the tables with odd multiples */ - secp256k1_ecmult_table_precomp_ge_storage_var(ret->pre_g, &gj, WINDOW_G); + secp256k1_ecmult_table_precomp_ge_storage_var(*ctx->pre_g, &gj, WINDOW_G); #ifdef USE_ENDOMORPHISM { secp256k1_gej_t g_128j; int i; + + ctx->pre_g_128 = (secp256k1_ge_storage_t (*)[])checked_malloc(sizeof((*ctx->pre_g_128)[0]) * ECMULT_TABLE_SIZE(WINDOW_G)); + /* calculate 2^128*generator */ g_128j = gj; - for (i = 0; i < 128; i++) + for (i = 0; i < 128; i++) { secp256k1_gej_double_var(&g_128j, &g_128j); - secp256k1_ecmult_table_precomp_ge_storage_var(ret->pre_g_128, &g_128j, WINDOW_G); + } + secp256k1_ecmult_table_precomp_ge_storage_var(*ctx->pre_g_128, &g_128j, WINDOW_G); } #endif +} - /* Set the global pointer to the precomputation table. */ - secp256k1_ecmult_consts = ret; +static void secp256k1_ecmult_context_clone(secp256k1_ecmult_context_t *dst, + const secp256k1_ecmult_context_t *src) { + if (src->pre_g == NULL) { + dst->pre_g = NULL; + } else { + size_t size = sizeof((*dst->pre_g)[0]) * ECMULT_TABLE_SIZE(WINDOW_G); + dst->pre_g = (secp256k1_ge_storage_t (*)[])checked_malloc(size); + memcpy(dst->pre_g, src->pre_g, size); + } +#ifdef USE_ENDOMORPHISM + if (src->pre_g_128 == NULL) { + dst->pre_g_128 = NULL; + } else { + size_t size = sizeof((*dst->pre_g_128)[0]) * ECMULT_TABLE_SIZE(WINDOW_G); + dst->pre_g_128 = (secp256k1_ge_storage_t (*)[])checked_malloc(size); + memcpy(dst->pre_g_128, src->pre_g_128, size); + } +#endif } -static void secp256k1_ecmult_stop(void) { - secp256k1_ecmult_consts_t *c; - if (secp256k1_ecmult_consts == NULL) - return; +static int secp256k1_ecmult_context_is_built(const secp256k1_ecmult_context_t *ctx) { + return ctx->pre_g != NULL; +} - c = (secp256k1_ecmult_consts_t*)secp256k1_ecmult_consts; - secp256k1_ecmult_consts = NULL; - free(c); +static void secp256k1_ecmult_context_clear(secp256k1_ecmult_context_t *ctx) { + free(ctx->pre_g); +#ifdef USE_ENDOMORPHISM + free(ctx->pre_g_128); +#endif + secp256k1_ecmult_context_init(ctx); } /** Convert a number to WNAF notation. The number becomes represented by sum(2^i * wnaf[i], i=0..bits), @@ -186,11 +207,10 @@ static int secp256k1_ecmult_wnaf(int *wnaf, const secp256k1_scalar_t *a, int w) return set_bits; } -static void secp256k1_ecmult(secp256k1_gej_t *r, const secp256k1_gej_t *a, const secp256k1_scalar_t *na, const secp256k1_scalar_t *ng) { +static void secp256k1_ecmult(const secp256k1_ecmult_context_t *ctx, secp256k1_gej_t *r, const secp256k1_gej_t *a, const secp256k1_scalar_t *na, const secp256k1_scalar_t *ng) { secp256k1_gej_t tmpj; secp256k1_gej_t pre_a[ECMULT_TABLE_SIZE(WINDOW_A)]; secp256k1_ge_t tmpa; - const secp256k1_ecmult_consts_t *c = secp256k1_ecmult_consts; #ifdef USE_ENDOMORPHISM secp256k1_gej_t pre_a_lam[ECMULT_TABLE_SIZE(WINDOW_A)]; secp256k1_scalar_t na_1, na_lam; @@ -223,7 +243,9 @@ static void secp256k1_ecmult(secp256k1_gej_t *r, const secp256k1_gej_t *a, const VERIFY_CHECK(bits_na_1 <= 130); VERIFY_CHECK(bits_na_lam <= 130); bits = bits_na_1; - if (bits_na_lam > bits) bits = bits_na_lam; + if (bits_na_lam > bits) { + bits = bits_na_lam; + } #else /* build wnaf representation for na. */ bits_na = secp256k1_ecmult_wnaf(wnaf_na, na, WINDOW_A); @@ -234,8 +256,9 @@ static void secp256k1_ecmult(secp256k1_gej_t *r, const secp256k1_gej_t *a, const secp256k1_ecmult_table_precomp_gej_var(pre_a, a, WINDOW_A); #ifdef USE_ENDOMORPHISM - for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) + for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) { secp256k1_gej_mul_lambda(&pre_a_lam[i], &pre_a[i]); + } /* split ng into ng_1 and ng_128 (where gn = gn_1 + gn_128*2^128, and gn_1 and gn_128 are ~128 bit) */ secp256k1_scalar_split_128(&ng_1, &ng_128, ng); @@ -243,11 +266,17 @@ static void secp256k1_ecmult(secp256k1_gej_t *r, const secp256k1_gej_t *a, const /* Build wnaf representation for ng_1 and ng_128 */ bits_ng_1 = secp256k1_ecmult_wnaf(wnaf_ng_1, &ng_1, WINDOW_G); bits_ng_128 = secp256k1_ecmult_wnaf(wnaf_ng_128, &ng_128, WINDOW_G); - if (bits_ng_1 > bits) bits = bits_ng_1; - if (bits_ng_128 > bits) bits = bits_ng_128; + if (bits_ng_1 > bits) { + bits = bits_ng_1; + } + if (bits_ng_128 > bits) { + bits = bits_ng_128; + } #else bits_ng = secp256k1_ecmult_wnaf(wnaf_ng, ng, WINDOW_G); - if (bits_ng > bits) bits = bits_ng; + if (bits_ng > bits) { + bits = bits_ng; + } #endif secp256k1_gej_set_infinity(r); @@ -265,11 +294,11 @@ static void secp256k1_ecmult(secp256k1_gej_t *r, const secp256k1_gej_t *a, const secp256k1_gej_add_var(r, r, &tmpj); } if (i < bits_ng_1 && (n = wnaf_ng_1[i])) { - ECMULT_TABLE_GET_GE_STORAGE(&tmpa, c->pre_g, n, WINDOW_G); + ECMULT_TABLE_GET_GE_STORAGE(&tmpa, *ctx->pre_g, n, WINDOW_G); secp256k1_gej_add_ge_var(r, r, &tmpa); } if (i < bits_ng_128 && (n = wnaf_ng_128[i])) { - ECMULT_TABLE_GET_GE_STORAGE(&tmpa, c->pre_g_128, n, WINDOW_G); + ECMULT_TABLE_GET_GE_STORAGE(&tmpa, *ctx->pre_g_128, n, WINDOW_G); secp256k1_gej_add_ge_var(r, r, &tmpa); } #else @@ -278,7 +307,7 @@ static void secp256k1_ecmult(secp256k1_gej_t *r, const secp256k1_gej_t *a, const secp256k1_gej_add_var(r, r, &tmpj); } if (i < bits_ng && (n = wnaf_ng[i])) { - ECMULT_TABLE_GET_GE_STORAGE(&tmpa, c->pre_g, n, WINDOW_G); + ECMULT_TABLE_GET_GE_STORAGE(&tmpa, *ctx->pre_g, n, WINDOW_G); secp256k1_gej_add_ge_var(r, r, &tmpa); } #endif diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index 9e6d7d3c04..41b280892d 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -113,4 +113,7 @@ static void secp256k1_fe_from_storage(secp256k1_fe_t *r, const secp256k1_fe_stor /** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. */ static void secp256k1_fe_storage_cmov(secp256k1_fe_storage_t *r, const secp256k1_fe_storage_t *a, int flag); +/** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. */ +static void secp256k1_fe_cmov(secp256k1_fe_t *r, const secp256k1_fe_t *a, int flag); + #endif diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 0afbb18a4a..871b91f912 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -236,8 +236,9 @@ static int secp256k1_fe_normalizes_to_zero_var(secp256k1_fe_t *r) { z1 = z0 ^ 0x3D0UL; /* Fast return path should catch the majority of cases */ - if ((z0 != 0UL) & (z1 != 0x3FFFFFFUL)) + if ((z0 != 0UL) & (z1 != 0x3FFFFFFUL)) { return 0; + } t1 = r->n[1]; t2 = r->n[2]; @@ -315,8 +316,12 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe_t *a, const secp256k1_fe_t *b secp256k1_fe_verify(b); #endif for (i = 9; i >= 0; i--) { - if (a->n[i] > b->n[i]) return 1; - if (a->n[i] < b->n[i]) return -1; + if (a->n[i] > b->n[i]) { + return 1; + } + if (a->n[i] < b->n[i]) { + return -1; + } } return 0; } @@ -1063,6 +1068,26 @@ static void secp256k1_fe_sqr(secp256k1_fe_t *r, const secp256k1_fe_t *a) { #endif } +static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe_t *r, const secp256k1_fe_t *a, int flag) { + uint32_t mask0, mask1; + mask0 = flag + ~((uint32_t)0); + mask1 = ~mask0; + r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1); + r->n[1] = (r->n[1] & mask0) | (a->n[1] & mask1); + r->n[2] = (r->n[2] & mask0) | (a->n[2] & mask1); + r->n[3] = (r->n[3] & mask0) | (a->n[3] & mask1); + r->n[4] = (r->n[4] & mask0) | (a->n[4] & mask1); + r->n[5] = (r->n[5] & mask0) | (a->n[5] & mask1); + r->n[6] = (r->n[6] & mask0) | (a->n[6] & mask1); + r->n[7] = (r->n[7] & mask0) | (a->n[7] & mask1); + r->n[8] = (r->n[8] & mask0) | (a->n[8] & mask1); + r->n[9] = (r->n[9] & mask0) | (a->n[9] & mask1); +#ifdef VERIFY + r->magnitude = (r->magnitude & mask0) | (a->magnitude & mask1); + r->normalized = (r->normalized & mask0) | (a->normalized & mask1); +#endif +} + static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage_t *r, const secp256k1_fe_storage_t *a, int flag) { uint32_t mask0, mask1; mask0 = flag + ~((uint32_t)0); diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index 2f9c8704a8..bda4c3dfc2 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -209,8 +209,9 @@ static int secp256k1_fe_normalizes_to_zero_var(secp256k1_fe_t *r) { z1 = z0 ^ 0x1000003D0ULL; /* Fast return path should catch the majority of cases */ - if ((z0 != 0ULL) & (z1 != 0xFFFFFFFFFFFFFULL)) + if ((z0 != 0ULL) & (z1 != 0xFFFFFFFFFFFFFULL)) { return 0; + } t1 = r->n[1]; t2 = r->n[2]; @@ -277,8 +278,12 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe_t *a, const secp256k1_fe_t *b secp256k1_fe_verify(b); #endif for (i = 4; i >= 0; i--) { - if (a->n[i] > b->n[i]) return 1; - if (a->n[i] < b->n[i]) return -1; + if (a->n[i] > b->n[i]) { + return 1; + } + if (a->n[i] < b->n[i]) { + return -1; + } } return 0; } @@ -399,6 +404,21 @@ static void secp256k1_fe_sqr(secp256k1_fe_t *r, const secp256k1_fe_t *a) { #endif } +static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe_t *r, const secp256k1_fe_t *a, int flag) { + uint64_t mask0, mask1; + mask0 = flag + ~((uint64_t)0); + mask1 = ~mask0; + r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1); + r->n[1] = (r->n[1] & mask0) | (a->n[1] & mask1); + r->n[2] = (r->n[2] & mask0) | (a->n[2] & mask1); + r->n[3] = (r->n[3] & mask0) | (a->n[3] & mask1); + r->n[4] = (r->n[4] & mask0) | (a->n[4] & mask1); +#ifdef VERIFY + r->magnitude = (r->magnitude & mask0) | (a->magnitude & mask1); + r->normalized = (r->normalized & mask0) | (a->normalized & mask1); +#endif +} + static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage_t *r, const secp256k1_fe_storage_t *a, int flag) { uint64_t mask0, mask1; mask0 = flag + ~((uint64_t)0); diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 047914cf28..e6ec11e8f2 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -44,47 +44,69 @@ static int secp256k1_fe_sqrt_var(secp256k1_fe_t *r, const secp256k1_fe_t *a) { secp256k1_fe_mul(&x3, &x3, a); x6 = x3; - for (j=0; j<3; j++) secp256k1_fe_sqr(&x6, &x6); + for (j=0; j<3; j++) { + secp256k1_fe_sqr(&x6, &x6); + } secp256k1_fe_mul(&x6, &x6, &x3); x9 = x6; - for (j=0; j<3; j++) secp256k1_fe_sqr(&x9, &x9); + for (j=0; j<3; j++) { + secp256k1_fe_sqr(&x9, &x9); + } secp256k1_fe_mul(&x9, &x9, &x3); x11 = x9; - for (j=0; j<2; j++) secp256k1_fe_sqr(&x11, &x11); + for (j=0; j<2; j++) { + secp256k1_fe_sqr(&x11, &x11); + } secp256k1_fe_mul(&x11, &x11, &x2); x22 = x11; - for (j=0; j<11; j++) secp256k1_fe_sqr(&x22, &x22); + for (j=0; j<11; j++) { + secp256k1_fe_sqr(&x22, &x22); + } secp256k1_fe_mul(&x22, &x22, &x11); x44 = x22; - for (j=0; j<22; j++) secp256k1_fe_sqr(&x44, &x44); + for (j=0; j<22; j++) { + secp256k1_fe_sqr(&x44, &x44); + } secp256k1_fe_mul(&x44, &x44, &x22); x88 = x44; - for (j=0; j<44; j++) secp256k1_fe_sqr(&x88, &x88); + for (j=0; j<44; j++) { + secp256k1_fe_sqr(&x88, &x88); + } secp256k1_fe_mul(&x88, &x88, &x44); x176 = x88; - for (j=0; j<88; j++) secp256k1_fe_sqr(&x176, &x176); + for (j=0; j<88; j++) { + secp256k1_fe_sqr(&x176, &x176); + } secp256k1_fe_mul(&x176, &x176, &x88); x220 = x176; - for (j=0; j<44; j++) secp256k1_fe_sqr(&x220, &x220); + for (j=0; j<44; j++) { + secp256k1_fe_sqr(&x220, &x220); + } secp256k1_fe_mul(&x220, &x220, &x44); x223 = x220; - for (j=0; j<3; j++) secp256k1_fe_sqr(&x223, &x223); + for (j=0; j<3; j++) { + secp256k1_fe_sqr(&x223, &x223); + } secp256k1_fe_mul(&x223, &x223, &x3); /* The final result is then assembled using a sliding window over the blocks. */ t1 = x223; - for (j=0; j<23; j++) secp256k1_fe_sqr(&t1, &t1); + for (j=0; j<23; j++) { + secp256k1_fe_sqr(&t1, &t1); + } secp256k1_fe_mul(&t1, &t1, &x22); - for (j=0; j<6; j++) secp256k1_fe_sqr(&t1, &t1); + for (j=0; j<6; j++) { + secp256k1_fe_sqr(&t1, &t1); + } secp256k1_fe_mul(&t1, &t1, &x2); secp256k1_fe_sqr(&t1, &t1); secp256k1_fe_sqr(r, &t1); @@ -111,51 +133,77 @@ static void secp256k1_fe_inv(secp256k1_fe_t *r, const secp256k1_fe_t *a) { secp256k1_fe_mul(&x3, &x3, a); x6 = x3; - for (j=0; j<3; j++) secp256k1_fe_sqr(&x6, &x6); + for (j=0; j<3; j++) { + secp256k1_fe_sqr(&x6, &x6); + } secp256k1_fe_mul(&x6, &x6, &x3); x9 = x6; - for (j=0; j<3; j++) secp256k1_fe_sqr(&x9, &x9); + for (j=0; j<3; j++) { + secp256k1_fe_sqr(&x9, &x9); + } secp256k1_fe_mul(&x9, &x9, &x3); x11 = x9; - for (j=0; j<2; j++) secp256k1_fe_sqr(&x11, &x11); + for (j=0; j<2; j++) { + secp256k1_fe_sqr(&x11, &x11); + } secp256k1_fe_mul(&x11, &x11, &x2); x22 = x11; - for (j=0; j<11; j++) secp256k1_fe_sqr(&x22, &x22); + for (j=0; j<11; j++) { + secp256k1_fe_sqr(&x22, &x22); + } secp256k1_fe_mul(&x22, &x22, &x11); x44 = x22; - for (j=0; j<22; j++) secp256k1_fe_sqr(&x44, &x44); + for (j=0; j<22; j++) { + secp256k1_fe_sqr(&x44, &x44); + } secp256k1_fe_mul(&x44, &x44, &x22); x88 = x44; - for (j=0; j<44; j++) secp256k1_fe_sqr(&x88, &x88); + for (j=0; j<44; j++) { + secp256k1_fe_sqr(&x88, &x88); + } secp256k1_fe_mul(&x88, &x88, &x44); x176 = x88; - for (j=0; j<88; j++) secp256k1_fe_sqr(&x176, &x176); + for (j=0; j<88; j++) { + secp256k1_fe_sqr(&x176, &x176); + } secp256k1_fe_mul(&x176, &x176, &x88); x220 = x176; - for (j=0; j<44; j++) secp256k1_fe_sqr(&x220, &x220); + for (j=0; j<44; j++) { + secp256k1_fe_sqr(&x220, &x220); + } secp256k1_fe_mul(&x220, &x220, &x44); x223 = x220; - for (j=0; j<3; j++) secp256k1_fe_sqr(&x223, &x223); + for (j=0; j<3; j++) { + secp256k1_fe_sqr(&x223, &x223); + } secp256k1_fe_mul(&x223, &x223, &x3); /* The final result is then assembled using a sliding window over the blocks. */ t1 = x223; - for (j=0; j<23; j++) secp256k1_fe_sqr(&t1, &t1); + for (j=0; j<23; j++) { + secp256k1_fe_sqr(&t1, &t1); + } secp256k1_fe_mul(&t1, &t1, &x22); - for (j=0; j<5; j++) secp256k1_fe_sqr(&t1, &t1); + for (j=0; j<5; j++) { + secp256k1_fe_sqr(&t1, &t1); + } secp256k1_fe_mul(&t1, &t1, a); - for (j=0; j<3; j++) secp256k1_fe_sqr(&t1, &t1); + for (j=0; j<3; j++) { + secp256k1_fe_sqr(&t1, &t1); + } secp256k1_fe_mul(&t1, &t1, &x2); - for (j=0; j<2; j++) secp256k1_fe_sqr(&t1, &t1); + for (j=0; j<2; j++) { + secp256k1_fe_sqr(&t1, &t1); + } secp256k1_fe_mul(r, a, &t1); } @@ -188,8 +236,9 @@ static void secp256k1_fe_inv_var(secp256k1_fe_t *r, const secp256k1_fe_t *a) { static void secp256k1_fe_inv_all_var(size_t len, secp256k1_fe_t *r, const secp256k1_fe_t *a) { secp256k1_fe_t u; size_t i; - if (len < 1) + if (len < 1) { return; + } VERIFY_CHECK((r + len <= a) || (a + len <= r)); diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index d1e5834909..0b08b3b991 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -115,4 +115,7 @@ static void secp256k1_ge_from_storage(secp256k1_ge_t *r, const secp256k1_ge_stor /** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. */ static void secp256k1_ge_storage_cmov(secp256k1_ge_storage_t *r, const secp256k1_ge_storage_t *a, int flag); +/** Rescale a jacobian point by b which must be non-zero. Constant-time. */ +static void secp256k1_gej_rescale(secp256k1_gej_t *r, const secp256k1_fe_t *b); + #endif diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 8d8c359c5a..0f64576fbb 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -77,14 +77,14 @@ static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge_t *r, const se secp256k1_fe_t *azi; size_t i; size_t count = 0; - az = checked_malloc(sizeof(secp256k1_fe_t) * len); + az = (secp256k1_fe_t *)checked_malloc(sizeof(secp256k1_fe_t) * len); for (i = 0; i < len; i++) { if (!a[i].infinity) { az[count++] = a[i].z; } } - azi = checked_malloc(sizeof(secp256k1_fe_t) * count); + azi = (secp256k1_fe_t *)checked_malloc(sizeof(secp256k1_fe_t) * count); secp256k1_fe_inv_all_var(count, azi, az); free(az); @@ -138,11 +138,13 @@ static int secp256k1_ge_set_xo_var(secp256k1_ge_t *r, const secp256k1_fe_t *x, i r->infinity = 0; secp256k1_fe_set_int(&c, 7); secp256k1_fe_add(&c, &x3); - if (!secp256k1_fe_sqrt_var(&r->y, &c)) + if (!secp256k1_fe_sqrt_var(&r->y, &c)) { return 0; + } secp256k1_fe_normalize_var(&r->y); - if (secp256k1_fe_is_odd(&r->y) != odd) + if (secp256k1_fe_is_odd(&r->y) != odd) { secp256k1_fe_negate(&r->y, &r->y, 1); + } return 1; } @@ -176,8 +178,9 @@ static int secp256k1_gej_is_infinity(const secp256k1_gej_t *a) { static int secp256k1_gej_is_valid_var(const secp256k1_gej_t *a) { secp256k1_fe_t y2, x3, z2, z6; - if (a->infinity) + if (a->infinity) { return 0; + } /** y^2 = x^3 + 7 * (Y/Z^3)^2 = (X/Z^2)^3 + 7 * Y^2 / Z^6 = X^3 / Z^6 + 7 @@ -195,8 +198,9 @@ static int secp256k1_gej_is_valid_var(const secp256k1_gej_t *a) { static int secp256k1_ge_is_valid_var(const secp256k1_ge_t *a) { secp256k1_fe_t y2, x3, c; - if (a->infinity) + if (a->infinity) { return 0; + } /* y^2 = x^3 + 7 */ secp256k1_fe_sqr(&y2, &a->y); secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x); @@ -321,7 +325,8 @@ static void secp256k1_gej_add_ge_var(secp256k1_gej_t *r, const secp256k1_gej_t * } static void secp256k1_gej_add_ge(secp256k1_gej_t *r, const secp256k1_gej_t *a, const secp256k1_ge_t *b) { - /* Operations: 7 mul, 5 sqr, 5 normalize, 19 mul_int/add/negate */ + /* Operations: 7 mul, 5 sqr, 5 normalize, 17 mul_int/add/negate/cmov */ + static const secp256k1_fe_t fe_1 = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 1); secp256k1_fe_t zz, u1, u2, s1, s2, z, t, m, n, q, rr; int infinity; VERIFY_CHECK(!b->infinity); @@ -383,17 +388,25 @@ static void secp256k1_gej_add_ge(secp256k1_gej_t *r, const secp256k1_gej_t *a, c secp256k1_fe_mul_int(&r->y, 4 * (1 - a->infinity)); /* r->y = Y3 = 4*R*(3*Q-2*R^2)-4*M^4 (4) */ /** In case a->infinity == 1, the above code results in r->x, r->y, and r->z all equal to 0. - * Add b->x to x, b->y to y, and 1 to z in that case. + * Replace r with b->x, b->y, 1 in that case. */ - t = b->x; secp256k1_fe_mul_int(&t, a->infinity); - secp256k1_fe_add(&r->x, &t); - t = b->y; secp256k1_fe_mul_int(&t, a->infinity); - secp256k1_fe_add(&r->y, &t); - secp256k1_fe_set_int(&t, a->infinity); - secp256k1_fe_add(&r->z, &t); + secp256k1_fe_cmov(&r->x, &b->x, a->infinity); + secp256k1_fe_cmov(&r->y, &b->y, a->infinity); + secp256k1_fe_cmov(&r->z, &fe_1, a->infinity); r->infinity = infinity; } +static void secp256k1_gej_rescale(secp256k1_gej_t *r, const secp256k1_fe_t *s) { + /* Operations: 4 mul, 1 sqr */ + secp256k1_fe_t zz; + VERIFY_CHECK(!secp256k1_fe_is_zero(s)); + secp256k1_fe_sqr(&zz, s); + secp256k1_fe_mul(&r->x, &r->x, &zz); /* r->x *= s^2 */ + secp256k1_fe_mul(&r->y, &r->y, &zz); + secp256k1_fe_mul(&r->y, &r->y, s); /* r->y *= s^3 */ + secp256k1_fe_mul(&r->z, &r->z, s); /* r->z *= s */ +} + static void secp256k1_ge_to_storage(secp256k1_ge_storage_t *r, const secp256k1_ge_t *a) { secp256k1_fe_t x, y; VERIFY_CHECK(!a->infinity); diff --git a/src/secp256k1/src/hash_impl.h b/src/secp256k1/src/hash_impl.h index 60fdbf7718..9828827bcd 100644 --- a/src/secp256k1/src/hash_impl.h +++ b/src/secp256k1/src/hash_impl.h @@ -176,13 +176,15 @@ static void secp256k1_hmac_sha256_initialize(secp256k1_hmac_sha256_t *hash, cons } secp256k1_sha256_initialize(&hash->outer); - for (n = 0; n < 64; n++) + for (n = 0; n < 64; n++) { rkey[n] ^= 0x5c; + } secp256k1_sha256_write(&hash->outer, rkey, 64); secp256k1_sha256_initialize(&hash->inner); - for (n = 0; n < 64; n++) + for (n = 0; n < 64; n++) { rkey[n] ^= 0x5c ^ 0x36; + } secp256k1_sha256_write(&hash->inner, rkey, 64); memset(rkey, 0, 64); } @@ -205,15 +207,17 @@ static void secp256k1_rfc6979_hmac_sha256_initialize(secp256k1_rfc6979_hmac_sha2 static const unsigned char zero[1] = {0x00}; static const unsigned char one[1] = {0x01}; - memset(rng->v, 0x01, 32); - memset(rng->k, 0x00, 32); + memset(rng->v, 0x01, 32); /* RFC6979 3.2.b. */ + memset(rng->k, 0x00, 32); /* RFC6979 3.2.c. */ + /* RFC6979 3.2.d. */ secp256k1_hmac_sha256_initialize(&hmac, rng->k, 32); secp256k1_hmac_sha256_write(&hmac, rng->v, 32); secp256k1_hmac_sha256_write(&hmac, zero, 1); secp256k1_hmac_sha256_write(&hmac, key, keylen); secp256k1_hmac_sha256_write(&hmac, msg, msglen); if (rnd && rndlen) { + /* RFC6979 3.6 "Additional data". */ secp256k1_hmac_sha256_write(&hmac, rnd, rndlen); } secp256k1_hmac_sha256_finalize(&hmac, rng->k); @@ -221,12 +225,14 @@ static void secp256k1_rfc6979_hmac_sha256_initialize(secp256k1_rfc6979_hmac_sha2 secp256k1_hmac_sha256_write(&hmac, rng->v, 32); secp256k1_hmac_sha256_finalize(&hmac, rng->v); + /* RFC6979 3.2.f. */ secp256k1_hmac_sha256_initialize(&hmac, rng->k, 32); secp256k1_hmac_sha256_write(&hmac, rng->v, 32); secp256k1_hmac_sha256_write(&hmac, one, 1); secp256k1_hmac_sha256_write(&hmac, key, keylen); secp256k1_hmac_sha256_write(&hmac, msg, msglen); if (rnd && rndlen) { + /* RFC6979 3.6 "Additional data". */ secp256k1_hmac_sha256_write(&hmac, rnd, rndlen); } secp256k1_hmac_sha256_finalize(&hmac, rng->k); @@ -237,6 +243,7 @@ static void secp256k1_rfc6979_hmac_sha256_initialize(secp256k1_rfc6979_hmac_sha2 } static void secp256k1_rfc6979_hmac_sha256_generate(secp256k1_rfc6979_hmac_sha256_t *rng, unsigned char *out, size_t outlen) { + /* RFC6979 3.2.h. */ static const unsigned char zero[1] = {0x00}; if (rng->retry) { secp256k1_hmac_sha256_t hmac; diff --git a/src/secp256k1/src/num_gmp_impl.h b/src/secp256k1/src/num_gmp_impl.h index 3e4b92d329..dbbc458d5d 100644 --- a/src/secp256k1/src/num_gmp_impl.h +++ b/src/secp256k1/src/num_gmp_impl.h @@ -54,7 +54,9 @@ static void secp256k1_num_set_bin(secp256k1_num_t *r, const unsigned char *a, un VERIFY_CHECK(len <= NUM_LIMBS*2); r->limbs = len; r->neg = 0; - while (r->limbs > 1 && r->data[r->limbs-1]==0) r->limbs--; + while (r->limbs > 1 && r->data[r->limbs-1]==0) { + r->limbs--; + } } static void secp256k1_num_add_abs(secp256k1_num_t *r, const secp256k1_num_t *a, const secp256k1_num_t *b) { @@ -70,7 +72,9 @@ static void secp256k1_num_sub_abs(secp256k1_num_t *r, const secp256k1_num_t *a, mp_limb_t c = mpn_sub(r->data, a->data, a->limbs, b->data, b->limbs); VERIFY_CHECK(c == 0); r->limbs = a->limbs; - while (r->limbs > 1 && r->data[r->limbs-1]==0) r->limbs--; + while (r->limbs > 1 && r->data[r->limbs-1]==0) { + r->limbs--; + } } static void secp256k1_num_mod(secp256k1_num_t *r, const secp256k1_num_t *m) { @@ -82,7 +86,9 @@ static void secp256k1_num_mod(secp256k1_num_t *r, const secp256k1_num_t *m) { mpn_tdiv_qr(t, r->data, 0, r->data, r->limbs, m->data, m->limbs); memset(t, 0, sizeof(t)); r->limbs = m->limbs; - while (r->limbs > 1 && r->data[r->limbs-1]==0) r->limbs--; + while (r->limbs > 1 && r->data[r->limbs-1]==0) { + r->limbs--; + } } if (r->neg && (r->limbs > 1 || r->data[0] != 0)) { @@ -125,7 +131,9 @@ static void secp256k1_num_mod_inverse(secp256k1_num_t *r, const secp256k1_num_t if (sn < 0) { mpn_sub(r->data, m->data, m->limbs, r->data, -sn); r->limbs = m->limbs; - while (r->limbs > 1 && r->data[r->limbs-1]==0) r->limbs--; + while (r->limbs > 1 && r->data[r->limbs-1]==0) { + r->limbs--; + } } else { r->limbs = sn; } @@ -143,15 +151,25 @@ static int secp256k1_num_is_neg(const secp256k1_num_t *a) { } static int secp256k1_num_cmp(const secp256k1_num_t *a, const secp256k1_num_t *b) { - if (a->limbs > b->limbs) return 1; - if (a->limbs < b->limbs) return -1; + if (a->limbs > b->limbs) { + return 1; + } + if (a->limbs < b->limbs) { + return -1; + } return mpn_cmp(a->data, b->data, a->limbs); } static int secp256k1_num_eq(const secp256k1_num_t *a, const secp256k1_num_t *b) { - if (a->limbs > b->limbs) return 0; - if (a->limbs < b->limbs) return 0; - if ((a->neg && !secp256k1_num_is_zero(a)) != (b->neg && !secp256k1_num_is_zero(b))) return 0; + if (a->limbs > b->limbs) { + return 0; + } + if (a->limbs < b->limbs) { + return 0; + } + if ((a->neg && !secp256k1_num_is_zero(a)) != (b->neg && !secp256k1_num_is_zero(b))) { + return 0; + } return mpn_cmp(a->data, b->data, a->limbs) == 0; } @@ -198,12 +216,15 @@ static void secp256k1_num_mul(secp256k1_num_t *r, const secp256k1_num_t *a, cons r->data[0] = 0; return; } - if (a->limbs >= b->limbs) + if (a->limbs >= b->limbs) { mpn_mul(tmp, a->data, a->limbs, b->data, b->limbs); - else + } else { mpn_mul(tmp, b->data, b->limbs, a->data, a->limbs); + } r->limbs = a->limbs + b->limbs; - if (r->limbs > 1 && tmp[r->limbs - 1]==0) r->limbs--; + if (r->limbs > 1 && tmp[r->limbs - 1]==0) { + r->limbs--; + } VERIFY_CHECK(r->limbs <= 2*NUM_LIMBS); mpn_copyi(r->data, tmp, r->limbs); r->neg = a->neg ^ b->neg; @@ -227,7 +248,9 @@ static void secp256k1_num_shift(secp256k1_num_t *r, int bits) { } } } - while (r->limbs>1 && r->data[r->limbs-1]==0) r->limbs--; + while (r->limbs>1 && r->data[r->limbs-1]==0) { + r->limbs--; + } } static void secp256k1_num_negate(secp256k1_num_t *r) { diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index 3acbe264ae..33824983e4 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -69,130 +69,168 @@ static void secp256k1_scalar_inverse(secp256k1_scalar_t *r, const secp256k1_scal secp256k1_scalar_mul(&x8, &x8, x); secp256k1_scalar_sqr(&x15, &x8); - for (i = 0; i < 6; i++) + for (i = 0; i < 6; i++) { secp256k1_scalar_sqr(&x15, &x15); + } secp256k1_scalar_mul(&x15, &x15, &x7); secp256k1_scalar_sqr(&x30, &x15); - for (i = 0; i < 14; i++) + for (i = 0; i < 14; i++) { secp256k1_scalar_sqr(&x30, &x30); + } secp256k1_scalar_mul(&x30, &x30, &x15); secp256k1_scalar_sqr(&x60, &x30); - for (i = 0; i < 29; i++) + for (i = 0; i < 29; i++) { secp256k1_scalar_sqr(&x60, &x60); + } secp256k1_scalar_mul(&x60, &x60, &x30); secp256k1_scalar_sqr(&x120, &x60); - for (i = 0; i < 59; i++) + for (i = 0; i < 59; i++) { secp256k1_scalar_sqr(&x120, &x120); + } secp256k1_scalar_mul(&x120, &x120, &x60); secp256k1_scalar_sqr(&x127, &x120); - for (i = 0; i < 6; i++) + for (i = 0; i < 6; i++) { secp256k1_scalar_sqr(&x127, &x127); + } secp256k1_scalar_mul(&x127, &x127, &x7); /* Then accumulate the final result (t starts at x127). */ t = &x127; - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) /* 0 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) /* 0 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 3; i++) /* 0 */ + for (i = 0; i < 3; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 4; i++) /* 0 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 5; i++) /* 00 */ + for (i = 0; i < 5; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 4; i++) /* 00 */ + for (i = 0; i < 4; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 5; i++) /* 0 */ + for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x4); /* 1111 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) /* 00 */ + for (i = 0; i < 3; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) /* 000 */ + for (i = 0; i < 4; i++) { /* 000 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 10; i++) /* 0000000 */ + for (i = 0; i < 10; i++) { /* 0000000 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 4; i++) /* 0 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 9; i++) /* 0 */ + for (i = 0; i < 9; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x8); /* 11111111 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) /* 00 */ + for (i = 0; i < 3; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) /* 00 */ + for (i = 0; i < 3; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 5; i++) /* 0 */ + for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x4); /* 1111 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 5; i++) /* 000 */ + for (i = 0; i < 5; i++) { /* 000 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 4; i++) /* 00 */ + for (i = 0; i < 4; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 2; i++) /* 0 */ + for (i = 0; i < 2; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 8; i++) /* 000000 */ + for (i = 0; i < 8; i++) { /* 000000 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 3; i++) /* 0 */ + for (i = 0; i < 3; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 3; i++) /* 00 */ + for (i = 0; i < 3; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 6; i++) /* 00000 */ + for (i = 0; i < 6; i++) { /* 00000 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 8; i++) /* 00 */ + for (i = 0; i < 8; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); + } secp256k1_scalar_mul(r, t, &x6); /* 111111 */ } diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 8c4eca4b62..d6192dc4ed 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -1,5 +1,5 @@ /********************************************************************** - * Copyright (c) 2013, 2014 Pieter Wuille * + * Copyright (c) 2013-2015 Pieter Wuille * * Distributed under the MIT software license, see the accompanying * * file COPYING or http://www.opensource.org/licenses/mit-license.php.* **********************************************************************/ @@ -19,26 +19,48 @@ #include "eckey_impl.h" #include "hash_impl.h" -void secp256k1_start(unsigned int flags) { - if (flags & SECP256K1_START_SIGN) { - secp256k1_ecmult_gen_start(); +struct secp256k1_context_struct { + secp256k1_ecmult_context_t ecmult_ctx; + secp256k1_ecmult_gen_context_t ecmult_gen_ctx; +}; + +secp256k1_context_t* secp256k1_context_create(int flags) { + secp256k1_context_t* ret = (secp256k1_context_t*)checked_malloc(sizeof(secp256k1_context_t)); + + secp256k1_ecmult_context_init(&ret->ecmult_ctx); + secp256k1_ecmult_gen_context_init(&ret->ecmult_gen_ctx); + + if (flags & SECP256K1_CONTEXT_SIGN) { + secp256k1_ecmult_gen_context_build(&ret->ecmult_gen_ctx); } - if (flags & SECP256K1_START_VERIFY) { - secp256k1_ecmult_start(); + if (flags & SECP256K1_CONTEXT_VERIFY) { + secp256k1_ecmult_context_build(&ret->ecmult_ctx); } + + return ret; } -void secp256k1_stop(void) { - secp256k1_ecmult_stop(); - secp256k1_ecmult_gen_stop(); +secp256k1_context_t* secp256k1_context_clone(const secp256k1_context_t* ctx) { + secp256k1_context_t* ret = (secp256k1_context_t*)checked_malloc(sizeof(secp256k1_context_t)); + secp256k1_ecmult_context_clone(&ret->ecmult_ctx, &ctx->ecmult_ctx); + secp256k1_ecmult_gen_context_clone(&ret->ecmult_gen_ctx, &ctx->ecmult_gen_ctx); + return ret; } -int secp256k1_ecdsa_verify(const unsigned char *msg32, const unsigned char *sig, int siglen, const unsigned char *pubkey, int pubkeylen) { +void secp256k1_context_destroy(secp256k1_context_t* ctx) { + secp256k1_ecmult_context_clear(&ctx->ecmult_ctx); + secp256k1_ecmult_gen_context_clear(&ctx->ecmult_gen_ctx); + + free(ctx); +} + +int secp256k1_ecdsa_verify(const secp256k1_context_t* ctx, const unsigned char *msg32, const unsigned char *sig, int siglen, const unsigned char *pubkey, int pubkeylen) { secp256k1_ge_t q; secp256k1_ecdsa_sig_t s; secp256k1_scalar_t m; int ret = -3; - DEBUG_CHECK(secp256k1_ecmult_consts != NULL); + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); DEBUG_CHECK(msg32 != NULL); DEBUG_CHECK(sig != NULL); DEBUG_CHECK(pubkey != NULL); @@ -47,7 +69,7 @@ int secp256k1_ecdsa_verify(const unsigned char *msg32, const unsigned char *sig, if (secp256k1_eckey_pubkey_parse(&q, pubkey, pubkeylen)) { if (secp256k1_ecdsa_sig_parse(&s, sig, siglen)) { - if (secp256k1_ecdsa_sig_verify(&s, &q, &m)) { + if (secp256k1_ecdsa_sig_verify(&ctx->ecmult_ctx, &s, &q, &m)) { /* success is 1, all other values are fail */ ret = 1; } else { @@ -66,7 +88,7 @@ int secp256k1_ecdsa_verify(const unsigned char *msg32, const unsigned char *sig, static int nonce_function_rfc6979(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, unsigned int counter, const void *data) { secp256k1_rfc6979_hmac_sha256_t rng; unsigned int i; - secp256k1_rfc6979_hmac_sha256_initialize(&rng, key32, 32, msg32, 32, data, data != NULL ? 32 : 0); + secp256k1_rfc6979_hmac_sha256_initialize(&rng, key32, 32, msg32, 32, (const unsigned char*)data, data != NULL ? 32 : 0); for (i = 0; i <= counter; i++) { secp256k1_rfc6979_hmac_sha256_generate(&rng, nonce32, 32); } @@ -77,13 +99,14 @@ static int nonce_function_rfc6979(unsigned char *nonce32, const unsigned char *m const secp256k1_nonce_function_t secp256k1_nonce_function_rfc6979 = nonce_function_rfc6979; const secp256k1_nonce_function_t secp256k1_nonce_function_default = nonce_function_rfc6979; -int secp256k1_ecdsa_sign(const unsigned char *msg32, unsigned char *signature, int *signaturelen, const unsigned char *seckey, secp256k1_nonce_function_t noncefp, const void* noncedata) { +int secp256k1_ecdsa_sign(const secp256k1_context_t* ctx, const unsigned char *msg32, unsigned char *signature, int *signaturelen, const unsigned char *seckey, secp256k1_nonce_function_t noncefp, const void* noncedata) { secp256k1_ecdsa_sig_t sig; secp256k1_scalar_t sec, non, msg; int ret = 0; int overflow = 0; unsigned int count = 0; - DEBUG_CHECK(secp256k1_ecmult_gen_consts != NULL); + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); DEBUG_CHECK(msg32 != NULL); DEBUG_CHECK(signature != NULL); DEBUG_CHECK(signaturelen != NULL); @@ -105,7 +128,7 @@ int secp256k1_ecdsa_sign(const unsigned char *msg32, unsigned char *signature, i secp256k1_scalar_set_b32(&non, nonce32, &overflow); memset(nonce32, 0, 32); if (!secp256k1_scalar_is_zero(&non) && !overflow) { - if (secp256k1_ecdsa_sig_sign(&sig, &sec, &msg, &non, NULL)) { + if (secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &sig, &sec, &msg, &non, NULL)) { break; } } @@ -124,13 +147,14 @@ int secp256k1_ecdsa_sign(const unsigned char *msg32, unsigned char *signature, i return ret; } -int secp256k1_ecdsa_sign_compact(const unsigned char *msg32, unsigned char *sig64, const unsigned char *seckey, secp256k1_nonce_function_t noncefp, const void* noncedata, int *recid) { +int secp256k1_ecdsa_sign_compact(const secp256k1_context_t* ctx, const unsigned char *msg32, unsigned char *sig64, const unsigned char *seckey, secp256k1_nonce_function_t noncefp, const void* noncedata, int *recid) { secp256k1_ecdsa_sig_t sig; secp256k1_scalar_t sec, non, msg; int ret = 0; int overflow = 0; unsigned int count = 0; - DEBUG_CHECK(secp256k1_ecmult_gen_consts != NULL); + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); DEBUG_CHECK(msg32 != NULL); DEBUG_CHECK(sig64 != NULL); DEBUG_CHECK(seckey != NULL); @@ -151,7 +175,7 @@ int secp256k1_ecdsa_sign_compact(const unsigned char *msg32, unsigned char *sig6 secp256k1_scalar_set_b32(&non, nonce32, &overflow); memset(nonce32, 0, 32); if (!secp256k1_scalar_is_zero(&non) && !overflow) { - if (secp256k1_ecdsa_sig_sign(&sig, &sec, &msg, &non, recid)) { + if (secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &sig, &sec, &msg, &non, recid)) { break; } } @@ -171,13 +195,14 @@ int secp256k1_ecdsa_sign_compact(const unsigned char *msg32, unsigned char *sig6 return ret; } -int secp256k1_ecdsa_recover_compact(const unsigned char *msg32, const unsigned char *sig64, unsigned char *pubkey, int *pubkeylen, int compressed, int recid) { +int secp256k1_ecdsa_recover_compact(const secp256k1_context_t* ctx, const unsigned char *msg32, const unsigned char *sig64, unsigned char *pubkey, int *pubkeylen, int compressed, int recid) { secp256k1_ge_t q; secp256k1_ecdsa_sig_t sig; secp256k1_scalar_t m; int ret = 0; int overflow = 0; - DEBUG_CHECK(secp256k1_ecmult_consts != NULL); + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); DEBUG_CHECK(msg32 != NULL); DEBUG_CHECK(sig64 != NULL); DEBUG_CHECK(pubkey != NULL); @@ -190,7 +215,7 @@ int secp256k1_ecdsa_recover_compact(const unsigned char *msg32, const unsigned c if (!overflow) { secp256k1_scalar_set_b32(&m, msg32, NULL); - if (secp256k1_ecdsa_sig_recover(&sig, &q, &m, recid)) { + if (secp256k1_ecdsa_sig_recover(&ctx->ecmult_ctx, &sig, &q, &m, recid)) { ret = secp256k1_eckey_pubkey_serialize(&q, pubkey, pubkeylen, compressed); } } @@ -198,11 +223,13 @@ int secp256k1_ecdsa_recover_compact(const unsigned char *msg32, const unsigned c return ret; } -int secp256k1_ec_seckey_verify(const unsigned char *seckey) { +int secp256k1_ec_seckey_verify(const secp256k1_context_t* ctx, const unsigned char *seckey) { secp256k1_scalar_t sec; int ret; int overflow; + DEBUG_CHECK(ctx != NULL); DEBUG_CHECK(seckey != NULL); + (void)ctx; secp256k1_scalar_set_b32(&sec, seckey, &overflow); ret = !secp256k1_scalar_is_zero(&sec) && !overflow; @@ -210,27 +237,30 @@ int secp256k1_ec_seckey_verify(const unsigned char *seckey) { return ret; } -int secp256k1_ec_pubkey_verify(const unsigned char *pubkey, int pubkeylen) { +int secp256k1_ec_pubkey_verify(const secp256k1_context_t* ctx, const unsigned char *pubkey, int pubkeylen) { secp256k1_ge_t q; + DEBUG_CHECK(ctx != NULL); DEBUG_CHECK(pubkey != NULL); + (void)ctx; return secp256k1_eckey_pubkey_parse(&q, pubkey, pubkeylen); } -int secp256k1_ec_pubkey_create(unsigned char *pubkey, int *pubkeylen, const unsigned char *seckey, int compressed) { +int secp256k1_ec_pubkey_create(const secp256k1_context_t* ctx, unsigned char *pubkey, int *pubkeylen, const unsigned char *seckey, int compressed) { secp256k1_gej_t pj; secp256k1_ge_t p; secp256k1_scalar_t sec; int overflow; int ret = 0; - DEBUG_CHECK(secp256k1_ecmult_gen_consts != NULL); + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); DEBUG_CHECK(pubkey != NULL); DEBUG_CHECK(pubkeylen != NULL); DEBUG_CHECK(seckey != NULL); secp256k1_scalar_set_b32(&sec, seckey, &overflow); if (!overflow) { - secp256k1_ecmult_gen(&pj, &sec); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pj, &sec); secp256k1_scalar_clear(&sec); secp256k1_ge_set_gej(&p, &pj); ret = secp256k1_eckey_pubkey_serialize(&p, pubkey, pubkeylen, compressed); @@ -241,11 +271,12 @@ int secp256k1_ec_pubkey_create(unsigned char *pubkey, int *pubkeylen, const unsi return ret; } -int secp256k1_ec_pubkey_decompress(unsigned char *pubkey, int *pubkeylen) { +int secp256k1_ec_pubkey_decompress(const secp256k1_context_t* ctx, unsigned char *pubkey, int *pubkeylen) { secp256k1_ge_t p; int ret = 0; DEBUG_CHECK(pubkey != NULL); DEBUG_CHECK(pubkeylen != NULL); + (void)ctx; if (secp256k1_eckey_pubkey_parse(&p, pubkey, *pubkeylen)) { ret = secp256k1_eckey_pubkey_serialize(&p, pubkey, pubkeylen, 0); @@ -253,13 +284,15 @@ int secp256k1_ec_pubkey_decompress(unsigned char *pubkey, int *pubkeylen) { return ret; } -int secp256k1_ec_privkey_tweak_add(unsigned char *seckey, const unsigned char *tweak) { +int secp256k1_ec_privkey_tweak_add(const secp256k1_context_t* ctx, unsigned char *seckey, const unsigned char *tweak) { secp256k1_scalar_t term; secp256k1_scalar_t sec; int ret = 0; int overflow = 0; + DEBUG_CHECK(ctx != NULL); DEBUG_CHECK(seckey != NULL); DEBUG_CHECK(tweak != NULL); + (void)ctx; secp256k1_scalar_set_b32(&term, tweak, &overflow); secp256k1_scalar_set_b32(&sec, seckey, NULL); @@ -274,12 +307,13 @@ int secp256k1_ec_privkey_tweak_add(unsigned char *seckey, const unsigned char *t return ret; } -int secp256k1_ec_pubkey_tweak_add(unsigned char *pubkey, int pubkeylen, const unsigned char *tweak) { +int secp256k1_ec_pubkey_tweak_add(const secp256k1_context_t* ctx, unsigned char *pubkey, int pubkeylen, const unsigned char *tweak) { secp256k1_ge_t p; secp256k1_scalar_t term; int ret = 0; int overflow = 0; - DEBUG_CHECK(secp256k1_ecmult_consts != NULL); + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); DEBUG_CHECK(pubkey != NULL); DEBUG_CHECK(tweak != NULL); @@ -287,7 +321,7 @@ int secp256k1_ec_pubkey_tweak_add(unsigned char *pubkey, int pubkeylen, const un if (!overflow) { ret = secp256k1_eckey_pubkey_parse(&p, pubkey, pubkeylen); if (ret) { - ret = secp256k1_eckey_pubkey_tweak_add(&p, &term); + ret = secp256k1_eckey_pubkey_tweak_add(&ctx->ecmult_ctx, &p, &term); } if (ret) { int oldlen = pubkeylen; @@ -299,13 +333,15 @@ int secp256k1_ec_pubkey_tweak_add(unsigned char *pubkey, int pubkeylen, const un return ret; } -int secp256k1_ec_privkey_tweak_mul(unsigned char *seckey, const unsigned char *tweak) { +int secp256k1_ec_privkey_tweak_mul(const secp256k1_context_t* ctx, unsigned char *seckey, const unsigned char *tweak) { secp256k1_scalar_t factor; secp256k1_scalar_t sec; int ret = 0; int overflow = 0; + DEBUG_CHECK(ctx != NULL); DEBUG_CHECK(seckey != NULL); DEBUG_CHECK(tweak != NULL); + (void)ctx; secp256k1_scalar_set_b32(&factor, tweak, &overflow); secp256k1_scalar_set_b32(&sec, seckey, NULL); @@ -319,12 +355,13 @@ int secp256k1_ec_privkey_tweak_mul(unsigned char *seckey, const unsigned char *t return ret; } -int secp256k1_ec_pubkey_tweak_mul(unsigned char *pubkey, int pubkeylen, const unsigned char *tweak) { +int secp256k1_ec_pubkey_tweak_mul(const secp256k1_context_t* ctx, unsigned char *pubkey, int pubkeylen, const unsigned char *tweak) { secp256k1_ge_t p; secp256k1_scalar_t factor; int ret = 0; int overflow = 0; - DEBUG_CHECK(secp256k1_ecmult_consts != NULL); + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); DEBUG_CHECK(pubkey != NULL); DEBUG_CHECK(tweak != NULL); @@ -332,7 +369,7 @@ int secp256k1_ec_pubkey_tweak_mul(unsigned char *pubkey, int pubkeylen, const un if (!overflow) { ret = secp256k1_eckey_pubkey_parse(&p, pubkey, pubkeylen); if (ret) { - ret = secp256k1_eckey_pubkey_tweak_mul(&p, &factor); + ret = secp256k1_eckey_pubkey_tweak_mul(&ctx->ecmult_ctx, &p, &factor); } if (ret) { int oldlen = pubkeylen; @@ -344,24 +381,27 @@ int secp256k1_ec_pubkey_tweak_mul(unsigned char *pubkey, int pubkeylen, const un return ret; } -int secp256k1_ec_privkey_export(const unsigned char *seckey, unsigned char *privkey, int *privkeylen, int compressed) { +int secp256k1_ec_privkey_export(const secp256k1_context_t* ctx, const unsigned char *seckey, unsigned char *privkey, int *privkeylen, int compressed) { secp256k1_scalar_t key; int ret = 0; DEBUG_CHECK(seckey != NULL); DEBUG_CHECK(privkey != NULL); DEBUG_CHECK(privkeylen != NULL); + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); secp256k1_scalar_set_b32(&key, seckey, NULL); - ret = secp256k1_eckey_privkey_serialize(privkey, privkeylen, &key, compressed); + ret = secp256k1_eckey_privkey_serialize(&ctx->ecmult_gen_ctx, privkey, privkeylen, &key, compressed); secp256k1_scalar_clear(&key); return ret; } -int secp256k1_ec_privkey_import(unsigned char *seckey, const unsigned char *privkey, int privkeylen) { +int secp256k1_ec_privkey_import(const secp256k1_context_t* ctx, unsigned char *seckey, const unsigned char *privkey, int privkeylen) { secp256k1_scalar_t key; int ret = 0; DEBUG_CHECK(seckey != NULL); DEBUG_CHECK(privkey != NULL); + (void)ctx; ret = secp256k1_eckey_privkey_parse(&key, privkey, privkeylen); if (ret) { @@ -370,3 +410,10 @@ int secp256k1_ec_privkey_import(unsigned char *seckey, const unsigned char *priv secp256k1_scalar_clear(&key); return ret; } + +int secp256k1_context_randomize(secp256k1_context_t* ctx, const unsigned char *seed32) { + DEBUG_CHECK(ctx != NULL); + DEBUG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); + secp256k1_ecmult_gen_blind(&ctx->ecmult_gen_ctx, seed32); + return 1; +} diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index f7f1acac64..d0e05057f2 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -1,5 +1,5 @@ /********************************************************************** - * Copyright (c) 2013, 2014 Pieter Wuille * + * Copyright (c) 2013, 2014, 2015 Pieter Wuille, Gregory Maxwell * * Distributed under the MIT software license, see the accompanying * * file COPYING or http://www.opensource.org/licenses/mit-license.php.* **********************************************************************/ @@ -24,6 +24,7 @@ #endif static int count = 64; +static secp256k1_context_t *ctx = NULL; void random_field_element_test(secp256k1_fe_t *fe) { do { @@ -55,8 +56,9 @@ void random_group_element_test(secp256k1_ge_t *ge) { secp256k1_fe_t fe; do { random_field_element_test(&fe); - if (secp256k1_ge_set_xo_var(ge, &fe, secp256k1_rand32() & 1)) + if (secp256k1_ge_set_xo_var(ge, &fe, secp256k1_rand32() & 1)) { break; + } } while(1); } @@ -81,8 +83,9 @@ void random_scalar_order_test(secp256k1_scalar_t *num) { int overflow = 0; secp256k1_rand256_test(b32); secp256k1_scalar_set_b32(num, b32, &overflow); - if (overflow || secp256k1_scalar_is_zero(num)) + if (overflow || secp256k1_scalar_is_zero(num)) { continue; + } break; } while(1); } @@ -93,12 +96,60 @@ void random_scalar_order(secp256k1_scalar_t *num) { int overflow = 0; secp256k1_rand256(b32); secp256k1_scalar_set_b32(num, b32, &overflow); - if (overflow || secp256k1_scalar_is_zero(num)) + if (overflow || secp256k1_scalar_is_zero(num)) { continue; + } break; } while(1); } +void run_context_tests(void) { + secp256k1_context_t *none = secp256k1_context_create(0); + secp256k1_context_t *sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_context_t *vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + secp256k1_context_t *both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + + secp256k1_gej_t pubj; + secp256k1_ge_t pub; + secp256k1_scalar_t msg, key, nonce; + secp256k1_ecdsa_sig_t sig; + + /*** clone and destroy all of them to make sure cloning was complete ***/ + { + secp256k1_context_t *ctx_tmp; + + ctx_tmp = none; none = secp256k1_context_clone(none); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = sign; sign = secp256k1_context_clone(sign); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = vrfy; vrfy = secp256k1_context_clone(vrfy); secp256k1_context_destroy(ctx_tmp); + ctx_tmp = both; both = secp256k1_context_clone(both); secp256k1_context_destroy(ctx_tmp); + } + + /*** attempt to use them ***/ + random_scalar_order_test(&msg); + random_scalar_order_test(&key); + secp256k1_ecmult_gen(&both->ecmult_gen_ctx, &pubj, &key); + secp256k1_ge_set_gej(&pub, &pubj); + + /* obtain a working nonce */ + do { + random_scalar_order_test(&nonce); + } while(!secp256k1_ecdsa_sig_sign(&both->ecmult_gen_ctx, &sig, &key, &msg, &nonce, NULL)); + + /* try signing */ + CHECK(secp256k1_ecdsa_sig_sign(&sign->ecmult_gen_ctx, &sig, &key, &msg, &nonce, NULL)); + CHECK(secp256k1_ecdsa_sig_sign(&both->ecmult_gen_ctx, &sig, &key, &msg, &nonce, NULL)); + + /* try verifying */ + CHECK(secp256k1_ecdsa_sig_verify(&vrfy->ecmult_ctx, &sig, &pub, &msg)); + CHECK(secp256k1_ecdsa_sig_verify(&both->ecmult_ctx, &sig, &pub, &msg)); + + /* cleanup */ + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(vrfy); + secp256k1_context_destroy(both); +} + /***** HASH TESTS *****/ void run_sha256_tests(void) { @@ -229,8 +280,9 @@ void run_rfc6979_hmac_sha256_tests(void) { #ifndef USE_NUM_NONE void random_num_negate(secp256k1_num_t *num) { - if (secp256k1_rand32() & 1) + if (secp256k1_rand32() & 1) { secp256k1_num_negate(num); + } } void random_num_order_test(secp256k1_num_t *num) { @@ -624,8 +676,9 @@ void random_fe_non_zero(secp256k1_fe_t *nz) { while (--tries >= 0) { random_fe(nz); secp256k1_fe_normalize(nz); - if (!secp256k1_fe_is_zero(nz)) + if (!secp256k1_fe_is_zero(nz)) { break; + } } /* Infinitesimal probability of spurious failure here */ CHECK(tries >= 0); @@ -700,12 +753,22 @@ void run_field_misc(void) { CHECK(secp256k1_fe_equal_var(&x, &x)); z = x; secp256k1_fe_add(&z,&y); - secp256k1_fe_normalize(&z); + /* Test fe conditional move; z is not normalized here. */ + q = x; + secp256k1_fe_cmov(&x, &z, 0); + secp256k1_fe_cmov(&x, &x, 1); + CHECK(memcmp(&x, &z, sizeof(x)) != 0); + CHECK(memcmp(&x, &q, sizeof(x)) == 0); + secp256k1_fe_cmov(&q, &z, 1); + CHECK(memcmp(&q, &z, sizeof(q)) == 0); /* Test storage conversion and conditional moves. */ + secp256k1_fe_normalize(&z); + CHECK(!secp256k1_fe_equal_var(&x, &z)); secp256k1_fe_to_storage(&xs, &x); secp256k1_fe_to_storage(&ys, &y); secp256k1_fe_to_storage(&zs, &z); secp256k1_fe_storage_cmov(&zs, &xs, 0); + secp256k1_fe_storage_cmov(&zs, &zs, 1); CHECK(memcmp(&xs, &zs, sizeof(xs)) != 0); secp256k1_fe_storage_cmov(&ys, &xs, 1); CHECK(memcmp(&xs, &ys, sizeof(xs)) == 0); @@ -765,14 +828,17 @@ void run_field_inv_all_var(void) { for (i = 0; i < count; i++) { size_t j; size_t len = (secp256k1_rand32() & 15) + 1; - for (j = 0; j < len; j++) + for (j = 0; j < len; j++) { random_fe_non_zero(&x[j]); + } secp256k1_fe_inv_all_var(len, xi, x); - for (j = 0; j < len; j++) + for (j = 0; j < len; j++) { CHECK(check_fe_inverse(&x[j], &xi[j])); + } secp256k1_fe_inv_all_var(len, xii, xi); - for (j = 0; j < len; j++) + for (j = 0; j < len; j++) { CHECK(check_fe_equal(&x[j], &xii[j])); + } } } @@ -844,18 +910,42 @@ void run_sqrt(void) { void ge_equals_ge(const secp256k1_ge_t *a, const secp256k1_ge_t *b) { CHECK(a->infinity == b->infinity); - if (a->infinity) + if (a->infinity) { return; + } CHECK(secp256k1_fe_equal_var(&a->x, &b->x)); CHECK(secp256k1_fe_equal_var(&b->y, &b->y)); } +/* This compares jacobian points including their Z, not just their geometric meaning. */ +int gej_xyz_equals_gej(const secp256k1_gej_t *a, const secp256k1_gej_t *b) { + secp256k1_gej_t a2; + secp256k1_gej_t b2; + int ret = 1; + ret &= a->infinity == b->infinity; + if (ret && !a->infinity) { + a2 = *a; + b2 = *b; + secp256k1_fe_normalize(&a2.x); + secp256k1_fe_normalize(&a2.y); + secp256k1_fe_normalize(&a2.z); + secp256k1_fe_normalize(&b2.x); + secp256k1_fe_normalize(&b2.y); + secp256k1_fe_normalize(&b2.z); + ret &= secp256k1_fe_cmp_var(&a2.x, &b2.x) == 0; + ret &= secp256k1_fe_cmp_var(&a2.y, &b2.y) == 0; + ret &= secp256k1_fe_cmp_var(&a2.z, &b2.z) == 0; + } + return ret; +} + void ge_equals_gej(const secp256k1_ge_t *a, const secp256k1_gej_t *b) { secp256k1_fe_t z2s; secp256k1_fe_t u1, u2, s1, s2; CHECK(a->infinity == b->infinity); - if (a->infinity) + if (a->infinity) { return; + } /* Check a.x * b.z^2 == b.x && a.y * b.z^3 == b.y, to avoid inverses. */ secp256k1_fe_sqr(&z2s, &b->z); secp256k1_fe_mul(&u1, &a->x, &z2s); @@ -874,8 +964,8 @@ void test_ge(void) { * All magnitudes are randomized. * All 17*17 combinations of points are added to eachother, using all applicable methods. */ - secp256k1_ge_t *ge = malloc(sizeof(secp256k1_ge_t) * (1 + 4 * runs)); - secp256k1_gej_t *gej = malloc(sizeof(secp256k1_gej_t) * (1 + 4 * runs)); + secp256k1_ge_t *ge = (secp256k1_ge_t *)malloc(sizeof(secp256k1_ge_t) * (1 + 4 * runs)); + secp256k1_gej_t *gej = (secp256k1_gej_t *)malloc(sizeof(secp256k1_gej_t) * (1 + 4 * runs)); secp256k1_gej_set_infinity(&gej[0]); secp256k1_ge_clear(&ge[0]); secp256k1_ge_set_gej_var(&ge[0], &gej[0]); @@ -951,7 +1041,7 @@ void test_ge(void) { /* Test adding all points together in random order equals infinity. */ { secp256k1_gej_t sum = SECP256K1_GEJ_CONST_INFINITY; - secp256k1_gej_t *gej_shuffled = malloc((4 * runs + 1) * sizeof(secp256k1_gej_t)); + secp256k1_gej_t *gej_shuffled = (secp256k1_gej_t *)malloc((4 * runs + 1) * sizeof(secp256k1_gej_t)); for (i = 0; i < 4 * runs + 1; i++) { gej_shuffled[i] = gej[i]; } @@ -972,9 +1062,12 @@ void test_ge(void) { /* Test batch gej -> ge conversion. */ { - secp256k1_ge_t *ge_set_all = malloc((4 * runs + 1) * sizeof(secp256k1_ge_t)); + secp256k1_ge_t *ge_set_all = (secp256k1_ge_t *)malloc((4 * runs + 1) * sizeof(secp256k1_ge_t)); secp256k1_ge_set_all_gej_var(4 * runs + 1, ge_set_all, gej); for (i = 0; i < 4 * runs + 1; i++) { + secp256k1_fe_t s; + random_fe_non_zero(&s); + secp256k1_gej_rescale(&gej[i], &s); ge_equals_gej(&ge_set_all[i], &gej[i]); } free(ge_set_all); @@ -1025,7 +1118,7 @@ void run_ecmult_chain(void) { x = a; for (i = 0; i < 200*count; i++) { /* in each iteration, compute X = xn*X + gn*G; */ - secp256k1_ecmult(&x, &x, &xn, &gn); + secp256k1_ecmult(&ctx->ecmult_ctx, &x, &x, &xn, &gn); /* also compute ae and ge: the actual accumulated factors for A and G */ /* if X was (ae*A+ge*G), xn*X + gn*G results in (xn*ae*A + (xn*ge+gn)*G) */ secp256k1_scalar_mul(&ae, &ae, &xn); @@ -1051,7 +1144,7 @@ void run_ecmult_chain(void) { } } /* redo the computation, but directly with the resulting ae and ge coefficients: */ - secp256k1_ecmult(&x2, &a, &ae, &ge); + secp256k1_ecmult(&ctx->ecmult_ctx, &x2, &a, &ae, &ge); secp256k1_gej_neg(&x2, &x2); secp256k1_gej_add_var(&x2, &x2, &x); CHECK(secp256k1_gej_is_infinity(&x2)); @@ -1067,8 +1160,8 @@ void test_point_times_order(const secp256k1_gej_t *point) { int psize = 65; random_scalar_order_test(&x); secp256k1_scalar_negate(&nx, &x); - secp256k1_ecmult(&res1, point, &x, &x); /* calc res1 = x * point + x * G; */ - secp256k1_ecmult(&res2, point, &nx, &nx); /* calc res2 = (order - x) * point + (order - x) * G; */ + secp256k1_ecmult(&ctx->ecmult_ctx, &res1, point, &x, &x); /* calc res1 = x * point + x * G; */ + secp256k1_ecmult(&ctx->ecmult_ctx, &res2, point, &nx, &nx); /* calc res2 = (order - x) * point + (order - x) * G; */ secp256k1_gej_add_var(&res1, &res1, &res2); CHECK(secp256k1_gej_is_infinity(&res1)); CHECK(secp256k1_gej_is_valid_var(&res1) == 0); @@ -1141,17 +1234,96 @@ void run_wnaf(void) { secp256k1_scalar_t n; for (i = 0; i < count; i++) { random_scalar_order(&n); - if (i % 1) - secp256k1_scalar_negate(&n, &n); test_wnaf(&n, 4+(i%10)); } } +void test_ecmult_constants(void) { + /* Test ecmult_gen() for [0..36) and [order-36..0). */ + secp256k1_scalar_t x; + secp256k1_gej_t r; + secp256k1_ge_t ng; + int i; + int j; + secp256k1_ge_neg(&ng, &secp256k1_ge_const_g); + for (i = 0; i < 36; i++ ) { + secp256k1_scalar_set_int(&x, i); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &r, &x); + for (j = 0; j < i; j++) { + if (j == i - 1) { + ge_equals_gej(&secp256k1_ge_const_g, &r); + } + secp256k1_gej_add_ge(&r, &r, &ng); + } + CHECK(secp256k1_gej_is_infinity(&r)); + } + for (i = 1; i <= 36; i++ ) { + secp256k1_scalar_set_int(&x, i); + secp256k1_scalar_negate(&x, &x); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &r, &x); + for (j = 0; j < i; j++) { + if (j == i - 1) { + ge_equals_gej(&ng, &r); + } + secp256k1_gej_add_ge(&r, &r, &secp256k1_ge_const_g); + } + CHECK(secp256k1_gej_is_infinity(&r)); + } +} + +void run_ecmult_constants(void) { + test_ecmult_constants(); +} + +void test_ecmult_gen_blind(void) { + /* Test ecmult_gen() blinding and confirm that the blinding changes, the affline points match, and the z's don't match. */ + secp256k1_scalar_t key; + secp256k1_scalar_t b; + unsigned char seed32[32]; + secp256k1_gej_t pgej; + secp256k1_gej_t pgej2; + secp256k1_gej_t i; + secp256k1_ge_t pge; + random_scalar_order_test(&key); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pgej, &key); + secp256k1_rand256(seed32); + b = ctx->ecmult_gen_ctx.blind; + i = ctx->ecmult_gen_ctx.initial; + secp256k1_ecmult_gen_blind(&ctx->ecmult_gen_ctx, seed32); + CHECK(!secp256k1_scalar_eq(&b, &ctx->ecmult_gen_ctx.blind)); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pgej2, &key); + CHECK(!gej_xyz_equals_gej(&pgej, &pgej2)); + CHECK(!gej_xyz_equals_gej(&i, &ctx->ecmult_gen_ctx.initial)); + secp256k1_ge_set_gej(&pge, &pgej); + ge_equals_gej(&pge, &pgej2); +} + +void test_ecmult_gen_blind_reset(void) { + /* Test ecmult_gen() blinding reset and confirm that the blinding is consistent. */ + secp256k1_scalar_t b; + secp256k1_gej_t initial; + secp256k1_ecmult_gen_blind(&ctx->ecmult_gen_ctx, 0); + b = ctx->ecmult_gen_ctx.blind; + initial = ctx->ecmult_gen_ctx.initial; + secp256k1_ecmult_gen_blind(&ctx->ecmult_gen_ctx, 0); + CHECK(secp256k1_scalar_eq(&b, &ctx->ecmult_gen_ctx.blind)); + CHECK(gej_xyz_equals_gej(&initial, &ctx->ecmult_gen_ctx.initial)); +} + +void run_ecmult_gen_blind(void) { + int i; + test_ecmult_gen_blind_reset(); + for (i = 0; i < 10; i++) { + test_ecmult_gen_blind(); + } +} + + void random_sign(secp256k1_ecdsa_sig_t *sig, const secp256k1_scalar_t *key, const secp256k1_scalar_t *msg, int *recid) { secp256k1_scalar_t nonce; do { random_scalar_order_test(&nonce); - } while(!secp256k1_ecdsa_sig_sign(sig, key, msg, &nonce, recid)); + } while(!secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, sig, key, msg, &nonce, recid)); } void test_ecdsa_sign_verify(void) { @@ -1164,15 +1336,17 @@ void test_ecdsa_sign_verify(void) { int getrec; random_scalar_order_test(&msg); random_scalar_order_test(&key); - secp256k1_ecmult_gen(&pubj, &key); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pubj, &key); secp256k1_ge_set_gej(&pub, &pubj); getrec = secp256k1_rand32()&1; random_sign(&sig, &key, &msg, getrec?&recid:NULL); - if (getrec) CHECK(recid >= 0 && recid < 4); - CHECK(secp256k1_ecdsa_sig_verify(&sig, &pub, &msg)); + if (getrec) { + CHECK(recid >= 0 && recid < 4); + } + CHECK(secp256k1_ecdsa_sig_verify(&ctx->ecmult_ctx, &sig, &pub, &msg)); secp256k1_scalar_set_int(&one, 1); secp256k1_scalar_add(&msg, &msg, &one); - CHECK(!secp256k1_ecdsa_sig_verify(&sig, &pub, &msg)); + CHECK(!secp256k1_ecdsa_sig_verify(&ctx->ecmult_ctx, &sig, &pub, &msg)); } void run_ecdsa_sign_verify(void) { @@ -1192,7 +1366,9 @@ static int precomputed_nonce_function(unsigned char *nonce32, const unsigned cha static int nonce_function_test_fail(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, unsigned int counter, const void *data) { /* Dummy nonce generator that has a fatal error on the first counter value. */ - if (counter == 0) return 0; + if (counter == 0) { + return 0; + } return nonce_function_rfc6979(nonce32, msg32, key32, counter - 1, data); } @@ -1200,7 +1376,9 @@ static int nonce_function_test_retry(unsigned char *nonce32, const unsigned char /* Dummy nonce generator that produces unacceptable nonces for the first several counter values. */ if (counter < 3) { memset(nonce32, counter==0 ? 0 : 255, 32); - if (counter == 2) nonce32[31]--; + if (counter == 2) { + nonce32[31]--; + } return 1; } if (counter < 5) { @@ -1211,12 +1389,16 @@ static int nonce_function_test_retry(unsigned char *nonce32, const unsigned char 0xBF,0xD2,0x5E,0x8C,0xD0,0x36,0x41,0x41 }; memcpy(nonce32, order, 32); - if (counter == 4) nonce32[31]++; + if (counter == 4) { + nonce32[31]++; + } return 1; } /* Retry rate of 6979 is negligible esp. as we only call this in determinstic tests. */ /* If someone does fine a case where it retries for secp256k1, we'd like to know. */ - if (counter > 5) return 0; + if (counter > 5) { + return 0; + } return nonce_function_rfc6979(nonce32, msg32, key32, counter - 5, data); } @@ -1257,16 +1439,16 @@ void test_ecdsa_end_to_end(void) { } /* Construct and verify corresponding public key. */ - CHECK(secp256k1_ec_seckey_verify(privkey) == 1); - CHECK(secp256k1_ec_pubkey_create(pubkey, &pubkeylen, privkey, (secp256k1_rand32() & 3) != 0) == 1); + CHECK(secp256k1_ec_seckey_verify(ctx, privkey) == 1); + CHECK(secp256k1_ec_pubkey_create(ctx, pubkey, &pubkeylen, privkey, (secp256k1_rand32() & 3) != 0) == 1); if (secp256k1_rand32() & 1) { - CHECK(secp256k1_ec_pubkey_decompress(pubkey, &pubkeylen)); + CHECK(secp256k1_ec_pubkey_decompress(ctx, pubkey, &pubkeylen)); } - CHECK(secp256k1_ec_pubkey_verify(pubkey, pubkeylen)); + CHECK(secp256k1_ec_pubkey_verify(ctx, pubkey, pubkeylen)); /* Verify private key import and export. */ - CHECK(secp256k1_ec_privkey_export(privkey, seckey, &seckeylen, secp256k1_rand32() % 2) == 1); - CHECK(secp256k1_ec_privkey_import(privkey2, seckey, seckeylen) == 1); + CHECK(secp256k1_ec_privkey_export(ctx, privkey, seckey, &seckeylen, secp256k1_rand32() % 2) == 1); + CHECK(secp256k1_ec_privkey_import(ctx, privkey2, seckey, seckeylen) == 1); CHECK(memcmp(privkey, privkey2, 32) == 0); /* Optionally tweak the keys using addition. */ @@ -1277,11 +1459,13 @@ void test_ecdsa_end_to_end(void) { unsigned char pubkey2[65]; int pubkeylen2 = 65; secp256k1_rand256_test(rnd); - ret1 = secp256k1_ec_privkey_tweak_add(privkey, rnd); - ret2 = secp256k1_ec_pubkey_tweak_add(pubkey, pubkeylen, rnd); + ret1 = secp256k1_ec_privkey_tweak_add(ctx, privkey, rnd); + ret2 = secp256k1_ec_pubkey_tweak_add(ctx, pubkey, pubkeylen, rnd); CHECK(ret1 == ret2); - if (ret1 == 0) return; - CHECK(secp256k1_ec_pubkey_create(pubkey2, &pubkeylen2, privkey, pubkeylen == 33) == 1); + if (ret1 == 0) { + return; + } + CHECK(secp256k1_ec_pubkey_create(ctx, pubkey2, &pubkeylen2, privkey, pubkeylen == 33) == 1); CHECK(memcmp(pubkey, pubkey2, pubkeylen) == 0); } @@ -1293,25 +1477,27 @@ void test_ecdsa_end_to_end(void) { unsigned char pubkey2[65]; int pubkeylen2 = 65; secp256k1_rand256_test(rnd); - ret1 = secp256k1_ec_privkey_tweak_mul(privkey, rnd); - ret2 = secp256k1_ec_pubkey_tweak_mul(pubkey, pubkeylen, rnd); + ret1 = secp256k1_ec_privkey_tweak_mul(ctx, privkey, rnd); + ret2 = secp256k1_ec_pubkey_tweak_mul(ctx, pubkey, pubkeylen, rnd); CHECK(ret1 == ret2); - if (ret1 == 0) return; - CHECK(secp256k1_ec_pubkey_create(pubkey2, &pubkeylen2, privkey, pubkeylen == 33) == 1); + if (ret1 == 0) { + return; + } + CHECK(secp256k1_ec_pubkey_create(ctx, pubkey2, &pubkeylen2, privkey, pubkeylen == 33) == 1); CHECK(memcmp(pubkey, pubkey2, pubkeylen) == 0); } /* Sign. */ - CHECK(secp256k1_ecdsa_sign(message, signature, &signaturelen, privkey, NULL, NULL) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, message, signature, &signaturelen, privkey, NULL, NULL) == 1); CHECK(signaturelen > 0); - CHECK(secp256k1_ecdsa_sign(message, signature2, &signaturelen2, privkey, NULL, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, message, signature2, &signaturelen2, privkey, NULL, extra) == 1); CHECK(signaturelen2 > 0); extra[31] = 1; - CHECK(secp256k1_ecdsa_sign(message, signature3, &signaturelen3, privkey, NULL, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, message, signature3, &signaturelen3, privkey, NULL, extra) == 1); CHECK(signaturelen3 > 0); extra[31] = 0; extra[0] = 1; - CHECK(secp256k1_ecdsa_sign(message, signature4, &signaturelen4, privkey, NULL, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, message, signature4, &signaturelen4, privkey, NULL, extra) == 1); CHECK(signaturelen3 > 0); CHECK((signaturelen != signaturelen2) || (memcmp(signature, signature2, signaturelen) != 0)); CHECK((signaturelen != signaturelen3) || (memcmp(signature, signature3, signaturelen) != 0)); @@ -1320,24 +1506,24 @@ void test_ecdsa_end_to_end(void) { CHECK((signaturelen4 != signaturelen2) || (memcmp(signature4, signature2, signaturelen4) != 0)); CHECK((signaturelen4 != signaturelen) || (memcmp(signature4, signature, signaturelen4) != 0)); /* Verify. */ - CHECK(secp256k1_ecdsa_verify(message, signature, signaturelen, pubkey, pubkeylen) == 1); - CHECK(secp256k1_ecdsa_verify(message, signature2, signaturelen2, pubkey, pubkeylen) == 1); - CHECK(secp256k1_ecdsa_verify(message, signature3, signaturelen3, pubkey, pubkeylen) == 1); - CHECK(secp256k1_ecdsa_verify(message, signature4, signaturelen4, pubkey, pubkeylen) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, message, signature, signaturelen, pubkey, pubkeylen) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, message, signature2, signaturelen2, pubkey, pubkeylen) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, message, signature3, signaturelen3, pubkey, pubkeylen) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, message, signature4, signaturelen4, pubkey, pubkeylen) == 1); /* Destroy signature and verify again. */ signature[signaturelen - 1 - secp256k1_rand32() % 20] += 1 + (secp256k1_rand32() % 255); - CHECK(secp256k1_ecdsa_verify(message, signature, signaturelen, pubkey, pubkeylen) != 1); + CHECK(secp256k1_ecdsa_verify(ctx, message, signature, signaturelen, pubkey, pubkeylen) != 1); /* Compact sign. */ - CHECK(secp256k1_ecdsa_sign_compact(message, csignature, privkey, NULL, NULL, &recid) == 1); + CHECK(secp256k1_ecdsa_sign_compact(ctx, message, csignature, privkey, NULL, NULL, &recid) == 1); CHECK(!is_empty_compact_signature(csignature)); /* Recover. */ - CHECK(secp256k1_ecdsa_recover_compact(message, csignature, recpubkey, &recpubkeylen, pubkeylen == 33, recid) == 1); + CHECK(secp256k1_ecdsa_recover_compact(ctx, message, csignature, recpubkey, &recpubkeylen, pubkeylen == 33, recid) == 1); CHECK(recpubkeylen == pubkeylen); CHECK(memcmp(pubkey, recpubkey, pubkeylen) == 0); /* Destroy signature and verify again. */ csignature[secp256k1_rand32() % 64] += 1 + (secp256k1_rand32() % 255); - CHECK(secp256k1_ecdsa_recover_compact(message, csignature, recpubkey, &recpubkeylen, pubkeylen == 33, recid) != 1 || + CHECK(secp256k1_ecdsa_recover_compact(ctx, message, csignature, recpubkey, &recpubkeylen, pubkeylen == 33, recid) != 1 || memcmp(pubkey, recpubkey, pubkeylen) != 0); CHECK(recpubkeylen == pubkeylen); @@ -1351,7 +1537,9 @@ void test_random_pubkeys(void) { uint32_t r = secp256k1_rand32(); int len = (r & 3) == 0 ? 65 : 33; r>>=2; - if ((r & 3) == 0) len = (r & 252) >> 3; + if ((r & 3) == 0) { + len = (r & 252) >> 3; + } r>>=8; if (len == 65) { in[0] = (r & 2) ? 4 : (r & 1? 6 : 7); @@ -1359,10 +1547,16 @@ void test_random_pubkeys(void) { in[0] = (r & 1) ? 2 : 3; } r>>=2; - if ((r & 7) == 0) in[0] = (r & 2040) >> 3; + if ((r & 7) == 0) { + in[0] = (r & 2040) >> 3; + } r>>=11; - if (len > 1) secp256k1_rand256(&in[1]); - if (len > 33) secp256k1_rand256(&in[33]); + if (len > 1) { + secp256k1_rand256(&in[1]); + } + if (len > 33) { + secp256k1_rand256(&in[33]); + } if (secp256k1_eckey_pubkey_parse(&elem, in, len)) { unsigned char out[65]; unsigned char firstb; @@ -1374,7 +1568,9 @@ void test_random_pubkeys(void) { CHECK(size == len); CHECK(memcmp(&in[1], &out[1], len-1) == 0); /* ... except for the type of hybrid inputs. */ - if ((in[0] != 6) && (in[0] != 7)) CHECK(in[0] == out[0]); + if ((in[0] != 6) && (in[0] != 7)) { + CHECK(in[0] == out[0]); + } size = 65; CHECK(secp256k1_eckey_pubkey_serialize(&elem, in, &size, 0)); CHECK(size == 65); @@ -1384,8 +1580,11 @@ void test_random_pubkeys(void) { in[0] = (r & 1) ? 6 : 7; res = secp256k1_eckey_pubkey_parse(&elem2, in, size); if (firstb == 2 || firstb == 3) { - if (in[0] == firstb + 4) CHECK(res); - else CHECK(!res); + if (in[0] == firstb + 4) { + CHECK(res); + } else { + CHECK(!res); + } } if (res) { ge_equals_ge(&elem,&elem2); @@ -1447,10 +1646,10 @@ void test_ecdsa_edge_cases(void) { int pubkeyblen = 33; int recid; - CHECK(!secp256k1_ecdsa_recover_compact(msg32, sig64, pubkey, &pubkeylen, 0, 0)); - CHECK(secp256k1_ecdsa_recover_compact(msg32, sig64, pubkey, &pubkeylen, 0, 1)); - CHECK(!secp256k1_ecdsa_recover_compact(msg32, sig64, pubkey, &pubkeylen, 0, 2)); - CHECK(!secp256k1_ecdsa_recover_compact(msg32, sig64, pubkey, &pubkeylen, 0, 3)); + CHECK(!secp256k1_ecdsa_recover_compact(ctx, msg32, sig64, pubkey, &pubkeylen, 0, 0)); + CHECK(secp256k1_ecdsa_recover_compact(ctx, msg32, sig64, pubkey, &pubkeylen, 0, 1)); + CHECK(!secp256k1_ecdsa_recover_compact(ctx, msg32, sig64, pubkey, &pubkeylen, 0, 2)); + CHECK(!secp256k1_ecdsa_recover_compact(ctx, msg32, sig64, pubkey, &pubkeylen, 0, 3)); for (recid = 0; recid < 4; recid++) { int i; @@ -1495,42 +1694,44 @@ void test_ecdsa_edge_cases(void) { 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, 0x41, 0x45, 0x02, 0x01, 0x04 }; - CHECK(secp256k1_ecdsa_recover_compact(msg32, sigb64, pubkeyb, &pubkeyblen, 1, recid)); - CHECK(secp256k1_ecdsa_verify(msg32, sigbder, sizeof(sigbder), pubkeyb, pubkeyblen) == 1); + CHECK(secp256k1_ecdsa_recover_compact(ctx, msg32, sigb64, pubkeyb, &pubkeyblen, 1, recid)); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbder, sizeof(sigbder), pubkeyb, pubkeyblen) == 1); for (recid2 = 0; recid2 < 4; recid2++) { unsigned char pubkey2b[33]; int pubkey2blen = 33; - CHECK(secp256k1_ecdsa_recover_compact(msg32, sigb64, pubkey2b, &pubkey2blen, 1, recid2)); + CHECK(secp256k1_ecdsa_recover_compact(ctx, msg32, sigb64, pubkey2b, &pubkey2blen, 1, recid2)); /* Verifying with (order + r,4) should always fail. */ - CHECK(secp256k1_ecdsa_verify(msg32, sigbderlong, sizeof(sigbderlong), pubkey2b, pubkey2blen) != 1); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbderlong, sizeof(sigbderlong), pubkey2b, pubkey2blen) != 1); } /* DER parsing tests. */ /* Zero length r/s. */ - CHECK(secp256k1_ecdsa_verify(msg32, sigcder_zr, sizeof(sigcder_zr), pubkeyb, pubkeyblen) == -2); - CHECK(secp256k1_ecdsa_verify(msg32, sigcder_zs, sizeof(sigcder_zs), pubkeyb, pubkeyblen) == -2); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigcder_zr, sizeof(sigcder_zr), pubkeyb, pubkeyblen) == -2); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigcder_zs, sizeof(sigcder_zs), pubkeyb, pubkeyblen) == -2); /* Leading zeros. */ - CHECK(secp256k1_ecdsa_verify(msg32, sigbderalt1, sizeof(sigbderalt1), pubkeyb, pubkeyblen) == 1); - CHECK(secp256k1_ecdsa_verify(msg32, sigbderalt2, sizeof(sigbderalt2), pubkeyb, pubkeyblen) == 1); - CHECK(secp256k1_ecdsa_verify(msg32, sigbderalt3, sizeof(sigbderalt3), pubkeyb, pubkeyblen) == 1); - CHECK(secp256k1_ecdsa_verify(msg32, sigbderalt4, sizeof(sigbderalt4), pubkeyb, pubkeyblen) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbderalt1, sizeof(sigbderalt1), pubkeyb, pubkeyblen) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbderalt2, sizeof(sigbderalt2), pubkeyb, pubkeyblen) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbderalt3, sizeof(sigbderalt3), pubkeyb, pubkeyblen) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbderalt4, sizeof(sigbderalt4), pubkeyb, pubkeyblen) == 1); sigbderalt3[4] = 1; - CHECK(secp256k1_ecdsa_verify(msg32, sigbderalt3, sizeof(sigbderalt3), pubkeyb, pubkeyblen) == -2); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbderalt3, sizeof(sigbderalt3), pubkeyb, pubkeyblen) == -2); sigbderalt4[7] = 1; - CHECK(secp256k1_ecdsa_verify(msg32, sigbderalt4, sizeof(sigbderalt4), pubkeyb, pubkeyblen) == -2); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbderalt4, sizeof(sigbderalt4), pubkeyb, pubkeyblen) == -2); /* Damage signature. */ sigbder[7]++; - CHECK(secp256k1_ecdsa_verify(msg32, sigbder, sizeof(sigbder), pubkeyb, pubkeyblen) == 0); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbder, sizeof(sigbder), pubkeyb, pubkeyblen) == 0); sigbder[7]--; - CHECK(secp256k1_ecdsa_verify(msg32, sigbder, 6, pubkeyb, pubkeyblen) == -2); - CHECK(secp256k1_ecdsa_verify(msg32, sigbder, sizeof(sigbder)-1, pubkeyb, pubkeyblen) == -2); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbder, 6, pubkeyb, pubkeyblen) == -2); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbder, sizeof(sigbder)-1, pubkeyb, pubkeyblen) == -2); for(i = 0; i < 8; i++) { int c; unsigned char orig = sigbder[i]; /*Try every single-byte change.*/ for (c = 0; c < 256; c++) { - if (c == orig ) continue; + if (c == orig ) { + continue; + } sigbder[i] = c; - CHECK(secp256k1_ecdsa_verify(msg32, sigbder, sizeof(sigbder), pubkeyb, pubkeyblen) == + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigbder, sizeof(sigbder), pubkeyb, pubkeyblen) == (i==4 || i==7) ? 0 : -2 ); } sigbder[i] = orig; @@ -1547,10 +1748,10 @@ void test_ecdsa_edge_cases(void) { secp256k1_scalar_negate(&sig.s, &sig.s); secp256k1_scalar_inverse(&sig.s, &sig.s); secp256k1_scalar_set_int(&sig.r, 1); - secp256k1_ecmult_gen(&keyj, &sig.r); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &keyj, &sig.r); secp256k1_ge_set_gej(&key, &keyj); msg = sig.s; - CHECK(secp256k1_ecdsa_sig_verify(&sig, &key, &msg) == 0); + CHECK(secp256k1_ecdsa_sig_verify(&ctx->ecmult_ctx, &sig, &key, &msg) == 0); } /* Test r/s equal to zero */ @@ -1569,18 +1770,18 @@ void test_ecdsa_edge_cases(void) { }; unsigned char pubkeyc[65]; int pubkeyclen = 65; - CHECK(secp256k1_ecdsa_recover_compact(msg32, sigc64, pubkeyc, &pubkeyclen, 0, 0) == 1); - CHECK(secp256k1_ecdsa_verify(msg32, sigcder, sizeof(sigcder), pubkeyc, pubkeyclen) == 1); + CHECK(secp256k1_ecdsa_recover_compact(ctx, msg32, sigc64, pubkeyc, &pubkeyclen, 0, 0) == 1); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigcder, sizeof(sigcder), pubkeyc, pubkeyclen) == 1); sigcder[4] = 0; sigc64[31] = 0; - CHECK(secp256k1_ecdsa_recover_compact(msg32, sigc64, pubkeyb, &pubkeyblen, 1, 0) == 0); - CHECK(secp256k1_ecdsa_verify(msg32, sigcder, sizeof(sigcder), pubkeyc, pubkeyclen) == 0); + CHECK(secp256k1_ecdsa_recover_compact(ctx, msg32, sigc64, pubkeyb, &pubkeyblen, 1, 0) == 0); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigcder, sizeof(sigcder), pubkeyc, pubkeyclen) == 0); sigcder[4] = 1; sigcder[7] = 0; sigc64[31] = 1; sigc64[63] = 0; - CHECK(secp256k1_ecdsa_recover_compact(msg32, sigc64, pubkeyb, &pubkeyblen, 1, 0) == 0); - CHECK(secp256k1_ecdsa_verify(msg32, sigcder, sizeof(sigcder), pubkeyc, pubkeyclen) == 0); + CHECK(secp256k1_ecdsa_recover_compact(ctx, msg32, sigc64, pubkeyb, &pubkeyblen, 1, 0) == 0); + CHECK(secp256k1_ecdsa_verify(ctx, msg32, sigcder, sizeof(sigcder), pubkeyc, pubkeyclen) == 0); } /*Signature where s would be zero.*/ @@ -1611,18 +1812,18 @@ void test_ecdsa_edge_cases(void) { }; unsigned char sig[72]; int siglen = 72; - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, precomputed_nonce_function, nonce) == 0); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, precomputed_nonce_function, nonce) == 0); CHECK(siglen == 0); - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, precomputed_nonce_function, nonce2) == 0); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, precomputed_nonce_function, nonce2) == 0); CHECK(siglen == 0); msg[31] = 0xaa; siglen = 72; - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, precomputed_nonce_function, nonce) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, precomputed_nonce_function, nonce) == 1); CHECK(siglen > 0); - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, precomputed_nonce_function, nonce2) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, precomputed_nonce_function, nonce2) == 1); CHECK(siglen > 0); siglen = 10; - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, precomputed_nonce_function, nonce) != 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, precomputed_nonce_function, nonce) != 1); CHECK(siglen == 0); } @@ -1644,41 +1845,41 @@ void test_ecdsa_edge_cases(void) { msg[31] = 1; /* High key results in signature failure. */ memset(key, 0xFF, 32); - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, NULL, extra) == 0); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, NULL, extra) == 0); CHECK(siglen == 0); /* Zero key results in signature failure. */ memset(key, 0, 32); - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, NULL, extra) == 0); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, NULL, extra) == 0); CHECK(siglen == 0); /* Nonce function failure results in signature failure. */ key[31] = 1; - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, nonce_function_test_fail, extra) == 0); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, nonce_function_test_fail, extra) == 0); CHECK(siglen == 0); - CHECK(secp256k1_ecdsa_sign_compact(msg, sig, key, nonce_function_test_fail, extra, &recid) == 0); + CHECK(secp256k1_ecdsa_sign_compact(ctx, msg, sig, key, nonce_function_test_fail, extra, &recid) == 0); CHECK(is_empty_compact_signature(sig)); /* The retry loop successfully makes its way to the first good value. */ siglen = 72; - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, nonce_function_test_retry, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, nonce_function_test_retry, extra) == 1); CHECK(siglen > 0); - CHECK(secp256k1_ecdsa_sign(msg, sig2, &siglen2, key, nonce_function_rfc6979, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig2, &siglen2, key, nonce_function_rfc6979, extra) == 1); CHECK(siglen > 0); CHECK((siglen == siglen2) && (memcmp(sig, sig2, siglen) == 0)); - CHECK(secp256k1_ecdsa_sign_compact(msg, sig, key, nonce_function_test_retry, extra, &recid) == 1); + CHECK(secp256k1_ecdsa_sign_compact(ctx, msg, sig, key, nonce_function_test_retry, extra, &recid) == 1); CHECK(!is_empty_compact_signature(sig)); - CHECK(secp256k1_ecdsa_sign_compact(msg, sig2, key, nonce_function_rfc6979, extra, &recid2) == 1); + CHECK(secp256k1_ecdsa_sign_compact(ctx, msg, sig2, key, nonce_function_rfc6979, extra, &recid2) == 1); CHECK(!is_empty_compact_signature(sig2)); CHECK((recid == recid2) && (memcmp(sig, sig2, 64) == 0)); /* The default nonce function is determinstic. */ siglen = 72; siglen2 = 72; - CHECK(secp256k1_ecdsa_sign(msg, sig, &siglen, key, NULL, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig, &siglen, key, NULL, extra) == 1); CHECK(siglen > 0); - CHECK(secp256k1_ecdsa_sign(msg, sig2, &siglen2, key, NULL, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig2, &siglen2, key, NULL, extra) == 1); CHECK(siglen2 > 0); CHECK((siglen == siglen2) && (memcmp(sig, sig2, siglen) == 0)); - CHECK(secp256k1_ecdsa_sign_compact(msg, sig, key, NULL, extra, &recid) == 1); + CHECK(secp256k1_ecdsa_sign_compact(ctx, msg, sig, key, NULL, extra, &recid) == 1); CHECK(!is_empty_compact_signature(sig)); - CHECK(secp256k1_ecdsa_sign_compact(msg, sig2, key, NULL, extra, &recid2) == 1); + CHECK(secp256k1_ecdsa_sign_compact(ctx, msg, sig2, key, NULL, extra, &recid2) == 1); CHECK(!is_empty_compact_signature(sig)); CHECK((recid == recid2) && (memcmp(sig, sig2, 64) == 0)); /* The default nonce function changes output with different messages. */ @@ -1686,7 +1887,7 @@ void test_ecdsa_edge_cases(void) { int j; siglen2 = 72; msg[0] = i; - CHECK(secp256k1_ecdsa_sign(msg, sig2, &siglen2, key, NULL, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig2, &siglen2, key, NULL, extra) == 1); CHECK(!is_empty_compact_signature(sig)); CHECK(secp256k1_ecdsa_sig_parse(&s[i], sig2, siglen2)); for (j = 0; j < i; j++) { @@ -1700,7 +1901,7 @@ void test_ecdsa_edge_cases(void) { int j; siglen2 = 72; key[0] = i - 256; - CHECK(secp256k1_ecdsa_sign(msg, sig2, &siglen2, key, NULL, extra) == 1); + CHECK(secp256k1_ecdsa_sign(ctx, msg, sig2, &siglen2, key, NULL, extra) == 1); CHECK(secp256k1_ecdsa_sig_parse(&s[i], sig2, siglen2)); for (j = 0; j < i; j++) { CHECK(!secp256k1_scalar_eq(&s[i].r, &s[j].r)); @@ -1719,8 +1920,8 @@ void test_ecdsa_edge_cases(void) { 0xbf, 0xd2, 0x5e, 0x8c, 0xd0, 0x36, 0x41, 0x41, }; int outlen = 300; - CHECK(!secp256k1_ec_privkey_export(seckey, privkey, &outlen, 0)); - CHECK(!secp256k1_ec_privkey_export(seckey, privkey, &outlen, 1)); + CHECK(!secp256k1_ec_privkey_export(ctx, seckey, privkey, &outlen, 0)); + CHECK(!secp256k1_ec_privkey_export(ctx, seckey, privkey, &outlen, 1)); } } @@ -1735,7 +1936,7 @@ EC_KEY *get_openssl_key(const secp256k1_scalar_t *key) { const unsigned char* pbegin = privkey; int compr = secp256k1_rand32() & 1; EC_KEY *ec_key = EC_KEY_new_by_curve_name(NID_secp256k1); - CHECK(secp256k1_eckey_privkey_serialize(privkey, &privkeylen, key, compr)); + CHECK(secp256k1_eckey_privkey_serialize(&ctx->ecmult_gen_ctx, privkey, &privkeylen, key, compr)); CHECK(d2i_ECPrivateKey(&ec_key, &pbegin, privkeylen)); CHECK(EC_KEY_check_key(ec_key)); return ec_key; @@ -1756,16 +1957,16 @@ void test_ecdsa_openssl(void) { secp256k1_rand256_test(message); secp256k1_scalar_set_b32(&msg, message, NULL); random_scalar_order_test(&key); - secp256k1_ecmult_gen(&qj, &key); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &qj, &key); secp256k1_ge_set_gej(&q, &qj); ec_key = get_openssl_key(&key); CHECK(ec_key); CHECK(ECDSA_sign(0, message, sizeof(message), signature, &sigsize, ec_key)); CHECK(secp256k1_ecdsa_sig_parse(&sig, signature, sigsize)); - CHECK(secp256k1_ecdsa_sig_verify(&sig, &q, &msg)); + CHECK(secp256k1_ecdsa_sig_verify(&ctx->ecmult_ctx, &sig, &q, &msg)); secp256k1_scalar_set_int(&one, 1); secp256k1_scalar_add(&msg2, &msg, &one); - CHECK(!secp256k1_ecdsa_sig_verify(&sig, &q, &msg2)); + CHECK(!secp256k1_ecdsa_sig_verify(&ctx->ecmult_ctx, &sig, &q, &msg2)); random_sign(&sig, &key, &msg, NULL); CHECK(secp256k1_ecdsa_sig_serialize(signature, &secp_sigsize, &sig)); @@ -1825,10 +2026,13 @@ int main(int argc, char **argv) { printf("random seed = %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n", seed16[0], seed16[1], seed16[2], seed16[3], seed16[4], seed16[5], seed16[6], seed16[7], seed16[8], seed16[9], seed16[10], seed16[11], seed16[12], seed16[13], seed16[14], seed16[15]); /* initialize */ - secp256k1_start(SECP256K1_START_SIGN | SECP256K1_START_VERIFY); + run_context_tests(); + ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - /* initializing a second time shouldn't cause any harm or memory leaks. */ - secp256k1_start(SECP256K1_START_SIGN | SECP256K1_START_VERIFY); + if (secp256k1_rand32() & 1) { + secp256k1_rand256(run32); + CHECK(secp256k1_context_randomize(ctx, secp256k1_rand32() & 1 ? run32 : NULL)); + } run_sha256_tests(); run_hmac_sha256_tests(); @@ -1858,6 +2062,8 @@ int main(int argc, char **argv) { run_wnaf(); run_point_times_order(); run_ecmult_chain(); + run_ecmult_constants(); + run_ecmult_gen_blind(); /* ecdsa tests */ run_random_pubkeys(); @@ -1872,9 +2078,6 @@ int main(int argc, char **argv) { printf("random run = %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n", run32[0], run32[1], run32[2], run32[3], run32[4], run32[5], run32[6], run32[7], run32[8], run32[9], run32[10], run32[11], run32[12], run32[13], run32[14], run32[15]); /* shutdown */ - secp256k1_stop(); - - /* shutting down twice shouldn't cause any double frees. */ - secp256k1_stop(); + secp256k1_context_destroy(ctx); return 0; } diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 2e2cc2214b..34b311b804 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -59,6 +59,24 @@ public: bool GetStats(CCoinsStats& stats) const { return false; } }; + +class CCoinsViewCacheTest : public CCoinsViewCache +{ +public: + CCoinsViewCacheTest(CCoinsView* base) : CCoinsViewCache(base) {} + + void SelfTest() const + { + // Manually recompute the dynamic usage of the whole data, and compare it. + size_t ret = memusage::DynamicUsage(cacheCoins); + for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { + ret += memusage::DynamicUsage(it->second.coins); + } + BOOST_CHECK_EQUAL(memusage::DynamicUsage(*this), ret); + } + +}; + } BOOST_FIXTURE_TEST_SUITE(coins_tests, BasicTestingSetup) @@ -90,8 +108,8 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. - std::vector<CCoinsViewCache*> stack; // A stack of CCoinsViewCaches on top. - stack.push_back(new CCoinsViewCache(&base)); // Start with one cache. + std::vector<CCoinsViewCacheTest*> stack; // A stack of CCoinsViewCaches on top. + stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. // Use a limited set of random transaction ids, so we do test overwriting entries. std::vector<uint256> txids; @@ -136,6 +154,9 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) missed_an_entry = true; } } + BOOST_FOREACH(const CCoinsViewCacheTest *test, stack) { + test->SelfTest(); + } } if (insecure_rand() % 100 == 0) { @@ -152,7 +173,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) } else { removed_all_caches = true; } - stack.push_back(new CCoinsViewCache(tip)); + stack.push_back(new CCoinsViewCacheTest(tip)); if (stack.size() == 4) { reached_4_caches = true; } diff --git a/src/test/data/script_valid.json b/src/test/data/script_valid.json index 064dde8464..a4e15faeaf 100644 --- a/src/test/data/script_valid.json +++ b/src/test/data/script_valid.json @@ -74,12 +74,14 @@ ["1 1", "VERIFY", "P2SH,STRICTENC"], ["1 0x05 0x01 0x00 0x00 0x00 0x00", "VERIFY", "P2SH,STRICTENC", "values >4 bytes can be cast to boolean"], +["1 0x01 0x80", "IF 0 ENDIF", "P2SH,STRICTENC", "negative 0 is false"], ["10 0 11 TOALTSTACK DROP FROMALTSTACK", "ADD 21 EQUAL", "P2SH,STRICTENC"], ["'gavin_was_here' TOALTSTACK 11 FROMALTSTACK", "'gavin_was_here' EQUALVERIFY 11 EQUAL", "P2SH,STRICTENC"], ["0 IFDUP", "DEPTH 1 EQUALVERIFY 0 EQUAL", "P2SH,STRICTENC"], ["1 IFDUP", "DEPTH 2 EQUALVERIFY 1 EQUALVERIFY 1 EQUAL", "P2SH,STRICTENC"], +["0x05 0x0100000000 IFDUP", "DEPTH 2 EQUALVERIFY 0x05 0x0100000000 EQUAL", "P2SH,STRICTENC", "IFDUP dups non ints"], ["0 DROP", "DEPTH 0 EQUAL", "P2SH,STRICTENC"], ["0", "DUP 1 ADD 1 EQUALVERIFY 0 EQUAL", "P2SH,STRICTENC"], ["0 1", "NIP", "P2SH,STRICTENC"], @@ -408,6 +410,7 @@ ["0 0", "EQUAL", "P2SH,STRICTENC"], ["0 0", "EQUALVERIFY 1", "P2SH,STRICTENC"], +["0 0 1", "EQUAL EQUAL", "P2SH,STRICTENC", "OP_0 and bools must have identical byte representations"], ["0", "1ADD", "P2SH,STRICTENC"], ["2", "1SUB", "P2SH,STRICTENC"], diff --git a/src/test/data/tx_invalid.json b/src/test/data/tx_invalid.json index 456e0d2f7b..31d33c63fb 100644 --- a/src/test/data/tx_invalid.json +++ b/src/test/data/tx_invalid.json @@ -19,6 +19,12 @@ [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0x5b6462475454710f3c22f5fdf0b40704c92f25c3 EQUALVERIFY CHECKSIGVERIFY 1 0x4c 0x47 0x3044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a01"]], "01000000010001000000000000000000000000000000000000000000000000000000000000000000006b4c473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "P2SH"], +["This is the nearly-standard transaction with CHECKSIGVERIFY 1 instead of CHECKSIG from tx_valid.json"], +["but with the signature duplicated in the scriptPubKey with a different hashtype suffix"], +["See FindAndDelete, which will only remove if the signature, including the hash type, matches"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0x5b6462475454710f3c22f5fdf0b40704c92f25c3 EQUALVERIFY CHECKSIGVERIFY 1 0x47 0x3044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a81"]], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "P2SH"], + ["An invalid P2SH Transaction"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7a052c840ba73af26755de42cf01cc9e0a49fef0 EQUAL"]], "010000000100010000000000000000000000000000000000000000000000000000000000000000000009085768617420697320ffffffff010000000000000000015100000000", "P2SH"], diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp new file mode 100644 index 0000000000..cb64ee7c69 --- /dev/null +++ b/src/test/policyestimator_tests.cpp @@ -0,0 +1,186 @@ +// Copyright (c) 2011-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "policy/fees.h" +#include "txmempool.h" +#include "uint256.h" +#include "util.h" + +#include "test/test_bitcoin.h" + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) +{ + CTxMemPool mpool(CFeeRate(1000)); + CAmount basefee(2000); + double basepri = 1e6; + CAmount deltaFee(100); + double deltaPri=5e5; + std::vector<CAmount> feeV[2]; + std::vector<double> priV[2]; + + // Populate vectors of increasing fees or priorities + for (int j = 0; j < 10; j++) { + //V[0] is for fee transactions + feeV[0].push_back(basefee * (j+1)); + priV[0].push_back(0); + //V[1] is for priority transactions + feeV[1].push_back(CAmount(0)); + priV[1].push_back(basepri * pow(10, j+1)); + } + + // Store the hashes of transactions that have been + // added to the mempool by their associate fee/pri + // txHashes[j] is populated with transactions either of + // fee = basefee * (j+1) OR pri = 10^6 * 10^(j+1) + std::vector<uint256> txHashes[10]; + + // Create a transaction template + CScript garbage; + for (unsigned int i = 0; i < 128; i++) + garbage.push_back('X'); + CMutableTransaction tx; + std::list<CTransaction> dummyConflicted; + tx.vin.resize(1); + tx.vin[0].scriptSig = garbage; + tx.vout.resize(1); + tx.vout[0].nValue=0LL; + CFeeRate baseRate(basefee, ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); + + // Create a fake block + std::vector<CTransaction> block; + int blocknum = 0; + + // Loop through 200 blocks + // At a decay .998 and 4 fee transactions per block + // This makes the tx count about 1.33 per bucket, above the 1 threshold + while (blocknum < 200) { + for (int j = 0; j < 10; j++) { // For each fee/pri multiple + for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + tx.vin[0].prevout.n = 10000*blocknum+100*j+k; // make transaction unique + uint256 hash = tx.GetHash(); + mpool.addUnchecked(hash, CTxMemPoolEntry(tx, feeV[k/4][j], GetTime(), priV[k/4][j], blocknum, mpool.HasNoInputsOf(tx))); + txHashes[j].push_back(hash); + } + } + //Create blocks where higher fee/pri txs are included more often + for (int h = 0; h <= blocknum%10; h++) { + // 10/10 blocks add highest fee/pri transactions + // 9/10 blocks add 2nd highest and so on until ... + // 1/10 blocks add lowest fee/pri transactions + while (txHashes[9-h].size()) { + CTransaction btx; + if (mpool.lookup(txHashes[9-h].back(), btx)) + block.push_back(btx); + txHashes[9-h].pop_back(); + } + } + mpool.removeForBlock(block, ++blocknum, dummyConflicted); + block.clear(); + if (blocknum == 30) { + // At this point we should need to combine 5 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 8*baserate + BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(mpool.estimateFee(2).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(mpool.estimateFee(2).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); + } + } + + std::vector<CAmount> origFeeEst; + std::vector<double> origPriEst; + // Highest feerate is 10*baseRate and gets in all blocks, + // second highest feerate is 9*baseRate and gets in 9/10 blocks = 90%, + // third highest feerate is 8*base rate, and gets in 8/10 blocks = 80%, + // so estimateFee(1) should return 9*baseRate. + // Third highest feerate has 90% chance of being included by 2 blocks, + // so estimateFee(2) should return 8*baseRate etc... + for (int i = 1; i < 10;i++) { + origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); + origPriEst.push_back(mpool.estimatePriority(i)); + if (i > 1) { // Fee estimates should be monotonically decreasing + BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); + BOOST_CHECK(origPriEst[i-1] <= origPriEst[i-2]); + } + BOOST_CHECK(origFeeEst[i-1] < (10-i)*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(origFeeEst[i-1] > (10-i)*baseRate.GetFeePerK() - deltaFee); + BOOST_CHECK(origPriEst[i-1] < pow(10,10-i) * basepri + deltaPri); + BOOST_CHECK(origPriEst[i-1] > pow(10,10-i) * basepri - deltaPri); + } + + // Mine 50 more blocks with no transactions happening, estimates shouldn't change + // We haven't decayed the moving average enough so we still have enough data points in every bucket + while (blocknum < 250) + mpool.removeForBlock(block, ++blocknum, dummyConflicted); + + for (int i = 1; i < 10;i++) { + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i-1] + deltaPri); + BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); + } + + + // Mine 15 more blocks with lots of transactions happening and not getting mined + // Estimates should go up + while (blocknum < 265) { + for (int j = 0; j < 10; j++) { // For each fee/pri multiple + for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + tx.vin[0].prevout.n = 10000*blocknum+100*j+k; + uint256 hash = tx.GetHash(); + mpool.addUnchecked(hash, CTxMemPoolEntry(tx, feeV[k/4][j], GetTime(), priV[k/4][j], blocknum, mpool.HasNoInputsOf(tx))); + txHashes[j].push_back(hash); + } + } + mpool.removeForBlock(block, ++blocknum, dummyConflicted); + } + + for (int i = 1; i < 10;i++) { + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); + } + + // Mine all those transactions + // Estimates should still not be below original + for (int j = 0; j < 10; j++) { + while(txHashes[j].size()) { + CTransaction btx; + if (mpool.lookup(txHashes[j].back(), btx)) + block.push_back(btx); + txHashes[j].pop_back(); + } + } + mpool.removeForBlock(block, 265, dummyConflicted); + block.clear(); + for (int i = 1; i < 10;i++) { + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); + } + + // Mine 100 more blocks where everything is mined every block + // Estimates should be below original estimates (not possible for last estimate) + while (blocknum < 365) { + for (int j = 0; j < 10; j++) { // For each fee/pri multiple + for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + tx.vin[0].prevout.n = 10000*blocknum+100*j+k; + uint256 hash = tx.GetHash(); + mpool.addUnchecked(hash, CTxMemPoolEntry(tx, feeV[k/4][j], GetTime(), priV[k/4][j], blocknum, mpool.HasNoInputsOf(tx))); + CTransaction btx; + if (mpool.lookup(hash, btx)) + block.push_back(btx); + } + } + mpool.removeForBlock(block, ++blocknum, dummyConflicted); + block.clear(); + } + for (int i = 1; i < 9; i++) { + BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); + BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i-1] - deltaPri); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp new file mode 100644 index 0000000000..a26d0afaed --- /dev/null +++ b/src/test/scheduler_tests.cpp @@ -0,0 +1,110 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "random.h" +#include "scheduler.h" + +#include "test/test_bitcoin.h" + +#include <boost/bind.hpp> +#include <boost/random/mersenne_twister.hpp> +#include <boost/random/uniform_int_distribution.hpp> +#include <boost/thread.hpp> +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(scheduler_tests) + +static void microTask(CScheduler& s, boost::mutex& mutex, int& counter, int delta, boost::chrono::system_clock::time_point rescheduleTime) +{ + { + boost::unique_lock<boost::mutex> lock(mutex); + counter += delta; + } + boost::chrono::system_clock::time_point noTime = boost::chrono::system_clock::time_point::min(); + if (rescheduleTime != noTime) { + CScheduler::Function f = boost::bind(µTask, boost::ref(s), boost::ref(mutex), boost::ref(counter), -delta + 1, noTime); + s.schedule(f, rescheduleTime); + } +} + +static void MicroSleep(uint64_t n) +{ +#if defined(HAVE_WORKING_BOOST_SLEEP_FOR) + boost::this_thread::sleep_for(boost::chrono::microseconds(n)); +#elif defined(HAVE_WORKING_BOOST_SLEEP) + boost::this_thread::sleep(boost::posix_time::microseconds(n)); +#else + //should never get here + #error missing boost sleep implementation +#endif +} + +BOOST_AUTO_TEST_CASE(manythreads) +{ + // Stress test: hundreds of microsecond-scheduled tasks, + // serviced by 10 threads. + // + // So... ten shared counters, which if all the tasks execute + // properly will sum to the number of tasks done. + // Each task adds or subtracts from one of the counters a + // random amount, and then schedules another task 0-1000 + // microseconds in the future to subtract or add from + // the counter -random_amount+1, so in the end the shared + // counters should sum to the number of initial tasks performed. + CScheduler microTasks; + + boost::thread_group microThreads; + for (int i = 0; i < 5; i++) + microThreads.create_thread(boost::bind(&CScheduler::serviceQueue, µTasks)); + + boost::mutex counterMutex[10]; + int counter[10] = { 0 }; + boost::random::mt19937 rng(insecure_rand()); + boost::random::uniform_int_distribution<> zeroToNine(0, 9); + boost::random::uniform_int_distribution<> randomMsec(-11, 1000); + boost::random::uniform_int_distribution<> randomDelta(-1000, 1000); + + boost::chrono::system_clock::time_point start = boost::chrono::system_clock::now(); + boost::chrono::system_clock::time_point now = start; + + for (int i = 0; i < 100; i++) { + boost::chrono::system_clock::time_point t = now + boost::chrono::microseconds(randomMsec(rng)); + boost::chrono::system_clock::time_point tReschedule = now + boost::chrono::microseconds(500 + randomMsec(rng)); + int whichCounter = zeroToNine(rng); + CScheduler::Function f = boost::bind(µTask, boost::ref(microTasks), + boost::ref(counterMutex[whichCounter]), boost::ref(counter[whichCounter]), + randomDelta(rng), tReschedule); + microTasks.schedule(f, t); + } + + MicroSleep(600); + now = boost::chrono::system_clock::now(); + // More threads and more tasks: + for (int i = 0; i < 5; i++) + microThreads.create_thread(boost::bind(&CScheduler::serviceQueue, µTasks)); + for (int i = 0; i < 100; i++) { + boost::chrono::system_clock::time_point t = now + boost::chrono::microseconds(randomMsec(rng)); + boost::chrono::system_clock::time_point tReschedule = now + boost::chrono::microseconds(500 + randomMsec(rng)); + int whichCounter = zeroToNine(rng); + CScheduler::Function f = boost::bind(µTask, boost::ref(microTasks), + boost::ref(counterMutex[whichCounter]), boost::ref(counter[whichCounter]), + randomDelta(rng), tReschedule); + microTasks.schedule(f, t); + } + + // All 2,000 tasks should be finished within 2 milliseconds. Sleep a bit longer. + MicroSleep(2100); + + microThreads.interrupt_all(); + microThreads.join_all(); + + int counterSum = 0; + for (int i = 0; i < 10; i++) { + BOOST_CHECK(counter[i] != 0); + counterSum += counter[i]; + } + BOOST_CHECK_EQUAL(counterSum, 200); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 4057eccbed..c727303ea1 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -6,6 +6,7 @@ #include "test_bitcoin.h" +#include "key.h" #include "main.h" #include "random.h" #include "txdb.h" @@ -28,6 +29,7 @@ extern void noui_connect(); BasicTestingSetup::BasicTestingSetup() { + ECC_Start(); SetupEnvironment(); fPrintToDebugLog = false; // don't want to write to debug.log file fCheckBlockIndex = true; @@ -35,6 +37,7 @@ BasicTestingSetup::BasicTestingSetup() } BasicTestingSetup::~BasicTestingSetup() { + ECC_Stop(); } TestingSetup::TestingSetup() diff --git a/src/txdb.h b/src/txdb.h index 86e1c5d831..bef5dc9fd1 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -22,7 +22,7 @@ class uint256; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 100; //! max. -dbcache in (MiB) -static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 4096 : 1024; +static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache in (MiB) static const int64_t nMinDbCache = 4; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index d05e3c6eec..071fa9d52c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -8,28 +8,27 @@ #include "clientversion.h" #include "consensus/consensus.h" #include "main.h" +#include "policy/fees.h" #include "streams.h" #include "util.h" #include "utilmoneystr.h" #include "version.h" -#include <boost/circular_buffer.hpp> - using namespace std; CTxMemPoolEntry::CTxMemPoolEntry(): - nFee(0), nTxSize(0), nModSize(0), nTime(0), dPriority(0.0) + nFee(0), nTxSize(0), nModSize(0), nTime(0), dPriority(0.0), hadNoDependencies(false) { nHeight = MEMPOOL_HEIGHT; } CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _dPriority, - unsigned int _nHeight): - tx(_tx), nFee(_nFee), nTime(_nTime), dPriority(_dPriority), nHeight(_nHeight) + unsigned int _nHeight, bool poolHasNoInputsOf): + tx(_tx), nFee(_nFee), nTime(_nTime), dPriority(_dPriority), nHeight(_nHeight), + hadNoDependencies(poolHasNoInputsOf) { nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); - nModSize = tx.CalculateModifiedSize(nTxSize); } @@ -47,346 +46,15 @@ CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const return dResult; } -/** - * Keep track of fee/priority for transactions confirmed within N blocks - */ -class CBlockAverage -{ -private: - boost::circular_buffer<CFeeRate> feeSamples; - boost::circular_buffer<double> prioritySamples; - - template<typename T> std::vector<T> buf2vec(boost::circular_buffer<T> buf) const - { - std::vector<T> vec(buf.begin(), buf.end()); - return vec; - } - -public: - CBlockAverage() : feeSamples(100), prioritySamples(100) { } - - void RecordFee(const CFeeRate& feeRate) { - feeSamples.push_back(feeRate); - } - - void RecordPriority(double priority) { - prioritySamples.push_back(priority); - } - - size_t FeeSamples() const { return feeSamples.size(); } - size_t GetFeeSamples(std::vector<CFeeRate>& insertInto) const - { - BOOST_FOREACH(const CFeeRate& f, feeSamples) - insertInto.push_back(f); - return feeSamples.size(); - } - size_t PrioritySamples() const { return prioritySamples.size(); } - size_t GetPrioritySamples(std::vector<double>& insertInto) const - { - BOOST_FOREACH(double d, prioritySamples) - insertInto.push_back(d); - return prioritySamples.size(); - } - - /** - * Used as belt-and-suspenders check when reading to detect - * file corruption - */ - static bool AreSane(const CFeeRate fee, const CFeeRate& minRelayFee) - { - if (fee < CFeeRate(0)) - return false; - if (fee.GetFeePerK() > minRelayFee.GetFeePerK() * 10000) - return false; - return true; - } - static bool AreSane(const std::vector<CFeeRate>& vecFee, const CFeeRate& minRelayFee) - { - BOOST_FOREACH(CFeeRate fee, vecFee) - { - if (!AreSane(fee, minRelayFee)) - return false; - } - return true; - } - static bool AreSane(const double priority) - { - return priority >= 0; - } - static bool AreSane(const std::vector<double> vecPriority) - { - BOOST_FOREACH(double priority, vecPriority) - { - if (!AreSane(priority)) - return false; - } - return true; - } - - void Write(CAutoFile& fileout) const - { - std::vector<CFeeRate> vecFee = buf2vec(feeSamples); - fileout << vecFee; - std::vector<double> vecPriority = buf2vec(prioritySamples); - fileout << vecPriority; - } - - void Read(CAutoFile& filein, const CFeeRate& minRelayFee) { - std::vector<CFeeRate> vecFee; - filein >> vecFee; - if (AreSane(vecFee, minRelayFee)) - feeSamples.insert(feeSamples.end(), vecFee.begin(), vecFee.end()); - else - throw runtime_error("Corrupt fee value in estimates file."); - std::vector<double> vecPriority; - filein >> vecPriority; - if (AreSane(vecPriority)) - prioritySamples.insert(prioritySamples.end(), vecPriority.begin(), vecPriority.end()); - else - throw runtime_error("Corrupt priority value in estimates file."); - if (feeSamples.size() + prioritySamples.size() > 0) - LogPrint("estimatefee", "Read %d fee samples and %d priority samples\n", - feeSamples.size(), prioritySamples.size()); - } -}; - -class CMinerPolicyEstimator -{ -private: - /** - * Records observed averages transactions that confirmed within one block, two blocks, - * three blocks etc. - */ - std::vector<CBlockAverage> history; - std::vector<CFeeRate> sortedFeeSamples; - std::vector<double> sortedPrioritySamples; - - int nBestSeenHeight; - - /** - * nBlocksAgo is 0 based, i.e. transactions that confirmed in the highest seen block are - * nBlocksAgo == 0, transactions in the block before that are nBlocksAgo == 1 etc. - */ - void seenTxConfirm(const CFeeRate& feeRate, const CFeeRate& minRelayFee, double dPriority, int nBlocksAgo) - { - // Last entry records "everything else". - int nBlocksTruncated = min(nBlocksAgo, (int) history.size() - 1); - assert(nBlocksTruncated >= 0); - - // We need to guess why the transaction was included in a block-- either - // because it is high-priority or because it has sufficient fees. - bool sufficientFee = (feeRate > minRelayFee); - bool sufficientPriority = AllowFree(dPriority); - const char* assignedTo = "unassigned"; - if (sufficientFee && !sufficientPriority && CBlockAverage::AreSane(feeRate, minRelayFee)) - { - history[nBlocksTruncated].RecordFee(feeRate); - assignedTo = "fee"; - } - else if (sufficientPriority && !sufficientFee && CBlockAverage::AreSane(dPriority)) - { - history[nBlocksTruncated].RecordPriority(dPriority); - assignedTo = "priority"; - } - else - { - // Neither or both fee and priority sufficient to get confirmed: - // don't know why they got confirmed. - } - LogPrint("estimatefee", "Seen TX confirm: %s: %s fee/%g priority, took %d blocks\n", - assignedTo, feeRate.ToString(), dPriority, nBlocksAgo); - } - -public: - CMinerPolicyEstimator(int nEntries) : nBestSeenHeight(0) - { - history.resize(nEntries); - } - - void seenBlock(const std::vector<CTxMemPoolEntry>& entries, int nBlockHeight, const CFeeRate minRelayFee) - { - if (nBlockHeight <= nBestSeenHeight) - { - // Ignore side chains and re-orgs; assuming they are random - // they don't affect the estimate. - // And if an attacker can re-org the chain at will, then - // you've got much bigger problems than "attacker can influence - // transaction fees." - return; - } - nBestSeenHeight = nBlockHeight; - - // Fill up the history buckets based on how long transactions took - // to confirm. - std::vector<std::vector<const CTxMemPoolEntry*> > entriesByConfirmations; - entriesByConfirmations.resize(history.size()); - BOOST_FOREACH(const CTxMemPoolEntry& entry, entries) - { - // How many blocks did it take for miners to include this transaction? - int delta = nBlockHeight - entry.GetHeight(); - if (delta <= 0) - { - // Re-org made us lose height, this should only happen if we happen - // to re-org on a difficulty transition point: very rare! - continue; - } - if ((delta-1) >= (int)history.size()) - delta = history.size(); // Last bucket is catch-all - entriesByConfirmations.at(delta-1).push_back(&entry); - } - for (size_t i = 0; i < entriesByConfirmations.size(); i++) - { - std::vector<const CTxMemPoolEntry*> &e = entriesByConfirmations.at(i); - // Insert at most 10 random entries per bucket, otherwise a single block - // can dominate an estimate: - if (e.size() > 10) { - std::random_shuffle(e.begin(), e.end()); - e.resize(10); - } - BOOST_FOREACH(const CTxMemPoolEntry* entry, e) - { - // Fees are stored and reported as BTC-per-kb: - CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); - double dPriority = entry->GetPriority(entry->GetHeight()); // Want priority when it went IN - seenTxConfirm(feeRate, minRelayFee, dPriority, i); - } - } - - // After new samples are added, we have to clear the sorted lists, - // so they'll be resorted the next time someone asks for an estimate - sortedFeeSamples.clear(); - sortedPrioritySamples.clear(); - - for (size_t i = 0; i < history.size(); i++) { - if (history[i].FeeSamples() + history[i].PrioritySamples() > 0) - LogPrint("estimatefee", "estimates: for confirming within %d blocks based on %d/%d samples, fee=%s, prio=%g\n", - i, - history[i].FeeSamples(), history[i].PrioritySamples(), - estimateFee(i+1).ToString(), estimatePriority(i+1)); - } - } - - /** - * Can return CFeeRate(0) if we don't have any data for that many blocks back. nBlocksToConfirm is 1 based. - */ - CFeeRate estimateFee(int nBlocksToConfirm) - { - nBlocksToConfirm--; - - if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size()) - return CFeeRate(0); - - if (sortedFeeSamples.size() == 0) - { - for (size_t i = 0; i < history.size(); i++) - history.at(i).GetFeeSamples(sortedFeeSamples); - std::sort(sortedFeeSamples.begin(), sortedFeeSamples.end(), - std::greater<CFeeRate>()); - } - if (sortedFeeSamples.size() < 11) - { - // Eleven is Gavin's Favorite Number - // ... but we also take a maximum of 10 samples per block so eleven means - // we're getting samples from at least two different blocks - return CFeeRate(0); - } - - int nBucketSize = history.at(nBlocksToConfirm).FeeSamples(); - - // Estimates should not increase as number of confirmations goes up, - // but the estimates are noisy because confirmations happen discretely - // in blocks. To smooth out the estimates, use all samples in the history - // and use the nth highest where n is (number of samples in previous bucket + - // half the samples in nBlocksToConfirm bucket): - size_t nPrevSize = 0; - for (int i = 0; i < nBlocksToConfirm; i++) - nPrevSize += history.at(i).FeeSamples(); - size_t index = min(nPrevSize + nBucketSize/2, sortedFeeSamples.size()-1); - return sortedFeeSamples[index]; - } - double estimatePriority(int nBlocksToConfirm) - { - nBlocksToConfirm--; - - if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size()) - return -1; - - if (sortedPrioritySamples.size() == 0) - { - for (size_t i = 0; i < history.size(); i++) - history.at(i).GetPrioritySamples(sortedPrioritySamples); - std::sort(sortedPrioritySamples.begin(), sortedPrioritySamples.end(), - std::greater<double>()); - } - if (sortedPrioritySamples.size() < 11) - return -1.0; - - int nBucketSize = history.at(nBlocksToConfirm).PrioritySamples(); - - // Estimates should not increase as number of confirmations needed goes up, - // but the estimates are noisy because confirmations happen discretely - // in blocks. To smooth out the estimates, use all samples in the history - // and use the nth highest where n is (number of samples in previous buckets + - // half the samples in nBlocksToConfirm bucket). - size_t nPrevSize = 0; - for (int i = 0; i < nBlocksToConfirm; i++) - nPrevSize += history.at(i).PrioritySamples(); - size_t index = min(nPrevSize + nBucketSize/2, sortedPrioritySamples.size()-1); - return sortedPrioritySamples[index]; - } - - void Write(CAutoFile& fileout) const - { - fileout << nBestSeenHeight; - fileout << (uint32_t)history.size(); - BOOST_FOREACH(const CBlockAverage& entry, history) - { - entry.Write(fileout); - } - } - - void Read(CAutoFile& filein, const CFeeRate& minRelayFee) - { - int nFileBestSeenHeight; - filein >> nFileBestSeenHeight; - uint32_t numEntries; - filein >> numEntries; - if (numEntries <= 0 || numEntries > 10000) - throw runtime_error("Corrupt estimates file. Must have between 1 and 10k entries."); - - std::vector<CBlockAverage> fileHistory; - - for (size_t i = 0; i < numEntries; i++) - { - CBlockAverage entry; - entry.Read(filein, minRelayFee); - fileHistory.push_back(entry); - } - - // Now that we've processed the entire fee estimate data file and not - // thrown any errors, we can copy it to our history - nBestSeenHeight = nFileBestSeenHeight; - history = fileHistory; - assert(history.size() > 0); - } -}; - - CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) : - nTransactionsUpdated(0), - minRelayFee(_minRelayFee) + nTransactionsUpdated(0) { // Sanity checks off by default for performance, because otherwise // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool fSanityCheck = false; - // 25 blocks is a compromise between using a lot of disk/memory and - // trying to give accurate estimates to people who might be willing - // to wait a day or two to save a fraction of a penny in fees. - // Confirmation times for very-low-fee transactions that take more - // than an hour or three to confirm are highly variable. - minerPolicyEstimator = new CMinerPolicyEstimator(25); + minerPolicyEstimator = new CBlockPolicyEstimator(_minRelayFee); } CTxMemPool::~CTxMemPool() @@ -420,20 +88,20 @@ void CTxMemPool::AddTransactionsUpdated(unsigned int n) } -bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry) +bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate) { // Add to memory pool without checking anything. // Used by main.cpp AcceptToMemoryPool(), which DOES do // all the appropriate checks. LOCK(cs); - { - mapTx[hash] = entry; - const CTransaction& tx = mapTx[hash].GetTx(); - for (unsigned int i = 0; i < tx.vin.size(); i++) - mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i); - nTransactionsUpdated++; - totalTxSize += entry.GetTxSize(); - } + mapTx[hash] = entry; + const CTransaction& tx = mapTx[hash].GetTx(); + for (unsigned int i = 0; i < tx.vin.size(); i++) + mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i); + nTransactionsUpdated++; + totalTxSize += entry.GetTxSize(); + minerPolicyEstimator->processTransaction(entry, fCurrentEstimate); + return true; } @@ -479,6 +147,7 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& rem totalTxSize -= mapTx[hash].GetTxSize(); mapTx.erase(hash); nTransactionsUpdated++; + minerPolicyEstimator->removeTx(hash); } } } @@ -529,7 +198,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction> * Called when a block is connected. Removes from mempool and updates the miner fee estimator. */ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts) + std::list<CTransaction>& conflicts, bool fCurrentEstimate) { LOCK(cs); std::vector<CTxMemPoolEntry> entries; @@ -539,7 +208,6 @@ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned i if (mapTx.count(hash)) entries.push_back(mapTx[hash]); } - minerPolicyEstimator->seenBlock(entries, nBlockHeight, minRelayFee); BOOST_FOREACH(const CTransaction& tx, vtx) { std::list<CTransaction> dummy; @@ -547,9 +215,10 @@ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned i removeConflicts(tx, conflicts); ClearPrioritisation(tx.GetHash()); } + // After the txs in the new block have been removed from the mempool, update policy estimates + minerPolicyEstimator->processBlock(nBlockHeight, entries, fCurrentEstimate); } - void CTxMemPool::clear() { LOCK(cs); @@ -666,7 +335,7 @@ CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const { try { LOCK(cs); - fileout << 99900; // version required to read: 0.9.99 or later + fileout << 109900; // version required to read: 0.10.99 or later fileout << CLIENT_VERSION; // version that wrote the file minerPolicyEstimator->Write(fileout); } @@ -687,7 +356,7 @@ CTxMemPool::ReadFeeEstimates(CAutoFile& filein) return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); LOCK(cs); - minerPolicyEstimator->Read(filein, minRelayFee); + minerPolicyEstimator->Read(filein); } catch (const std::exception&) { LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)"); @@ -724,6 +393,13 @@ void CTxMemPool::ClearPrioritisation(const uint256 hash) mapDeltas.erase(hash); } +bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const +{ + for (unsigned int i = 0; i < tx.vin.size(); i++) + if (exists(tx.vin[i].prevout.hash)) + return false; + return true; +} CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } diff --git a/src/txmempool.h b/src/txmempool.h index 0732af67e6..7271a5f603 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -43,10 +43,11 @@ private: int64_t nTime; //! Local time when entering the mempool double dPriority; //! Priority when entering the mempool unsigned int nHeight; //! Chain height when entering the mempool + bool hadNoDependencies; //! Not dependent on any other txs when it entered the mempool public: CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, - int64_t _nTime, double _dPriority, unsigned int _nHeight); + int64_t _nTime, double _dPriority, unsigned int _nHeight, bool poolHasNoInputsOf = false); CTxMemPoolEntry(); CTxMemPoolEntry(const CTxMemPoolEntry& other); @@ -56,9 +57,10 @@ public: size_t GetTxSize() const { return nTxSize; } int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return nHeight; } + bool WasClearAtEntry() const { return hadNoDependencies; } }; -class CMinerPolicyEstimator; +class CBlockPolicyEstimator; /** An inpoint - a combination of a transaction and an index n into its vin */ class CInPoint @@ -88,9 +90,8 @@ class CTxMemPool private: bool fSanityCheck; //! Normally false, true if -checkmempool or -regtest unsigned int nTransactionsUpdated; - CMinerPolicyEstimator* minerPolicyEstimator; + CBlockPolicyEstimator* minerPolicyEstimator; - CFeeRate minRelayFee; //! Passed to constructor to avoid dependency on main uint64_t totalTxSize; //! sum of all mempool tx' byte sizes public: @@ -111,17 +112,22 @@ public: void check(const CCoinsViewCache *pcoins) const; void setSanityCheck(bool _fSanityCheck) { fSanityCheck = _fSanityCheck; } - bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry); + bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); void remove(const CTransaction &tx, std::list<CTransaction>& removed, bool fRecursive = false); void removeCoinbaseSpends(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight); void removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed); void removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts); + std::list<CTransaction>& conflicts, bool fCurrentEstimate = true); void clear(); void queryHashes(std::vector<uint256>& vtxid); void pruneSpent(const uint256& hash, CCoins &coins); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); + /** + * Check that none of this transactions inputs are in the mempool, and thus + * the tx is not dependent on other mempool transactions to be included in a block. + */ + bool HasNoInputsOf(const CTransaction& tx) const; /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const CAmount& nFeeDelta); @@ -139,7 +145,7 @@ public: return totalTxSize; } - bool exists(uint256 hash) + bool exists(uint256 hash) const { LOCK(cs); return (mapTx.count(hash) != 0); diff --git a/src/util.cpp b/src/util.cpp index c9e8242d47..da5b259aee 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -726,18 +726,20 @@ void RenameThread(const char* name) void SetupEnvironment() { - std::locale loc("C"); // On most POSIX systems (e.g. Linux, but not BSD) the environment's locale // may be invalid, in which case the "C" locale is used as fallback. #if !defined(WIN32) && !defined(MAC_OSX) && !defined(__FreeBSD__) && !defined(__OpenBSD__) try { - loc = std::locale(""); // Raises a runtime error if current locale is invalid + std::locale(""); // Raises a runtime error if current locale is invalid } catch (const std::runtime_error&) { setenv("LC_ALL", "C", 1); } #endif - // The path locale is lazy initialized and to avoid deinitialization errors + // The path locale is lazy initialized and to avoid deinitialization errors // in multithreading environments, it is set explicitly by the main thread. + // A dummy locale is used to extract the internal default locale, used by + // boost::filesystem::path, which is then used to explicitly imbue the path. + std::locale loc = boost::filesystem::path::imbue(std::locale::classic()); boost::filesystem::path::imbue(loc); } diff --git a/src/util.h b/src/util.h index 483d9d7858..4cc0faf4d7 100644 --- a/src/util.h +++ b/src/util.h @@ -203,43 +203,6 @@ void SetThreadPriority(int nPriority); void RenameThread(const char* name); /** - * Standard wrapper for do-something-forever thread functions. - * "Forever" really means until the thread is interrupted. - * Use it like: - * new boost::thread(boost::bind(&LoopForever<void (*)()>, "dumpaddr", &DumpAddresses, 900000)); - * or maybe: - * boost::function<void()> f = boost::bind(&FunctionWithArg, argument); - * threadGroup.create_thread(boost::bind(&LoopForever<boost::function<void()> >, "nothing", f, milliseconds)); - */ -template <typename Callable> void LoopForever(const char* name, Callable func, int64_t msecs) -{ - std::string s = strprintf("bitcoin-%s", name); - RenameThread(s.c_str()); - LogPrintf("%s thread start\n", name); - try - { - while (1) - { - MilliSleep(msecs); - func(); - } - } - catch (const boost::thread_interrupted&) - { - LogPrintf("%s thread stop\n", name); - throw; - } - catch (const std::exception& e) { - PrintExceptionContinue(&e, name); - throw; - } - catch (...) { - PrintExceptionContinue(NULL, name); - throw; - } -} - -/** * .. and a wrapper that just calls func once */ template <typename Callable> void TraceThread(const char* name, Callable func) |
