diff options
Diffstat (limited to 'src')
394 files changed, 17171 insertions, 4323 deletions
diff --git a/src/.clang-format b/src/.clang-format index a69c57f3e0..791b3b8f9f 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -34,14 +34,6 @@ IndentWidth: 4 KeepEmptyLinesAtTheStartOfBlocks: false MaxEmptyLinesToKeep: 2 NamespaceIndentation: None -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: false -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements @@ -51,6 +43,5 @@ SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false -Standard: Cpp11 -TabWidth: 8 +Standard: c++17 UseTab: Never diff --git a/src/Makefile.am b/src/Makefile.am index 52c8b85357..6cfd5a9050 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -6,7 +6,7 @@ print-%: FORCE @echo '$*'='$($*)' -DIST_SUBDIRS = secp256k1 univalue +DIST_SUBDIRS = secp256k1 AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) @@ -15,18 +15,7 @@ AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = -if EMBEDDED_UNIVALUE -LIBUNIVALUE = univalue/libunivalue.la - -$(LIBUNIVALUE): $(wildcard univalue/lib/*) $(wildcard univalue/include/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) -else -LIBUNIVALUE = $(UNIVALUE_LIBS) -endif - -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) - -BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) LIBBITCOIN_SERVER=libbitcoin_server.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -80,6 +69,7 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_ZMQ) lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) +noinst_LTLIBRARIES = bin_PROGRAMS = noinst_PROGRAMS = @@ -117,13 +107,13 @@ endif BITCOIN_CORE_H = \ addrdb.h \ addrman.h \ + addrman_impl.h \ attributes.h \ banman.h \ base58.h \ bech32.h \ blockencodings.h \ blockfilter.h \ - bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -131,6 +121,7 @@ BITCOIN_CORE_H = \ checkqueue.h \ clientversion.h \ coins.h \ + common/bloom.h \ compat.h \ compat/assumptions.h \ compat/byteswap.h \ @@ -176,6 +167,7 @@ BITCOIN_CORE_H = \ memusage.h \ merkleblock.h \ miner.h \ + minisketchwrapper.h \ net.h \ net_permissions.h \ net_processing.h \ @@ -261,6 +253,7 @@ BITCOIN_CORE_H = \ util/sock.h \ util/spanparsing.h \ util/string.h \ + util/syscall_sandbox.h \ util/system.h \ util/thread.h \ util/threadnames.h \ @@ -342,6 +335,7 @@ libbitcoin_server_a_SOURCES = \ init.cpp \ mapport.cpp \ miner.cpp \ + minisketchwrapper.cpp \ net.cpp \ net_processing.cpp \ node/blockstorage.cpp \ @@ -498,9 +492,9 @@ crypto_libbitcoin_crypto_shani_a_SOURCES = crypto/sha256_shani.cpp libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ - amount.h \ arith_uint256.cpp \ arith_uint256.h \ + consensus/amount.h \ consensus/merkle.cpp \ consensus/merkle.h \ consensus/params.h \ @@ -537,9 +531,9 @@ libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ base58.cpp \ bech32.cpp \ - bloom.cpp \ chainparams.cpp \ coins.cpp \ + common/bloom.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -611,6 +605,7 @@ libbitcoin_util_a_SOURCES = \ util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ + util/syscall_sandbox.cpp \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) @@ -710,10 +705,22 @@ bitcoin_tx_LDADD += $(BOOST_LIBS) # bitcoin-wallet binary # bitcoin_wallet_SOURCES = bitcoin-wallet.cpp +bitcoin_wallet_SOURCES += init/bitcoin-wallet.cpp bitcoin_wallet_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoin_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoin_wallet_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoin_wallet_LDADD = $(LIBBITCOIN_WALLET_TOOL) $(bitcoin_bin_ldadd) +bitcoin_wallet_LDADD = \ + $(LIBBITCOIN_WALLET_TOOL) \ + $(LIBBITCOIN_WALLET) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBSECP256K1) \ + $(BOOST_LIBS) \ + $(BDB_LIBS) \ + $(SQLITE_LIBS) if TARGET_WINDOWS bitcoin_wallet_SOURCES += bitcoin-wallet-res.rc @@ -794,7 +801,6 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) clean-local: -$(MAKE) -C secp256k1 clean - -$(MAKE) -C univalue clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h -rm -rf test/__pycache__ @@ -805,20 +811,8 @@ clean-local: $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ check-symbols: $(bin_PROGRAMS) -if TARGET_DARWIN - @echo "Checking macOS dynamic libraries..." - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_WINDOWS - @echo "Checking Windows dynamic libraries..." + @echo "Running symbol and dynamic library checks..." $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_LINUX - @echo "Checking glibc back compat..." - $(AM_V_at) CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif check-security: $(bin_PROGRAMS) if HARDEN @@ -861,10 +855,10 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) CLEANFILES += $(libbitcoin_ipc_mpgen_output) endif -if EMBEDDED_LEVELDB +include Makefile.minisketch.include + include Makefile.crc32c.include include Makefile.leveldb.include -endif include Makefile.test_util.include include Makefile.test_fuzz.include @@ -884,3 +878,5 @@ endif if ENABLE_QT_TESTS include Makefile.qttest.include endif + +include Makefile.univalue.include diff --git a/src/Makefile.crc32c.include b/src/Makefile.crc32c.include index 113272e65e..3cbe71792c 100644 --- a/src/Makefile.crc32c.include +++ b/src/Makefile.crc32c.include @@ -14,7 +14,6 @@ CRC32C_CPPFLAGS_INT += -I$(srcdir)/crc32c/include CRC32C_CPPFLAGS_INT += -DHAVE_BUILTIN_PREFETCH=@HAVE_BUILTIN_PREFETCH@ CRC32C_CPPFLAGS_INT += -DHAVE_MM_PREFETCH=@HAVE_MM_PREFETCH@ CRC32C_CPPFLAGS_INT += -DHAVE_STRONG_GETAUXVAL=@HAVE_STRONG_GETAUXVAL@ -CRC32C_CPPFLAGS_INT += -DHAVE_WEAK_GETAUXVAL=@HAVE_WEAK_GETAUXVAL@ CRC32C_CPPFLAGS_INT += -DCRC32C_TESTS_BUILT_WITH_GLOG=0 if ENABLE_SSE42 diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index ce1f93f11f..3bec92482a 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -8,9 +8,10 @@ LIBMEMENV_INT = leveldb/libmemenv.a EXTRA_LIBRARIES += $(LIBLEVELDB_INT) EXTRA_LIBRARIES += $(LIBMEMENV_INT) -LIBLEVELDB += $(LIBLEVELDB_INT) $(LIBCRC32C) -LIBMEMENV += $(LIBMEMENV_INT) +LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C) +LIBMEMENV = $(LIBMEMENV_INT) +LEVELDB_CPPFLAGS = LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include new file mode 100644 index 0000000000..b337f48349 --- /dev/null +++ b/src/Makefile.minisketch.include @@ -0,0 +1,43 @@ +include minisketch/sources.mk + +LIBMINISKETCH_CPPFLAGS= +LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32 + +LIBMINISKETCH = minisketch/libminisketch.a +MINISKETCH_LIBS = $(LIBMINISKETCH) + +if ENABLE_CLMUL +LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a +LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL +MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL) +endif + +if HAVE_CLZ +LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ +endif + +EXTRA_LIBRARIES += $(MINISKETCH_LIBS) + +minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT) +minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS) +minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) + +minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT) +minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT) +minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) +minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) + +if ENABLE_TESTS +if !ENABLE_FUZZ +MINISKETCH_TEST = minisketch/test +TESTS += $(MINISKETCH_TEST) +noinst_PROGRAMS += $(MINISKETCH_TEST) + +minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) +minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) +minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +minisketch_test_LDADD = $(MINISKETCH_LIBS) +minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) + +endif +endif diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index f4b0b3adbe..35d5b0004a 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -168,10 +168,10 @@ BITCOIN_QT_H = \ qt/walletview.h \ qt/winshutdownmonitor.h -RES_FONTS = \ +QT_RES_FONTS = \ qt/res/fonts/RobotoMono-Bold.ttf -RES_ICONS = \ +QT_RES_ICONS = \ qt/res/icons/add.png \ qt/res/icons/address-book.png \ qt/res/icons/bitcoin.ico \ @@ -287,9 +287,9 @@ if ENABLE_WALLET BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP) endif # ENABLE_WALLET -RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) +QT_RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) -BITCOIN_RC = qt/res/bitcoin-qt-res.rc +BITCOIN_QT_RC = qt/res/bitcoin-qt-res.rc BITCOIN_QT_INCLUDES = -DQT_NO_KEYWORDS -DQT_USE_QSTRINGBUILDER @@ -299,7 +299,7 @@ qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) if TARGET_DARWIN qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM) endif @@ -321,7 +321,7 @@ bitcoin_qt_cxxflags = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) bitcoin_qt_sources = qt/main.cpp if TARGET_WINDOWS - bitcoin_qt_sources += $(BITCOIN_RC) + bitcoin_qt_sources += $(BITCOIN_QT_RC) endif bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) if ENABLE_WALLET @@ -338,15 +338,15 @@ bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX qt_bitcoin_qt_CPPFLAGS = $(bitcoin_qt_cppflags) qt_bitcoin_qt_CXXFLAGS = $(bitcoin_qt_cxxflags) -qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) init/bitcoind.cpp +qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) init/bitcoin-qt.cpp qt_bitcoin_qt_LDADD = $(bitcoin_qt_ldadd) qt_bitcoin_qt_LDFLAGS = $(bitcoin_qt_ldflags) qt_bitcoin_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) bitcoin_gui_CPPFLAGS = $(bitcoin_qt_cppflags) bitcoin_gui_CXXFLAGS = $(bitcoin_qt_cxxflags) -bitcoin_gui_SOURCES = $(bitcoin_qt_sources) init/bitcoind.cpp -bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) +bitcoin_gui_SOURCES = $(bitcoin_qt_sources) init/bitcoin-gui.cpp +bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) $(LIBBITCOIN_IPC) $(LIBBITCOIN_UTIL) $(LIBMULTIPROCESS_LIBS) bitcoin_gui_LDFLAGS = $(bitcoin_qt_ldflags) bitcoin_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) @@ -371,7 +371,7 @@ $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin_locale --format-version 1 $(@D)/temp_$(<F) > $@ @rm $(@D)/temp_$(<F) -$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) +$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) @test -f $(RCC) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin --format-version 1 $< > $@ @@ -391,7 +391,7 @@ QT_BASE_TLD = $(shell tar tf $(QT_BASE_PATH) --exclude='*/*') bitcoin_qt_apk: FORCE mkdir -p $(APK_LIB_DIR) - cp $(dir $(CC))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) + cp $(dir $(lastword $(CC)))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/jar/src --strip-components=5 tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/java/src --strip-components=5 tar xf $(QT_BASE_PATH) -C qt/android/res/ $(QT_BASE_TLD)src/android/java/res --strip-components=5 diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 8a5521eeb5..797e1f9a97 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -28,7 +28,7 @@ qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_ $(QT_INCLUDES) $(QT_TEST_INCLUDES) qt_test_test_bitcoin_qt_SOURCES = \ - init/bitcoind.cpp \ + init/bitcoin-qt.cpp \ qt/test/apptests.cpp \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index a85a359960..da4b85665f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -102,6 +102,7 @@ BITCOIN_TESTS =\ test/mempool_tests.cpp \ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ + test/minisketch_tests.cpp \ test/miner_tests.cpp \ test/multisig_tests.cpp \ test/net_peer_eviction_tests.cpp \ @@ -119,8 +120,9 @@ BITCOIN_TESTS =\ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_p2sh_tests.cpp \ - test/script_tests.cpp \ + test/script_parse_tests.cpp \ test/script_standard_tests.cpp \ + test/script_tests.cpp \ test/scriptnum_tests.cpp \ test/serfloat_tests.cpp \ test/serialize_tests.cpp \ @@ -138,6 +140,7 @@ BITCOIN_TESTS =\ test/transaction_tests.cpp \ test/txindex_tests.cpp \ test/txrequest_tests.cpp \ + test/txpackage_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/uint256_tests.cpp \ @@ -169,6 +172,10 @@ if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif +if USE_SQLITE +FUZZ_WALLET_SRC = \ + wallet/test/fuzz/notifications.cpp +endif # USE_SQLITE BITCOIN_TEST_SUITE += \ wallet/test/util.cpp \ @@ -177,7 +184,7 @@ BITCOIN_TEST_SUITE += \ wallet/test/wallet_test_fixture.h \ wallet/test/init_test_fixture.cpp \ wallet/test/init_test_fixture.h -endif +endif # ENABLE_WALLET test_test_bitcoin_SOURCES = $(BITCOIN_TEST_SUITE) $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(EVENT_CFLAGS) @@ -187,7 +194,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ - $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) + $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) @@ -198,14 +205,13 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) - if ENABLE_FUZZ_BINARY test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fuzz_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) $(RUNTIME_LDFLAGS) +test_fuzz_fuzz_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) $(RUNTIME_LDFLAGS) test_fuzz_fuzz_SOURCES = \ + $(FUZZ_WALLET_SRC) \ test/fuzz/addition_overflow.cpp \ test/fuzz/addrman.cpp \ test/fuzz/asmap.cpp \ @@ -232,7 +238,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ test/fuzz/crypto_poly1305.cpp \ test/fuzz/cuckoocache.cpp \ - test/fuzz/data_stream.cpp \ test/fuzz/decode_tx.cpp \ test/fuzz/descriptor_parse.cpp \ test/fuzz/deserialize.cpp \ @@ -350,8 +355,26 @@ if ENABLE_BENCH endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check -if EMBEDDED_UNIVALUE - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check + +if !ENABLE_FUZZ +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester univalue/test/no_nul +noinst_PROGRAMS += $(UNIVALUE_TESTS) +TESTS += $(UNIVALUE_TESTS) + +univalue_test_unitester_SOURCES = $(UNIVALUE_TEST_UNITESTER_INT) +univalue_test_unitester_LDADD = $(LIBUNIVALUE) +univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" +univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) +univalue_test_no_nul_LDADD = $(LIBUNIVALUE) +univalue_test_no_nul_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) +univalue_test_object_LDADD = $(LIBUNIVALUE) +univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) endif %.cpp.test: %.cpp diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 85e50ebf70..0a3b99e7d2 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -9,6 +9,7 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ + test/util/chainstate.h \ test/util/logging.h \ test/util/mining.h \ test/util/net.h \ diff --git a/src/Makefile.univalue.include b/src/Makefile.univalue.include new file mode 100644 index 0000000000..3644e36368 --- /dev/null +++ b/src/Makefile.univalue.include @@ -0,0 +1,6 @@ +include univalue/sources.mk + +LIBUNIVALUE = libunivalue.la +noinst_LTLIBRARIES += $(LIBUNIVALUE) +libunivalue_la_SOURCES = $(UNIVALUE_LIB_SOURCES_INT) $(UNIVALUE_DIST_HEADERS_INT) $(UNIVALUE_LIB_HEADERS_INT) $(UNIVALUE_TEST_FILES_INT) +libunivalue_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 1e73750ce3..bdb1fc6b2b 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -58,7 +58,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (fileout.IsNull()) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to open file %s", __func__, pathTmp.string()); + return error("%s: Failed to open file %s", __func__, fs::PathToString(pathTmp)); } // Serialize @@ -70,7 +70,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (!FileCommit(fileout.Get())) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to flush file %s", __func__, pathTmp.string()); + return error("%s: Failed to flush file %s", __func__, fs::PathToString(pathTmp)); } fileout.fclose(); @@ -122,8 +122,8 @@ void DeserializeFileDB(const fs::path& path, Data& data, int version) } // namespace CBanDB::CBanDB(fs::path ban_list_path) - : m_banlist_dat(ban_list_path.string() + ".dat"), - m_banlist_json(ban_list_path.string() + ".json") + : m_banlist_dat(ban_list_path + ".dat"), + m_banlist_json(ban_list_path + ".json") { } @@ -143,7 +143,7 @@ bool CBanDB::Write(const banmap_t& banSet) bool CBanDB::Read(banmap_t& banSet) { if (fs::exists(m_banlist_dat)) { - LogPrintf("banlist.dat ignored because it can only be read by " PACKAGE_NAME " version 22.x. Remove %s to silence this warning.\n", m_banlist_dat); + LogPrintf("banlist.dat ignored because it can only be read by " PACKAGE_NAME " version 22.x. Remove %s to silence this warning.\n", fs::quoted(fs::PathToString(m_banlist_dat))); } // If the JSON banlist does not exist, then recreate it if (!fs::exists(m_banlist_json)) { @@ -155,7 +155,7 @@ bool CBanDB::Read(banmap_t& banSet) if (!util::ReadSettings(m_banlist_json, settings, errors)) { for (const auto& err : errors) { - LogPrintf("Cannot load banlist %s: %s\n", m_banlist_json.string(), err); + LogPrintf("Cannot load banlist %s: %s\n", fs::PathToString(m_banlist_json), err); } return false; } @@ -163,28 +163,28 @@ bool CBanDB::Read(banmap_t& banSet) try { BanMapFromJson(settings[JSON_KEY], banSet); } catch (const std::runtime_error& e) { - LogPrintf("Cannot parse banlist %s: %s\n", m_banlist_json.string(), e.what()); + LogPrintf("Cannot parse banlist %s: %s\n", fs::PathToString(m_banlist_json), e.what()); return false; } return true; } -bool DumpPeerAddresses(const ArgsManager& args, const CAddrMan& addr) +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr) { const auto pathAddr = args.GetDataDirNet() / "peers.dat"; return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION); } -void ReadFromStream(CAddrMan& addr, CDataStream& ssPeers) +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) { DeserializeDB(ssPeers, addr, false); } -std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<CAddrMan>& addrman) +std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) { - auto check_addrman = std::clamp<int32_t>(args.GetArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique<CAddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); + addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); int64_t nStart = GetTimeMillis(); const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -193,13 +193,13 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), GetTimeMillis() - nStart); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<CAddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); - LogPrintf("Creating peers.dat because the file was not found (%s)\n", path_addr); + addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { addrman = nullptr; return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), - e.what(), PACKAGE_BUGREPORT, path_addr); + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); } return std::nullopt; } @@ -215,7 +215,7 @@ std::vector<CAddress> ReadAnchors(const fs::path& anchors_db_path) std::vector<CAddress> anchors; try { DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); - LogPrintf("Loaded %i addresses from %s\n", anchors.size(), anchors_db_path.filename()); + LogPrintf("Loaded %i addresses from %s\n", anchors.size(), fs::quoted(fs::PathToString(anchors_db_path.filename()))); } catch (const std::exception&) { anchors.clear(); } diff --git a/src/addrdb.h b/src/addrdb.h index 33cc1f9204..19be4b5bb4 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -14,14 +14,14 @@ #include <vector> class ArgsManager; -class CAddrMan; +class AddrMan; class CAddress; class CDataStream; struct bilingual_str; -bool DumpPeerAddresses(const ArgsManager& args, const CAddrMan& addr); +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Only used by tests. */ -void ReadFromStream(CAddrMan& addr, CDataStream& ssPeers); +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers); /** Access to the banlist database (banlist.json) */ class CBanDB @@ -48,7 +48,7 @@ public: }; /** Returns an error string on failure */ -std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<CAddrMan>& addrman); +std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/addrman.cpp b/src/addrman.cpp index 772c34ae77..8a0433c40d 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -4,24 +4,29 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> +#include <addrman_impl.h> -#include <clientversion.h> #include <hash.h> #include <logging.h> +#include <logging/timer.h> #include <netaddress.h> +#include <protocol.h> +#include <random.h> #include <serialize.h> #include <streams.h> +#include <timedata.h> +#include <tinyformat.h> +#include <uint256.h> +#include <util/check.h> #include <cmath> #include <optional> -#include <unordered_map> -#include <unordered_set> /** Over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread */ static constexpr uint32_t ADDRMAN_TRIED_BUCKETS_PER_GROUP{8}; /** Over how many buckets entries with new addresses originating from a single group are spread */ static constexpr uint32_t ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP{64}; -/** Maximum number of times an address can be added to the new table */ +/** Maximum number of times an address can occur in the new table */ static constexpr int32_t ADDRMAN_NEW_BUCKETS_PER_ADDRESS{8}; /** How old addresses can maximally be */ static constexpr int64_t ADDRMAN_HORIZON_DAYS{30}; @@ -38,34 +43,28 @@ static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; /** The maximum time we'll spend trying to resolve a tried table collision, in seconds */ static constexpr int64_t ADDRMAN_TEST_WINDOW{40*60}; // 40 minutes -int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const +int AddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool>& asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); - int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i\n", ToStringIP(), mapped_as, tried_bucket); - return tried_bucket; + return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool> &asmap) const +int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool>& asmap) const { std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); - int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i\n", ToStringIP(), mapped_as, new_bucket); - return new_bucket; + return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } -int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const +int AddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? uint8_t{'N'} : uint8_t{'K'}) << nBucket << GetKey()).GetCheapHash(); return hash1 % ADDRMAN_BUCKET_SIZE; } -bool CAddrInfo::IsTerrible(int64_t nNow) const +bool AddrInfo::IsTerrible(int64_t nNow) const { if (nLastTry && nLastTry >= nNow - 60) // never remove things tried in the last minute return false; @@ -85,7 +84,7 @@ bool CAddrInfo::IsTerrible(int64_t nNow) const return false; } -double CAddrInfo::GetChance(int64_t nNow) const +double AddrInfo::GetChance(int64_t nNow) const { double fChance = 1.0; int64_t nSinceLastTry = std::max<int64_t>(nNow - nLastTry, 0); @@ -100,7 +99,7 @@ double CAddrInfo::GetChance(int64_t nNow) const return fChance; } -CAddrMan::CAddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio) +AddrManImpl::AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio) : insecure_rand{deterministic} , nKey{deterministic ? uint256{1} : insecure_rand.rand256()} , m_consistency_check_ratio{consistency_check_ratio} @@ -118,8 +117,13 @@ CAddrMan::CAddrMan(std::vector<bool> asmap, bool deterministic, int32_t consiste } } +AddrManImpl::~AddrManImpl() +{ + nKey.SetNull(); +} + template <typename Stream> -void CAddrMan::Serialize(Stream& s_) const +void AddrManImpl::Serialize(Stream& s_) const { LOCK(cs); @@ -169,7 +173,7 @@ void CAddrMan::Serialize(Stream& s_) const // Increment `lowest_compatible` iff a newly introduced format is incompatible with // the previous one. - static constexpr uint8_t lowest_compatible = Format::V3_BIP155; + static constexpr uint8_t lowest_compatible = Format::V4_MULTIPORT; s << static_cast<uint8_t>(INCOMPATIBILITY_BASE + lowest_compatible); s << nKey; @@ -182,7 +186,7 @@ void CAddrMan::Serialize(Stream& s_) const int nIds = 0; for (const auto& entry : mapInfo) { mapUnkIds[entry.first] = nIds; - const CAddrInfo &info = entry.second; + const AddrInfo& info = entry.second; if (info.nRefCount) { assert(nIds != nNew); // this means nNew was wrong, oh ow s << info; @@ -191,7 +195,7 @@ void CAddrMan::Serialize(Stream& s_) const } nIds = 0; for (const auto& entry : mapInfo) { - const CAddrInfo &info = entry.second; + const AddrInfo& info = entry.second; if (info.fInTried) { assert(nIds != nTried); // this means nTried was wrong, oh ow s << info; @@ -222,7 +226,7 @@ void CAddrMan::Serialize(Stream& s_) const } template <typename Stream> -void CAddrMan::Unserialize(Stream& s_) +void AddrManImpl::Unserialize(Stream& s_) { LOCK(cs); @@ -261,21 +265,21 @@ void CAddrMan::Unserialize(Stream& s_) if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nNew < 0) { throw std::ios_base::failure( - strprintf("Corrupt CAddrMan serialization: nNew=%d, should be in [0, %d]", + strprintf("Corrupt AddrMan serialization: nNew=%d, should be in [0, %d]", nNew, ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); } if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nTried < 0) { throw std::ios_base::failure( - strprintf("Corrupt CAddrMan serialization: nTried=%d, should be in [0, %d]", + strprintf("Corrupt AddrMan serialization: nTried=%d, should be in [0, %d]", nTried, ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); } // Deserialize entries from the new table. for (int n = 0; n < nNew; n++) { - CAddrInfo &info = mapInfo[n]; + AddrInfo& info = mapInfo[n]; s >> info; mapAddr[info] = n; info.nRandomPos = vRandom.size(); @@ -286,7 +290,7 @@ void CAddrMan::Unserialize(Stream& s_) // Deserialize entries from the tried table. int nLost = 0; for (int n = 0; n < nTried; n++) { - CAddrInfo info; + AddrInfo info; s >> info; int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); @@ -343,7 +347,7 @@ void CAddrMan::Unserialize(Stream& s_) for (auto bucket_entry : bucket_entries) { int bucket{bucket_entry.first}; const int entry_index{bucket_entry.second}; - CAddrInfo& info = mapInfo[entry_index]; + AddrInfo& info = mapInfo[entry_index]; // Don't store the entry in the new bucket if it's not a valid address for our addrman if (!info.IsValid()) continue; @@ -385,19 +389,15 @@ void CAddrMan::Unserialize(Stream& s_) LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions or invalid addresses\n", nLostUnk, nLost); } - Check(); + const int check_code{CheckAddrman()}; + if (check_code != 0) { + throw std::ios_base::failure(strprintf( + "Corrupt data. Consistency check failed with code %s", + check_code)); + } } -// explicit instantiation -template void CAddrMan::Serialize(CHashWriter& s) const; -template void CAddrMan::Serialize(CAutoFile& s) const; -template void CAddrMan::Serialize(CDataStream& s) const; -template void CAddrMan::Unserialize(CAutoFile& s); -template void CAddrMan::Unserialize(CHashVerifier<CAutoFile>& s); -template void CAddrMan::Unserialize(CDataStream& s); -template void CAddrMan::Unserialize(CHashVerifier<CDataStream>& s); - -CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) +AddrInfo* AddrManImpl::Find(const CService& addr, int* pnId) { AssertLockHeld(cs); @@ -412,12 +412,12 @@ CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) return nullptr; } -CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) +AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) { AssertLockHeld(cs); int nId = nIdCount++; - mapInfo[nId] = CAddrInfo(addr, addrSource); + mapInfo[nId] = AddrInfo(addr, addrSource); mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); vRandom.push_back(nId); @@ -426,7 +426,7 @@ CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, in return &mapInfo[nId]; } -void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const +void AddrManImpl::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const { AssertLockHeld(cs); @@ -450,12 +450,12 @@ void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const vRandom[nRndPos2] = nId1; } -void CAddrMan::Delete(int nId) +void AddrManImpl::Delete(int nId) { AssertLockHeld(cs); assert(mapInfo.count(nId) != 0); - CAddrInfo& info = mapInfo[nId]; + AddrInfo& info = mapInfo[nId]; assert(!info.fInTried); assert(info.nRefCount == 0); @@ -466,33 +466,37 @@ void CAddrMan::Delete(int nId) nNew--; } -void CAddrMan::ClearNew(int nUBucket, int nUBucketPos) +void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) { AssertLockHeld(cs); // if there is an entry in the specified bucket, delete it. if (vvNew[nUBucket][nUBucketPos] != -1) { int nIdDelete = vvNew[nUBucket][nUBucketPos]; - CAddrInfo& infoDelete = mapInfo[nIdDelete]; + AddrInfo& infoDelete = mapInfo[nIdDelete]; assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; vvNew[nUBucket][nUBucketPos] = -1; + LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToString(), nUBucket, nUBucketPos); if (infoDelete.nRefCount == 0) { Delete(nIdDelete); } } } -void CAddrMan::MakeTried(CAddrInfo& info, int nId) +void AddrManImpl::MakeTried(AddrInfo& info, int nId) { AssertLockHeld(cs); // remove the entry from all new buckets - for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - int pos = info.GetBucketPosition(nKey, true, bucket); + const int start_bucket{info.GetNewBucket(nKey, m_asmap)}; + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; ++n) { + const int bucket{(start_bucket + n) % ADDRMAN_NEW_BUCKET_COUNT}; + const int pos{info.GetBucketPosition(nKey, true, bucket)}; if (vvNew[bucket][pos] == nId) { vvNew[bucket][pos] = -1; info.nRefCount--; + if (info.nRefCount == 0) break; } } nNew--; @@ -508,7 +512,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) // find an item to evict int nIdEvict = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nIdEvict) == 1); - CAddrInfo& infoOld = mapInfo[nIdEvict]; + AddrInfo& infoOld = mapInfo[nIdEvict]; // Remove the to-be-evicted item from the tried set. infoOld.fInTried = false; @@ -525,6 +529,8 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) infoOld.nRefCount = 1; vvNew[nUBucket][nUBucketPos] = nIdEvict; nNew++; + LogPrint(BCLog::ADDRMAN, "Moved %s from tried[%i][%i] to new[%i][%i] to make space\n", + infoOld.ToString(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); } assert(vvTried[nKBucket][nKBucketPos] == -1); @@ -533,84 +539,15 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) info.fInTried = true; } -void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime) -{ - AssertLockHeld(cs); - - int nId; - - nLastGood = nTime; - - CAddrInfo* pinfo = Find(addr, &nId); - - // if not found, bail out - if (!pinfo) - return; - - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; - - // update info - info.nLastSuccess = nTime; - info.nLastTry = nTime; - info.nAttempts = 0; - // nTime is not updated here, to avoid leaking information about - // currently-connected peers. - - // if it is already in the tried set, don't do anything else - if (info.fInTried) - return; - - // find a bucket it is in now - int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); - int nUBucket = -1; - for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT; - int nBpos = info.GetBucketPosition(nKey, true, nB); - if (vvNew[nB][nBpos] == nId) { - nUBucket = nB; - break; - } - } - - // if no bucket is found, something bad happened; - // TODO: maybe re-add the node, but for now, just bail out - if (nUBucket == -1) - return; - - // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey, m_asmap); - int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); - - // Will moving this address into tried evict another entry? - if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { - // Output the entry we'd be colliding with, for debugging purposes - auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); - LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); - if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { - m_tried_collisions.insert(nId); - } - } else { - LogPrint(BCLog::ADDRMAN, "Moving %s to tried\n", addr.ToString()); - - // move nId to the tried tables - MakeTried(info, nId); - } -} - -bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) { AssertLockHeld(cs); if (!addr.IsRoutable()) return false; - bool fNew = false; int nId; - CAddrInfo* pinfo = Find(addr, &nId); + AddrInfo* pinfo = Find(addr, &nId); // Do not set a penalty for a source's self-announcement if (addr == source) { @@ -649,15 +586,14 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP pinfo = Create(addr, source, &nId); pinfo->nTime = std::max((int64_t)0, (int64_t)pinfo->nTime - nTimePenalty); nNew++; - fNew = true; } int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); + bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (vvNew[nUBucket][nUBucketPos] != nId) { - bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (!fInsert) { - CAddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; + AddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; if (infoExisting.IsTerrible() || (infoExisting.nRefCount > 1 && pinfo->nRefCount == 0)) { // Overwrite the existing new table entry. fInsert = true; @@ -667,31 +603,96 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP ClearNew(nUBucket, nUBucketPos); pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; + LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", + addr.ToString(), addr.GetMappedAS(m_asmap), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); } } } - return fNew; + return fInsert; } -void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nTime) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + int nId; + + nLastGood = nTime; + + AddrInfo* pinfo = Find(addr, &nId); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; - // check whether we are talking about the exact same CService (including same port) - if (info != addr) + // update info + info.nLastSuccess = nTime; + info.nLastTry = nTime; + info.nAttempts = 0; + // nTime is not updated here, to avoid leaking information about + // currently-connected peers. + + // if it is already in the tried set, don't do anything else + if (info.fInTried) return; + // if it is not in new, something bad happened + if (!Assume(info.nRefCount > 0)) { + return; + } + + // which tried bucket to move the entry to + int tried_bucket = info.GetTriedBucket(nKey, m_asmap); + int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); + + // Will moving this address into tried evict another entry? + if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { + if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { + m_tried_collisions.insert(nId); + } + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint(BCLog::ADDRMAN, "Collision with %s while attempting to move %s to tried table. Collisions=%d\n", + colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", + addr.ToString(), + m_tried_collisions.size()); + } else { + // move nId to the tried tables + MakeTried(info, nId); + LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", + addr.ToString(), addr.GetMappedAS(m_asmap), tried_bucket, tried_bucket_pos); + } +} + +bool AddrManImpl::Add_(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + int added{0}; + for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) { + added += AddSingle(*it, source, nTimePenalty) ? 1 : 0; + } + if (added > 0) { + LogPrint(BCLog::ADDRMAN, "Added %i addresses (of %i) from %s: %i tried, %i new\n", added, vAddr.size(), source.ToString(), nTried, nNew); + } + return added > 0; +} + +void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +{ + AssertLockHeld(cs); + + AddrInfo* pinfo = Find(addr); + + // if not found, bail out + if (!pinfo) + return; + + AddrInfo& info = *pinfo; + // update info info.nLastTry = nTime; if (fCountFailure && info.nLastCountAttempt < nLastGood) { @@ -700,15 +701,13 @@ void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) } } -CAddrInfo CAddrMan::Select_(bool newOnly) const +std::pair<CAddress, int64_t> AddrManImpl::Select_(bool newOnly) const { AssertLockHeld(cs); - if (vRandom.empty()) - return CAddrInfo(); + if (vRandom.empty()) return {}; - if (newOnly && nNew == 0) - return CAddrInfo(); + if (newOnly && nNew == 0) return {}; // Use a 50% chance for choosing between tried and new table entries. if (!newOnly && @@ -716,134 +715,62 @@ CAddrInfo CAddrMan::Select_(bool newOnly) const // use a tried node double fChanceFactor = 1.0; while (1) { + // Pick a tried bucket, and an initial position in that bucket. int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvTried[nKBucket][nKBucketPos]; + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE]; const auto it_found{mapInfo.find(nId)}; assert(it_found != mapInfo.end()); - const CAddrInfo& info{it_found->second}; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; + const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToString()); + return {info, info.nLastTry}; + } + // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; } } else { // use a new node double fChanceFactor = 1.0; while (1) { + // Pick a new bucket, and an initial position in that bucket. int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvNew[nUBucket][nUBucketPos]; + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE]; const auto it_found{mapInfo.find(nId)}; assert(it_found != mapInfo.end()); - const CAddrInfo& info{it_found->second}; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; - fChanceFactor *= 1.2; - } - } -} - -int CAddrMan::Check_() const -{ - AssertLockHeld(cs); - - // Run consistency checks 1 in m_consistency_check_ratio times if enabled - if (m_consistency_check_ratio == 0) return 0; - if (insecure_rand.randrange(m_consistency_check_ratio) >= 1) return 0; - - LogPrint(BCLog::ADDRMAN, "Addrman checks started: new %i, tried %i, total %u\n", nNew, nTried, vRandom.size()); - - std::unordered_set<int> setTried; - std::unordered_map<int, int> mapNew; - - if (vRandom.size() != (size_t)(nTried + nNew)) - return -7; - - for (const auto& entry : mapInfo) { - int n = entry.first; - const CAddrInfo& info = entry.second; - if (info.fInTried) { - if (!info.nLastSuccess) - return -1; - if (info.nRefCount) - return -2; - setTried.insert(n); - } else { - if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) - return -3; - if (!info.nRefCount) - return -4; - mapNew[n] = info.nRefCount; - } - const auto it{mapAddr.find(info)}; - if (it == mapAddr.end() || it->second != n) { - return -5; - } - if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) - return -14; - if (info.nLastTry < 0) - return -6; - if (info.nLastSuccess < 0) - return -8; - } - - if (setTried.size() != (size_t)nTried) - return -9; - if (mapNew.size() != (size_t)nNew) - return -10; - - for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvTried[n][i] != -1) { - if (!setTried.count(vvTried[n][i])) - return -11; - const auto it{mapInfo.find(vvTried[n][i])}; - if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { - return -17; - } - if (it->second.GetBucketPosition(nKey, false, n) != i) { - return -18; - } - setTried.erase(vvTried[n][i]); - } - } - } - - for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[n][i] != -1) { - if (!mapNew.count(vvNew[n][i])) - return -12; - const auto it{mapInfo.find(vvNew[n][i])}; - if (it == mapInfo.end() || it->second.GetBucketPosition(nKey, true, n) != i) { - return -19; - } - if (--mapNew[vvNew[n][i]] == 0) - mapNew.erase(vvNew[n][i]); + const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToString()); + return {info, info.nLastTry}; } + // Otherwise start over with a (likely) different bucket, and increased chance factor. + fChanceFactor *= 1.2; } } - - if (setTried.size()) - return -13; - if (mapNew.size()) - return -15; - if (nKey.IsNull()) - return -16; - - LogPrint(BCLog::ADDRMAN, "Addrman checks completed successfully\n"); - return 0; } -void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size_t max_pct, std::optional<Network> network) const +std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const { AssertLockHeld(cs); @@ -857,8 +784,9 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size // gather a list of random nodes, skipping those of low quality const int64_t now{GetAdjustedTime()}; + std::vector<CAddress> addresses; for (unsigned int n = 0; n < vRandom.size(); n++) { - if (vAddr.size() >= nNodes) + if (addresses.size() >= nNodes) break; int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; @@ -866,7 +794,7 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size const auto it{mapInfo.find(vRandom[n])}; assert(it != mapInfo.end()); - const CAddrInfo& ai{it->second}; + const AddrInfo& ai{it->second}; // Filter by network (optional) if (network != std::nullopt && ai.GetNetClass() != network) continue; @@ -874,25 +802,23 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size // Filter for quality if (ai.IsTerrible(now)) continue; - vAddr.push_back(ai); + addresses.push_back(ai); } + LogPrint(BCLog::ADDRMAN, "GetAddr returned %d random addresses\n", addresses.size()); + return addresses; } -void CAddrMan::Connected_(const CService& addr, int64_t nTime) +void AddrManImpl::Connected_(const CService& addr, int64_t nTime) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; + AddrInfo& info = *pinfo; // update info int64_t nUpdateInterval = 20 * 60; @@ -900,27 +826,23 @@ void CAddrMan::Connected_(const CService& addr, int64_t nTime) info.nTime = nTime; } -void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) +void AddrManImpl::SetServices_(const CService& addr, ServiceFlags nServices) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; + AddrInfo& info = *pinfo; // update info info.nServices = nServices; } -void CAddrMan::ResolveCollisions_() +void AddrManImpl::ResolveCollisions_() { AssertLockHeld(cs); @@ -933,7 +855,7 @@ void CAddrMan::ResolveCollisions_() if (mapInfo.count(id_new) != 1) { erase_collision = true; } else { - CAddrInfo& info_new = mapInfo[id_new]; + AddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); @@ -944,7 +866,7 @@ void CAddrMan::ResolveCollisions_() // Get the to-be-evicted address that is being tested int id_old = vvTried[tried_bucket][tried_bucket_pos]; - CAddrInfo& info_old = mapInfo[id_old]; + AddrInfo& info_old = mapInfo[id_old]; // Has successfully connected in last X hours if (GetAdjustedTime() - info_old.nLastSuccess < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { @@ -981,11 +903,11 @@ void CAddrMan::ResolveCollisions_() } } -CAddrInfo CAddrMan::SelectTriedCollision_() +std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_() { AssertLockHeld(cs); - if (m_tried_collisions.size() == 0) return CAddrInfo(); + if (m_tried_collisions.size() == 0) return {}; std::set<int>::iterator it = m_tried_collisions.begin(); @@ -996,16 +918,286 @@ CAddrInfo CAddrMan::SelectTriedCollision_() // If id_new not found in mapInfo remove it from m_tried_collisions if (mapInfo.count(id_new) != 1) { m_tried_collisions.erase(it); - return CAddrInfo(); + return {}; } - const CAddrInfo& newInfo = mapInfo[id_new]; + const AddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); - int id_old = vvTried[tried_bucket][tried_bucket_pos]; + const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; + return {info_old, info_old.nLastTry}; +} + +void AddrManImpl::Check() const +{ + AssertLockHeld(cs); + + // Run consistency checks 1 in m_consistency_check_ratio times if enabled + if (m_consistency_check_ratio == 0) return; + if (insecure_rand.randrange(m_consistency_check_ratio) >= 1) return; + + const int err{CheckAddrman()}; + if (err) { + LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); + assert(false); + } +} + +int AddrManImpl::CheckAddrman() const +{ + AssertLockHeld(cs); + + LOG_TIME_MILLIS_WITH_CATEGORY_MSG_ONCE( + strprintf("new %i, tried %i, total %u", nNew, nTried, vRandom.size()), BCLog::ADDRMAN); + + std::unordered_set<int> setTried; + std::unordered_map<int, int> mapNew; + + if (vRandom.size() != (size_t)(nTried + nNew)) + return -7; + + for (const auto& entry : mapInfo) { + int n = entry.first; + const AddrInfo& info = entry.second; + if (info.fInTried) { + if (!info.nLastSuccess) + return -1; + if (info.nRefCount) + return -2; + setTried.insert(n); + } else { + if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) + return -3; + if (!info.nRefCount) + return -4; + mapNew[n] = info.nRefCount; + } + const auto it{mapAddr.find(info)}; + if (it == mapAddr.end() || it->second != n) { + return -5; + } + if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) + return -14; + if (info.nLastTry < 0) + return -6; + if (info.nLastSuccess < 0) + return -8; + } + + if (setTried.size() != (size_t)nTried) + return -9; + if (mapNew.size() != (size_t)nNew) + return -10; + + for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvTried[n][i] != -1) { + if (!setTried.count(vvTried[n][i])) + return -11; + const auto it{mapInfo.find(vvTried[n][i])}; + if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { + return -17; + } + if (it->second.GetBucketPosition(nKey, false, n) != i) { + return -18; + } + setTried.erase(vvTried[n][i]); + } + } + } + + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[n][i] != -1) { + if (!mapNew.count(vvNew[n][i])) + return -12; + const auto it{mapInfo.find(vvNew[n][i])}; + if (it == mapInfo.end() || it->second.GetBucketPosition(nKey, true, n) != i) { + return -19; + } + if (--mapNew[vvNew[n][i]] == 0) + mapNew.erase(vvNew[n][i]); + } + } + } + + if (setTried.size()) + return -13; + if (mapNew.size()) + return -15; + if (nKey.IsNull()) + return -16; + + return 0; +} + +size_t AddrManImpl::size() const +{ + LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead + return vRandom.size(); +} + +bool AddrManImpl::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + LOCK(cs); + Check(); + auto ret = Add_(vAddr, source, nTimePenalty); + Check(); + return ret; +} + +void AddrManImpl::Good(const CService& addr, int64_t nTime) +{ + LOCK(cs); + Check(); + Good_(addr, /* test_before_evict */ true, nTime); + Check(); +} + +void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +{ + LOCK(cs); + Check(); + Attempt_(addr, fCountFailure, nTime); + Check(); +} + +void AddrManImpl::ResolveCollisions() +{ + LOCK(cs); + Check(); + ResolveCollisions_(); + Check(); +} + +std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision() +{ + LOCK(cs); + Check(); + const auto ret = SelectTriedCollision_(); + Check(); + return ret; +} + +std::pair<CAddress, int64_t> AddrManImpl::Select(bool newOnly) const +{ + LOCK(cs); + Check(); + const auto addrRet = Select_(newOnly); + Check(); + return addrRet; +} + +std::vector<CAddress> AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +{ + LOCK(cs); + Check(); + const auto addresses = GetAddr_(max_addresses, max_pct, network); + Check(); + return addresses; +} + +void AddrManImpl::Connected(const CService& addr, int64_t nTime) +{ + LOCK(cs); + Check(); + Connected_(addr, nTime); + Check(); +} + +void AddrManImpl::SetServices(const CService& addr, ServiceFlags nServices) +{ + LOCK(cs); + Check(); + SetServices_(addr, nServices); + Check(); +} + +const std::vector<bool>& AddrManImpl::GetAsmap() const +{ + return m_asmap; +} + +AddrMan::AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio) + : m_impl(std::make_unique<AddrManImpl>(std::move(asmap), deterministic, consistency_check_ratio)) {} + +AddrMan::~AddrMan() = default; + +template <typename Stream> +void AddrMan::Serialize(Stream& s_) const +{ + m_impl->Serialize<Stream>(s_); +} - return mapInfo[id_old]; +template <typename Stream> +void AddrMan::Unserialize(Stream& s_) +{ + m_impl->Unserialize<Stream>(s_); +} + +// explicit instantiation +template void AddrMan::Serialize(CHashWriter& s) const; +template void AddrMan::Serialize(CAutoFile& s) const; +template void AddrMan::Serialize(CDataStream& s) const; +template void AddrMan::Unserialize(CAutoFile& s); +template void AddrMan::Unserialize(CHashVerifier<CAutoFile>& s); +template void AddrMan::Unserialize(CDataStream& s); +template void AddrMan::Unserialize(CHashVerifier<CDataStream>& s); + +size_t AddrMan::size() const +{ + return m_impl->size(); +} + +bool AddrMan::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + return m_impl->Add(vAddr, source, nTimePenalty); +} + +void AddrMan::Good(const CService& addr, int64_t nTime) +{ + m_impl->Good(addr, nTime); +} + +void AddrMan::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +{ + m_impl->Attempt(addr, fCountFailure, nTime); +} + +void AddrMan::ResolveCollisions() +{ + m_impl->ResolveCollisions(); +} + +std::pair<CAddress, int64_t> AddrMan::SelectTriedCollision() +{ + return m_impl->SelectTriedCollision(); +} + +std::pair<CAddress, int64_t> AddrMan::Select(bool newOnly) const +{ + return m_impl->Select(newOnly); +} + +std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +{ + return m_impl->GetAddr(max_addresses, max_pct, network); +} + +void AddrMan::Connected(const CService& addr, int64_t nTime) +{ + m_impl->Connected(addr, nTime); +} + +void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) +{ + m_impl->SetServices(addr, nServices); +} + +const std::vector<bool>& AddrMan::GetAsmap() const +{ + return m_impl->GetAsmap(); } diff --git a/src/addrman.h b/src/addrman.h index 0885231ebc..455d84ca71 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -6,94 +6,22 @@ #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include <fs.h> -#include <logging.h> #include <netaddress.h> #include <protocol.h> -#include <sync.h> +#include <streams.h> #include <timedata.h> #include <cstdint> +#include <memory> #include <optional> -#include <set> -#include <unordered_map> +#include <utility> #include <vector> +class AddrManImpl; + /** Default for -checkaddrman */ static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0}; -/** - * Extended statistics about a CAddress - */ -class CAddrInfo : public CAddress -{ -public: - //! last try whatsoever by us (memory only) - int64_t nLastTry{0}; - - //! last counted attempt (memory only) - int64_t nLastCountAttempt{0}; - -private: - //! where knowledge about this address first came from - CNetAddr source; - - //! last successful connection by us - int64_t nLastSuccess{0}; - - //! connection attempts since last successful attempt - int nAttempts{0}; - - //! reference count in new sets (memory only) - int nRefCount{0}; - - //! in tried set? (memory only) - bool fInTried{false}; - - //! position in vRandom - mutable int nRandomPos{-1}; - - friend class CAddrMan; - friend class CAddrManDeterministic; - -public: - - SERIALIZE_METHODS(CAddrInfo, obj) - { - READWRITEAS(CAddress, obj); - READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); - } - - CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) - { - } - - CAddrInfo() : CAddress(), source() - { - } - - //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; - - //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; - - //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const - { - return GetNewBucket(nKey, source, asmap); - } - - //! Calculate in which position of a bucket to store this entry. - int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; - - //! Determine whether the statistics about this entry are bad enough so that it can just be deleted - bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; - - //! Calculate the relative chance this entry should be given when selecting nodes to connect to - double GetChance(int64_t nNow = GetAdjustedTime()) const; -}; - /** Stochastic address manager * * Design goals: @@ -123,298 +51,73 @@ public: * * Several indexes are kept for high performance. Setting m_consistency_check_ratio with the -checkaddrman * configuration option will introduce (expensive) consistency checks for the entire data structure. */ +class AddrMan +{ +protected: + const std::unique_ptr<AddrManImpl> m_impl; -/** Total number of buckets for tried addresses */ -static constexpr int32_t ADDRMAN_TRIED_BUCKET_COUNT_LOG2{8}; -static constexpr int ADDRMAN_TRIED_BUCKET_COUNT{1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2}; - -/** Total number of buckets for new addresses */ -static constexpr int32_t ADDRMAN_NEW_BUCKET_COUNT_LOG2{10}; -static constexpr int ADDRMAN_NEW_BUCKET_COUNT{1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2}; +public: + explicit AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio); -/** Maximum allowed number of entries in buckets for new and tried addresses */ -static constexpr int32_t ADDRMAN_BUCKET_SIZE_LOG2{6}; -static constexpr int ADDRMAN_BUCKET_SIZE{1 << ADDRMAN_BUCKET_SIZE_LOG2}; + ~AddrMan(); -/** - * Stochastical (IP) address manager - */ -class CAddrMan -{ -public: template <typename Stream> - void Serialize(Stream& s_) const EXCLUSIVE_LOCKS_REQUIRED(!cs); + void Serialize(Stream& s_) const; template <typename Stream> - void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); - - explicit CAddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio); - - ~CAddrMan() - { - nKey.SetNull(); - } + void Unserialize(Stream& s_); //! Return the number of (unique) addresses in all tables. - size_t size() const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead - return vRandom.size(); - } + size_t size() const; - //! Add addresses to addrman's new table. - bool Add(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty = 0) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - int nAdd = 0; - Check(); - for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) - nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; - Check(); - if (nAdd) { - LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); - } - return nAdd > 0; - } + /** + * Attempt to add one or more addresses to addrman's new table. + * + * @param[in] vAddr Address records to attempt to add. + * @param[in] source The address of the node that sent us these addr records. + * @param[in] nTimePenalty A "time penalty" to apply to the address record's nTime. If a peer + * sends us an address record with nTime=n, then we'll add it to our + * addrman with nTime=(n - nTimePenalty). + * @return true if at least one address is successfully added. */ + bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty = 0); - //! Mark an entry as accessible. - void Good(const CService &addr, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Good_(addr, /* test_before_evict */ true, nTime); - Check(); - } + //! Mark an entry as accessible, possibly moving it from "new" to "tried". + void Good(const CService& addr, int64_t nTime = GetAdjustedTime()); //! Mark an entry as connection attempted to. - void Attempt(const CService &addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Attempt_(addr, fCountFailure, nTime); - Check(); - } + void Attempt(const CService& addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()); //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - ResolveCollisions_(); - Check(); - } - - //! Randomly select an address in tried that another address is attempting to evict. - CAddrInfo SelectTriedCollision() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo ret = SelectTriedCollision_(); - Check(); - return ret; - } + void ResolveCollisions(); /** - * Choose an address to connect to. + * Randomly select an address in the tried table that another address is + * attempting to evict. + * + * @return CAddress The record for the selected tried peer. + * int64_t The last time we attempted to connect to that peer. */ - CAddrInfo Select(bool newOnly = false) const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo addrRet = Select_(newOnly); - Check(); - return addrRet; - } + std::pair<CAddress, int64_t> SelectTriedCollision(); /** - * Return all or many randomly selected addresses, optionally by network. + * Choose an address to connect to. * - * @param[in] max_addresses Maximum number of addresses to return (0 = all). - * @param[in] max_pct Maximum percentage of addresses to return (0 = all). - * @param[in] network Select only addresses of this network (nullopt = all). + * @param[in] newOnly Whether to only select addresses from the new table. + * @return CAddress The record for the selected peer. + * int64_t The last time we attempted to connect to that peer. */ - std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - std::vector<CAddress> vAddr; - GetAddr_(vAddr, max_addresses, max_pct, network); - Check(); - return vAddr; - } - - //! Outer function for Connected_() - void Connected(const CService &addr, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Connected_(addr, nTime); - Check(); - } - - void SetServices(const CService &addr, ServiceFlags nServices) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - SetServices_(addr, nServices); - Check(); - } - - const std::vector<bool>& GetAsmap() const { return m_asmap; } - -private: - //! A mutex to protect the inner data structures. - mutable Mutex cs; - - //! Source of random numbers for randomization in inner loops - mutable FastRandomContext insecure_rand GUARDED_BY(cs); - - //! secret key to randomize bucket select with - uint256 nKey; - - //! Serialization versions. - enum Format : uint8_t { - V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 - V1_DETERMINISTIC = 1, //!< for pre-asmap files - V2_ASMAP = 2, //!< for files including asmap version - V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format - }; - - //! The maximum format this software knows it can unserialize. Also, we always serialize - //! in this format. - //! The format (first byte in the serialized stream) can be higher than this and - //! still this software may be able to unserialize the file - if the second byte - //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. - static constexpr Format FILE_FORMAT = Format::V3_BIP155; - - //! The initial value of a field that is incremented every time an incompatible format - //! change is made (such that old software versions would not be able to parse and - //! understand the new file format). This is 32 because we overtook the "key size" - //! field which was 32 historically. - //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. - static constexpr uint8_t INCOMPATIBILITY_BASE = 32; - - //! last used nId - int nIdCount GUARDED_BY(cs){0}; - - //! table with information about all nIds - std::unordered_map<int, CAddrInfo> mapInfo GUARDED_BY(cs); - - //! find an nId based on its network address - std::unordered_map<CNetAddr, int, CNetAddrHash> mapAddr GUARDED_BY(cs); - - //! randomly-ordered vector of all nIds - //! This is mutable because it is unobservable outside the class, so any - //! changes to it (even in const methods) are also unobservable. - mutable std::vector<int> vRandom GUARDED_BY(cs); - - // number of "tried" entries - int nTried GUARDED_BY(cs){0}; - - //! list of "tried" buckets - int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! number of (unique) "new" entries - int nNew GUARDED_BY(cs){0}; - - //! list of "new" buckets - int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. - int64_t nLastGood GUARDED_BY(cs){1}; - - //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. - std::set<int> m_tried_collisions; - - /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ - const int32_t m_consistency_check_ratio; - - // Compressed IP->ASN mapping, loaded from a file when a node starts. - // Should be always empty if no file was provided. - // This mapping is then used for bucketing nodes in Addrman. - // - // If asmap is provided, nodes will be bucketed by - // AS they belong to, in order to make impossible for a node - // to connect to several nodes hosted in a single AS. - // This is done in response to Erebus attack, but also to generally - // diversify the connections every node creates, - // especially useful when a large fraction of nodes - // operate under a couple of cloud providers. - // - // If a new asmap was provided, the existing records - // would be re-bucketed accordingly. - const std::vector<bool> m_asmap; - - //! Find an entry. - CAddrInfo* Find(const CNetAddr& addr, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. - CAddrInfo* Create(const CAddress &addr, const CNetAddr &addrSource, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Swap two elements in vRandom. - void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) const EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Move an entry from the "new" table(s) to the "tried" table - void MakeTried(CAddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Delete an entry. It must not be in tried, and have refcount 0. - void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Clear a position in a "new" table. This is the only place where entries are actually deleted. - void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry "good", possibly moving it from "new" to "tried". - void Good_(const CService &addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Add an entry to the "new" table. - bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry as attempted to connect. - void Attempt_(const CService &addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Select an address to connect to, if newOnly is set to true, only the new table is selected from. - CAddrInfo Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Return a random to-be-evicted tried table address. - CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Consistency check - void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs) - { - AssertLockHeld(cs); - - const int err = Check_(); - if (err) { - LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); - assert(false); - } - } - - //! Perform consistency check. Returns an error code or zero. - int Check_() const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::pair<CAddress, int64_t> Select(bool newOnly = false) const; /** * Return all or many randomly selected addresses, optionally by network. * - * @param[out] vAddr Vector of randomly selected addresses from vRandom. * @param[in] max_addresses Maximum number of addresses to return (0 = all). * @param[in] max_pct Maximum percentage of addresses to return (0 = all). * @param[in] network Select only addresses of this network (nullopt = all). + * + * @return A vector of randomly selected addresses from vRandom. */ - void GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const; /** We have successfully connected to this peer. Calling this function * updates the CAddress's nTime, which is used in our IsTerrible() @@ -427,13 +130,12 @@ private: * @param[in] addr The address of the peer we were connected to * @param[in] nTime The time that we were last connected to this peer */ - void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Connected(const CService& addr, int64_t nTime = GetAdjustedTime()); //! Update an entry's service bits. - void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + void SetServices(const CService& addr, ServiceFlags nServices); - friend class CAddrManTest; - friend class CAddrManDeterministic; + const std::vector<bool>& GetAsmap() const; }; #endif // BITCOIN_ADDRMAN_H diff --git a/src/addrman_impl.h b/src/addrman_impl.h new file mode 100644 index 0000000000..10a65871c1 --- /dev/null +++ b/src/addrman_impl.h @@ -0,0 +1,278 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRMAN_IMPL_H +#define BITCOIN_ADDRMAN_IMPL_H + +#include <logging.h> +#include <logging/timer.h> +#include <netaddress.h> +#include <protocol.h> +#include <serialize.h> +#include <sync.h> +#include <uint256.h> + +#include <cstdint> +#include <optional> +#include <set> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +/** Total number of buckets for tried addresses */ +static constexpr int32_t ADDRMAN_TRIED_BUCKET_COUNT_LOG2{8}; +static constexpr int ADDRMAN_TRIED_BUCKET_COUNT{1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2}; +/** Total number of buckets for new addresses */ +static constexpr int32_t ADDRMAN_NEW_BUCKET_COUNT_LOG2{10}; +static constexpr int ADDRMAN_NEW_BUCKET_COUNT{1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2}; +/** Maximum allowed number of entries in buckets for new and tried addresses */ +static constexpr int32_t ADDRMAN_BUCKET_SIZE_LOG2{6}; +static constexpr int ADDRMAN_BUCKET_SIZE{1 << ADDRMAN_BUCKET_SIZE_LOG2}; + +/** + * Extended statistics about a CAddress + */ +class AddrInfo : public CAddress +{ +public: + //! last try whatsoever by us (memory only) + int64_t nLastTry{0}; + + //! last counted attempt (memory only) + int64_t nLastCountAttempt{0}; + + //! where knowledge about this address first came from + CNetAddr source; + + //! last successful connection by us + int64_t nLastSuccess{0}; + + //! connection attempts since last successful attempt + int nAttempts{0}; + + //! reference count in new sets (memory only) + int nRefCount{0}; + + //! in tried set? (memory only) + bool fInTried{false}; + + //! position in vRandom + mutable int nRandomPos{-1}; + + SERIALIZE_METHODS(AddrInfo, obj) + { + READWRITEAS(CAddress, obj); + READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); + } + + AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) + { + } + + AddrInfo() : CAddress(), source() + { + } + + //! Calculate in which "tried" bucket this entry belongs + int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; + + //! Calculate in which "new" bucket this entry belongs, given a certain source + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; + + //! Calculate in which "new" bucket this entry belongs, using its default source + int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const + { + return GetNewBucket(nKey, source, asmap); + } + + //! Calculate in which position of a bucket to store this entry. + int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; + + //! Determine whether the statistics about this entry are bad enough so that it can just be deleted + bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; + + //! Calculate the relative chance this entry should be given when selecting nodes to connect to + double GetChance(int64_t nNow = GetAdjustedTime()) const; +}; + +class AddrManImpl +{ +public: + AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio); + + ~AddrManImpl(); + + template <typename Stream> + void Serialize(Stream& s_) const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + template <typename Stream> + void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); + + size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Good(const CService& addr, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Attempt(const CService& addr, bool fCountFailure, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void ResolveCollisions() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair<CAddress, int64_t> SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair<CAddress, int64_t> Select(bool newOnly) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Connected(const CService& addr, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void SetServices(const CService& addr, ServiceFlags nServices) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + const std::vector<bool>& GetAsmap() const; + + friend class AddrManTest; + friend class AddrManDeterministic; + +private: + //! A mutex to protect the inner data structures. + mutable Mutex cs; + + //! Source of random numbers for randomization in inner loops + mutable FastRandomContext insecure_rand GUARDED_BY(cs); + + //! secret key to randomize bucket select with + uint256 nKey; + + //! Serialization versions. + enum Format : uint8_t { + V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 + V1_DETERMINISTIC = 1, //!< for pre-asmap files + V2_ASMAP = 2, //!< for files including asmap version + V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format + V4_MULTIPORT = 4, //!< adds support for multiple ports per IP + }; + + //! The maximum format this software knows it can unserialize. Also, we always serialize + //! in this format. + //! The format (first byte in the serialized stream) can be higher than this and + //! still this software may be able to unserialize the file - if the second byte + //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. + static constexpr Format FILE_FORMAT = Format::V4_MULTIPORT; + + //! The initial value of a field that is incremented every time an incompatible format + //! change is made (such that old software versions would not be able to parse and + //! understand the new file format). This is 32 because we overtook the "key size" + //! field which was 32 historically. + //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. + static constexpr uint8_t INCOMPATIBILITY_BASE = 32; + + //! last used nId + int nIdCount GUARDED_BY(cs){0}; + + //! table with information about all nIds + std::unordered_map<int, AddrInfo> mapInfo GUARDED_BY(cs); + + //! find an nId based on its network address and port. + std::unordered_map<CService, int, CServiceHash> mapAddr GUARDED_BY(cs); + + //! randomly-ordered vector of all nIds + //! This is mutable because it is unobservable outside the class, so any + //! changes to it (even in const methods) are also unobservable. + mutable std::vector<int> vRandom GUARDED_BY(cs); + + // number of "tried" entries + int nTried GUARDED_BY(cs){0}; + + //! list of "tried" buckets + int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! number of (unique) "new" entries + int nNew GUARDED_BY(cs){0}; + + //! list of "new" buckets + int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. + int64_t nLastGood GUARDED_BY(cs){1}; + + //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. + std::set<int> m_tried_collisions; + + /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ + const int32_t m_consistency_check_ratio; + + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + const std::vector<bool> m_asmap; + + //! Find an entry. + AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Swap two elements in vRandom. + void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Delete an entry. It must not be in tried, and have refcount 0. + void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Clear a position in a "new" table. This is the only place where entries are actually deleted. + void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Move an entry from the "new" table(s) to the "tried" table + void MakeTried(AddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + /** Attempt to add a single address to addrman's new table. + * @see AddrMan::Add() for parameters. */ + bool AddSingle(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Good_(const CService& addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); + + bool Add_(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair<CAddress, int64_t> Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair<CAddress, int64_t> SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Consistency check, taking into account m_consistency_check_ratio. + //! Will std::abort if an inconsistency is detected. + void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Perform consistency check, regardless of m_consistency_check_ratio. + //! @returns an error code or zero. + int CheckAddrman() const EXCLUSIVE_LOCKS_REQUIRED(cs); +}; + +#endif // BITCOIN_ADDRMAN_IMPL_H diff --git a/src/banman.h b/src/banman.h index 8a03a9e3fc..f495dab49d 100644 --- a/src/banman.h +++ b/src/banman.h @@ -6,7 +6,7 @@ #define BITCOIN_BANMAN_H #include <addrdb.h> -#include <bloom.h> +#include <common/bloom.h> #include <fs.h> #include <net_types.h> // For banmap_t #include <sync.h> diff --git a/src/bech32.cpp b/src/bech32.cpp index 288b14e023..9da2488ef2 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -66,6 +66,26 @@ uint32_t PolyMod(const data& v) // the above example, `c` initially corresponds to 1 mod g(x), and after processing 2 inputs of // v, it corresponds to x^2 + v0*x + v1 mod g(x). As 1 mod g(x) = 1, that is the starting value // for `c`. + + // The following Sage code constructs the generator used: + // + // B = GF(2) # Binary field + // BP.<b> = B[] # Polynomials over the binary field + // F_mod = b**5 + b**3 + 1 + // F.<f> = GF(32, modulus=F_mod, repr='int') # GF(32) definition + // FP.<x> = F[] # Polynomials over GF(32) + // E_mod = x**2 + F.fetch_int(9)*x + F.fetch_int(23) + // E.<e> = F.extension(E_mod) # GF(1024) extension field definition + // for p in divisors(E.order() - 1): # Verify e has order 1023. + // assert((e**p == 1) == (p % 1023 == 0)) + // G = lcm([(e**i).minpoly() for i in range(997,1000)]) + // print(G) # Print out the generator + // + // It demonstrates that g(x) is the least common multiple of the minimal polynomials + // of 3 consecutive powers (997,998,999) of a primitive element (e) of GF(1024). + // That guarantees it is, in fact, the generator of a primitive BCH code with cycle + // length 1023 and distance 4. See https://en.wikipedia.org/wiki/BCH_code for more details. + uint32_t c = 1; for (const auto v_i : v) { // We want to update `c` to correspond to a polynomial with one extra term. If the initial @@ -88,12 +108,21 @@ uint32_t PolyMod(const data& v) // Then compute c1*x^5 + c2*x^4 + c3*x^3 + c4*x^2 + c5*x + v_i: c = ((c & 0x1ffffff) << 5) ^ v_i; - // Finally, for each set bit n in c0, conditionally add {2^n}k(x): + // Finally, for each set bit n in c0, conditionally add {2^n}k(x). These constants can be + // computed using the following Sage code (continuing the code above): + // + // for i in [1,2,4,8,16]: # Print out {1,2,4,8,16}*(g(x) mod x^6), packed in hex integers. + // v = 0 + // for coef in reversed((F.fetch_int(i)*(G % x**6)).coefficients(sparse=True)): + // v = v*32 + coef.integer_representation() + // print("0x%x" % v) + // if (c0 & 1) c ^= 0x3b6a57b2; // k(x) = {29}x^5 + {22}x^4 + {20}x^3 + {21}x^2 + {29}x + {18} if (c0 & 2) c ^= 0x26508e6d; // {2}k(x) = {19}x^5 + {5}x^4 + x^3 + {3}x^2 + {19}x + {13} if (c0 & 4) c ^= 0x1ea119fa; // {4}k(x) = {15}x^5 + {10}x^4 + {2}x^3 + {6}x^2 + {15}x + {26} if (c0 & 8) c ^= 0x3d4233dd; // {8}k(x) = {30}x^5 + {20}x^4 + {4}x^3 + {12}x^2 + {30}x + {29} if (c0 & 16) c ^= 0x2a1462b3; // {16}k(x) = {21}x^5 + x^4 + {8}x^3 + {24}x^2 + {21}x + {19} + } return c; } @@ -125,7 +154,8 @@ Encoding VerifyChecksum(const std::string& hrp, const data& values) // PolyMod computes what value to xor into the final values to make the checksum 0. However, // if we required that the checksum was 0, it would be the case that appending a 0 to a valid // list of values would result in a new valid list. For that reason, Bech32 requires the - // resulting checksum to be 1 instead. In Bech32m, this constant was amended. + // resulting checksum to be 1 instead. In Bech32m, this constant was amended. See + // https://gist.github.com/sipa/14c248c288c3880a3b191f978a34508e for details. const uint32_t check = PolyMod(Cat(ExpandHRP(hrp), values)); if (check == EncodingConstant(Encoding::BECH32)) return Encoding::BECH32; if (check == EncodingConstant(Encoding::BECH32M)) return Encoding::BECH32M; diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index e5dd571a4c..d6834a239b 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -53,14 +53,14 @@ static void CreateAddresses() } } -static void AddAddressesToAddrMan(CAddrMan& addrman) +static void AddAddressesToAddrMan(AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { addrman.Add(g_addresses[source_i], g_sources[source_i]); } } -static void FillAddrMan(CAddrMan& addrman) +static void FillAddrMan(AddrMan& addrman) { CreateAddresses(); @@ -74,26 +74,26 @@ static void AddrManAdd(benchmark::Bench& bench) CreateAddresses(); bench.run([&] { - CAddrMan addrman{/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0}; + AddrMan addrman{/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0}; AddAddressesToAddrMan(addrman); }); } static void AddrManSelect(benchmark::Bench& bench) { - CAddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); FillAddrMan(addrman); bench.run([&] { const auto& address = addrman.Select(); - assert(address.GetPort() > 0); + assert(address.first.GetPort() > 0); }); } static void AddrManGetAddr(benchmark::Bench& bench) { - CAddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); FillAddrMan(addrman); @@ -103,41 +103,33 @@ static void AddrManGetAddr(benchmark::Bench& bench) }); } -static void AddrManGood(benchmark::Bench& bench) +static void AddrManAddThenGood(benchmark::Bench& bench) { - /* Create many CAddrMan objects - one to be modified at each loop iteration. - * This is necessary because the CAddrMan::Good() method modifies the - * object, affecting the timing of subsequent calls to the same method and - * we want to do the same amount of work in every loop iteration. */ - - bench.epochs(5).epochIterations(1); - const uint64_t addrman_count{bench.epochs() * bench.epochIterations()}; - Assert(addrman_count == 5U); - - std::vector<std::unique_ptr<CAddrMan>> addrmans(addrman_count); - for (size_t i{0}; i < addrman_count; ++i) { - addrmans[i] = std::make_unique<CAddrMan>(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); - FillAddrMan(*addrmans[i]); - } - - auto markSomeAsGood = [](CAddrMan& addrman) { + auto markSomeAsGood = [](AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { - if (addr_i % 32 == 0) { - addrman.Good(g_addresses[source_i][addr_i]); - } + addrman.Good(g_addresses[source_i][addr_i]); } } }; - uint64_t i = 0; + CreateAddresses(); + bench.run([&] { - markSomeAsGood(*addrmans.at(i)); - ++i; + // To make the benchmark independent of the number of evaluations, we always prepare a new addrman. + // This is necessary because AddrMan::Good() method modifies the object, affecting the timing of subsequent calls + // to the same method and we want to do the same amount of work in every loop iteration. + // + // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in + // AddrMan::Good() will still be noticeable. + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddAddressesToAddrMan(addrman); + + markSomeAsGood(addrman); }); } BENCHMARK(AddrManAdd); BENCHMARK(AddrManSelect); BENCHMARK(AddrManGetAddr); -BENCHMARK(AddrManGood); +BENCHMARK(AddrManAddThenGood); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 8e10862a37..bc3685818e 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <bench/nanobench.h> #include <bech32.h> #include <util/strencodings.h> diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 012057e792..030bc43396 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -4,11 +4,18 @@ #include <bench/bench.h> -#include <chainparams.h> #include <test/util/setup_common.h> -#include <validation.h> +#include <chrono> +#include <fstream> +#include <functional> +#include <iostream> +#include <map> #include <regex> +#include <string> +#include <vector> + +using namespace std::chrono_literals; const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; @@ -61,6 +68,12 @@ void benchmark::BenchRunner::RunAll(const Args& args) Bench bench; bench.name(p.first); + if (args.min_time > 0ms) { + // convert to nanos before dividing to reduce rounding errors + std::chrono::nanoseconds min_time_ns = args.min_time; + bench.minEpochTime(min_time_ns / bench.epochs()); + } + if (args.asymptote.empty()) { p.second(bench); } else { diff --git a/src/bench/bench.h b/src/bench/bench.h index c4fcd80e33..b0e4006ee3 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -41,11 +41,12 @@ using ankerl::nanobench::Bench; typedef std::function<void(Bench&)> BenchFunction; struct Args { - std::string regex_filter; bool is_list_only; + std::chrono::milliseconds min_time; std::vector<double> asymptote; std::string output_csv; std::string output_json; + std::string regex_filter; }; class BenchRunner diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index aab777cac1..83609d2787 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -4,21 +4,28 @@ #include <bench/bench.h> +#include <clientversion.h> #include <crypto/sha256.h> #include <util/strencodings.h> #include <util/system.h> -#include <memory> +#include <chrono> +#include <cstdint> +#include <iostream> +#include <sstream> +#include <vector> static const char* DEFAULT_BENCH_FILTER = ".*"; +static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; static void SetupBenchArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); - argsman.AddArg("-asymptote=n1,n2,n3,...", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-asymptote=<n1,n2,n3,...>", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-min_time=<milliseconds>", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-output_csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-output_json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } @@ -48,17 +55,62 @@ int main(int argc, char** argv) } if (HelpRequested(argsman)) { - std::cout << argsman.GetHelpMessage(); + std::cout << "Usage: bench_bitcoin [options]\n" + "\n" + << argsman.GetHelpMessage() + << "Description:\n" + "\n" + " bench_bitcoin executes microbenchmarks. The quality of the benchmark results\n" + " highly depend on the stability of the machine. It can sometimes be difficult\n" + " to get stable, repeatable results, so here are a few tips:\n" + "\n" + " * Use pyperf [1] to disable frequency scaling, turbo boost etc. For best\n" + " results, use CPU pinning and CPU isolation (see [2]).\n" + "\n" + " * Each call of run() should do exactly the same work. E.g. inserting into\n" + " a std::vector doesn't do that as it will reallocate on certain calls. Make\n" + " sure each run has exactly the same preconditions.\n" + "\n" + " * If results are still not reliable, increase runtime with e.g.\n" + " -min_time=5000 to let a benchmark run for at least 5 seconds.\n" + "\n" + " * bench_bitcoin uses nanobench [3] for which there is extensive\n" + " documentation available online.\n" + "\n" + "Environment Variables:\n" + "\n" + " To attach a profiler you can run a benchmark in endless mode. This can be\n" + " done with the environment variable NANOBENCH_ENDLESS. E.g. like so:\n" + "\n" + " NANOBENCH_ENDLESS=MuHash ./bench_bitcoin -filter=MuHash\n" + "\n" + " In rare cases it can be useful to suppress stability warnings. This can be\n" + " done with the environment variable NANOBENCH_SUPPRESS_WARNINGS, e.g:\n" + "\n" + " NANOBENCH_SUPPRESS_WARNINGS=1 ./bench_bitcoin\n" + "\n" + "Notes:\n" + "\n" + " 1. pyperf\n" + " https://github.com/psf/pyperf\n" + "\n" + " 2. CPU pinning & isolation\n" + " https://pyperf.readthedocs.io/en/latest/system.html\n" + "\n" + " 3. nanobench\n" + " https://github.com/martinus/nanobench\n" + "\n"; return EXIT_SUCCESS; } benchmark::Args args; - args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); - args.is_list_only = argsman.GetBoolArg("-list", false); args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); + args.is_list_only = argsman.GetBoolArg("-list", false); + args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min_time", DEFAULT_MIN_TIME_MS)); args.output_csv = argsman.GetArg("-output_csv", ""); args.output_json = argsman.GetArg("-output_json", ""); + args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); benchmark::BenchRunner::RunAll(args); diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index b4b33d115f..0577ab80e3 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -34,10 +34,10 @@ static void AssembleBlock(benchmark::Bench& bench) txs.at(b) = MakeTransactionRef(tx); } { - LOCK(::cs_main); // Required for ::AcceptToMemoryPool. + LOCK(::cs_main); for (const auto& txr : txs) { - const MempoolAcceptResult res = ::AcceptToMemoryPool(test_setup->m_node.chainman->ActiveChainstate(), *test_setup->m_node.mempool, txr, false /* bypass_limits */); + const MempoolAcceptResult res = test_setup->m_node.chainman->ProcessTransaction(txr); assert(res.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 934b574f8b..f6a8c56743 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -32,8 +32,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); - wallet.SetupLegacyScriptPubKeyMan(); + CWallet wallet(chain.get(), "", gArgs, CreateDummyWalletDatabase()); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -65,10 +64,6 @@ static void CoinSelection(benchmark::Bench& bench) } typedef std::set<CInputCoin> CoinSet; -static NodeContext testNode; -static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); -std::vector<std::unique_ptr<CWalletTx>> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup>& set) @@ -76,10 +71,9 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup> CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - std::unique_ptr<CWalletTx> wtx = std::make_unique<CWalletTx>(MakeTransactionRef(std::move(tx))); + CInputCoin coin(MakeTransactionRef(tx), nInput); set.emplace_back(); - set.back().Insert(COutput(testWallet, *wtx, nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); - wtxn.emplace_back(std::move(wtx)); + set.back().Insert(coin, 0, true, 0, 0, false); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) @@ -97,7 +91,6 @@ static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) static void BnBExhaustion(benchmark::Bench& bench) { // Setup - testWallet.SetupLegacyScriptPubKeyMan(); std::vector<OutputGroup> utxo_pool; CoinSet selection; CAmount value_ret = 0; diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 30fe11be6b..d36e504bfc 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -110,9 +110,9 @@ static void MuHash(benchmark::Bench& bench) { MuHash3072 acc; unsigned char key[32] = {0}; - int i = 0; + uint32_t i = 0; bench.run([&] { - key[0] = ++i; + key[0] = ++i & 0xFF; acc *= MuHash3072(key); }); } @@ -134,10 +134,6 @@ static void MuHashDiv(benchmark::Bench& bench) FastRandomContext rng(true); MuHash3072 muhash{rng.randbytes(32)}; - for (size_t i = 0; i < bench.epochIterations(); ++i) { - acc *= muhash; - } - bench.run([&] { acc /= muhash; }); diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index f28768efc8..a0a82ea359 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -6,6 +6,7 @@ #include <policy/policy.h> #include <test/util/setup_common.h> #include <txmempool.h> +#include <validation.h> #include <vector> @@ -26,14 +27,8 @@ struct Available { Available(CTransactionRef& ref, size_t tx_count) : ref(ref), tx_count(tx_count){} }; -static void ComplexMemPool(benchmark::Bench& bench) +static std::vector<CTransactionRef> CreateOrderedCoins(FastRandomContext& det_rand, int childTxs, int min_ancestors) { - int childTxs = 800; - if (bench.complexityN() > 1) { - childTxs = static_cast<int>(bench.complexityN()); - } - - FastRandomContext det_rand{true}; std::vector<Available> available_coins; std::vector<CTransactionRef> ordered_coins; // Create some base transactions @@ -58,8 +53,10 @@ static void ComplexMemPool(benchmark::Bench& bench) size_t idx = det_rand.randrange(available_coins.size()); Available coin = available_coins[idx]; uint256 hash = coin.ref->GetHash(); - // biased towards taking just one ancestor, but maybe more - size_t n_to_take = det_rand.randrange(2) == 0 ? 1 : 1+det_rand.randrange(coin.ref->vout.size() - coin.vin_left); + // biased towards taking min_ancestors parents, but maybe more + size_t n_to_take = det_rand.randrange(2) == 0 ? + min_ancestors : + min_ancestors + det_rand.randrange(coin.ref->vout.size() - coin.vin_left); for (size_t i = 0; i < n_to_take; ++i) { tx.vin.emplace_back(); tx.vin.back().prevout = COutPoint(hash, coin.vin_left++); @@ -79,6 +76,17 @@ static void ComplexMemPool(benchmark::Bench& bench) ordered_coins.emplace_back(MakeTransactionRef(tx)); available_coins.emplace_back(ordered_coins.back(), tx_counter++); } + return ordered_coins; +} + +static void ComplexMemPool(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + int childTxs = 800; + if (bench.complexityN() > 1) { + childTxs = static_cast<int>(bench.complexityN()); + } + std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 1); const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN); CTxMemPool pool; LOCK2(cs_main, pool.cs); @@ -91,4 +99,21 @@ static void ComplexMemPool(benchmark::Bench& bench) }); } +static void MempoolCheck(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + const int childTxs = bench.complexityN() > 1 ? static_cast<int>(bench.complexityN()) : 2000; + const std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 5); + const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN, {"-checkmempool=1"}); + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + const CCoinsViewCache& coins_tip = testing_setup.get()->m_node.chainman->ActiveChainstate().CoinsTip(); + for (auto& tx : ordered_coins) AddTx(tx, pool); + + bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { + pool.check(coins_tip, /* spendheight */ 2); + }); +} + BENCHMARK(ComplexMemPool); +BENCHMARK(MempoolCheck); diff --git a/src/bench/nanobench.h b/src/bench/nanobench.h index 030d6ebf6a..27df08fb69 100644 --- a/src/bench/nanobench.h +++ b/src/bench/nanobench.h @@ -33,7 +33,7 @@ // see https://semver.org/ #define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes #define ANKERL_NANOBENCH_VERSION_MINOR 3 // backwards-compatible changes -#define ANKERL_NANOBENCH_VERSION_PATCH 4 // backwards-compatible bug fixes +#define ANKERL_NANOBENCH_VERSION_PATCH 6 // backwards-compatible bug fixes /////////////////////////////////////////////////////////////////////////////////////////////////// // public facing api - as minimal as possible @@ -88,13 +88,15 @@ } while (0) #endif -#if defined(__linux__) && defined(PERF_EVENT_IOC_ID) && defined(PERF_COUNT_HW_REF_CPU_CYCLES) && defined(PERF_FLAG_FD_CLOEXEC) && \ - !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS) -// only enable perf counters on kernel 3.14 which seems to have all the necessary defines. The three PERF_... defines are not in -// kernel 2.6.32 (all others are). -# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 1 -#else -# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0 +#define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0 +#if defined(__linux__) && !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS) +# include <linux/version.h> +# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) +// PERF_COUNT_HW_REF_CPU_CYCLES only available since kernel 3.3 +// PERF_FLAG_FD_CLOEXEC since kernel 3.14 +# undef ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS +# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 1 +# endif #endif #if defined(__clang__) @@ -2210,20 +2212,20 @@ struct IterationLogic::Impl { columns.emplace_back(10, 1, "err%", "%", rErrorMedian * 100.0); double rInsMedian = -1.0; - if (mResult.has(Result::Measure::instructions)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::instructions)) { rInsMedian = mResult.median(Result::Measure::instructions); columns.emplace_back(18, 2, "ins/" + mBench.unit(), "", rInsMedian / mBench.batch()); } double rCycMedian = -1.0; - if (mResult.has(Result::Measure::cpucycles)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::cpucycles)) { rCycMedian = mResult.median(Result::Measure::cpucycles); columns.emplace_back(18, 2, "cyc/" + mBench.unit(), "", rCycMedian / mBench.batch()); } if (rInsMedian > 0.0 && rCycMedian > 0.0) { columns.emplace_back(9, 3, "IPC", "", rCycMedian <= 0.0 ? 0.0 : rInsMedian / rCycMedian); } - if (mResult.has(Result::Measure::branchinstructions)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::branchinstructions)) { double rBraMedian = mResult.median(Result::Measure::branchinstructions); columns.emplace_back(17, 2, "bra/" + mBench.unit(), "", rBraMedian / mBench.batch()); if (mResult.has(Result::Measure::branchmisses)) { @@ -2402,6 +2404,14 @@ public: return (a + divisor / 2) / divisor; } + ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") + static inline uint32_t mix(uint32_t x) noexcept { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + return x; + } + template <typename Op> ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") void calibrate(Op&& op) { @@ -2441,15 +2451,10 @@ public: uint64_t const numIters = 100000U + (std::random_device{}() & 3); uint64_t n = numIters; uint32_t x = 1234567; - auto fn = [&]() { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - }; beginMeasure(); while (n-- > 0) { - fn(); + x = mix(x); } endMeasure(); detail::doNotOptimizeAway(x); @@ -2459,8 +2464,8 @@ public: beginMeasure(); while (n-- > 0) { // we now run *twice* so we can easily calculate the overhead - fn(); - fn(); + x = mix(x); + x = mix(x); } endMeasure(); detail::doNotOptimizeAway(x); diff --git a/src/bench/peer_eviction.cpp b/src/bench/peer_eviction.cpp index 46fd9d999e..8429f18613 100644 --- a/src/bench/peer_eviction.cpp +++ b/src/bench/peer_eviction.cpp @@ -20,19 +20,17 @@ static void EvictionProtectionCommon( { using Candidates = std::vector<NodeEvictionCandidate>; FastRandomContext random_context{true}; - bench.warmup(100).epochIterations(1100); Candidates candidates{GetRandomNodeEvictionCandidates(num_candidates, random_context)}; for (auto& c : candidates) { candidate_setup_fn(c); } - std::vector<Candidates> copies{ - static_cast<size_t>(bench.epochs() * bench.epochIterations()), candidates}; - size_t i{0}; + bench.run([&] { - ProtectEvictionCandidatesByRatio(copies.at(i)); - ++i; + // creating a copy has an overhead of about 3%, so it does not influence the benchmark results much. + auto copy = candidates; + ProtectEvictionCandidatesByRatio(copy); }); } diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 997ab56549..30bc1d5fdf 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -4,7 +4,7 @@ #include <bench/bench.h> -#include <bloom.h> +#include <common/bloom.h> static void RollingBloom(benchmark::Bench& bench) { @@ -13,16 +13,16 @@ static void RollingBloom(benchmark::Bench& bench) uint32_t count = 0; bench.run([&] { count++; - data[0] = count; - data[1] = count >> 8; - data[2] = count >> 16; - data[3] = count >> 24; + data[0] = count & 0xFF; + data[1] = (count >> 8) & 0xFF; + data[2] = (count >> 16) & 0xFF; + data[3] = (count >> 24) & 0xFF; filter.insert(data); - data[0] = count >> 24; - data[1] = count >> 16; - data[2] = count >> 8; - data[3] = count; + data[0] = (count >> 24) & 0xFF; + data[1] = (count >> 16) & 0xFF; + data[2] = (count >> 8) & 0xFF; + data[3] = count & 0xFF; filter.contains(data); }); } diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index c8886a4c23..3bef64f720 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -40,7 +40,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) { TestBlockAndIndex data; bench.run([&] { - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); ankerl::nanobench::doNotOptimizeAway(univalue); }); } @@ -50,7 +50,7 @@ BENCHMARK(BlockToJsonVerbose); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { TestBlockAndIndex data; - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); bench.run([&] { auto str = univalue.write(); ankerl::nanobench::doNotOptimizeAway(str); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index a205d8b6e7..8e3ca59496 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -14,21 +14,22 @@ #include <optional> -static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_watchonly, const bool add_mine) +static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", gArgs, CreateMockWalletDatabase()}; { - wallet.SetupLegacyScriptPubKeyMan(); + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } auto handler = test_setup->m_node.chain->handleNotifications({&wallet, [](CWallet*) {}}); const std::optional<std::string> address_mine{add_mine ? std::optional<std::string>{getnewaddress(wallet)} : std::nullopt}; - if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { generatetoaddress(test_setup->m_node, address_mine.value_or(ADDRESS_WATCHONLY)); @@ -42,14 +43,13 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b if (set_dirty) wallet.MarkDirty(); bal = GetBalance(wallet); if (add_mine) assert(bal.m_mine_trusted > 0); - if (add_watchonly) assert(bal.m_watchonly_trusted > 0); }); } -static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_watchonly */ true, /* add_mine */ true); } -static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ true); } -static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ false, /* add_mine */ true); } -static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ false); } +static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_mine */ true); } +static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } +static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } +static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ false); } BENCHMARK(WalletBalanceDirty); BENCHMARK(WalletBalanceClean); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 4ccd1f4fee..338d32ef6b 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -74,7 +74,7 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0). Pass \"help\" for detailed help documentation.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); SetupChainParamsBaseOptions(argsman); - argsman.AddArg("-color=<when>", strprintf("Color setting for CLI output (default: %s). Valid values: always, auto (add color codes when standard output is connected to a terminal and OS is not WIN32), never.", DEFAULT_COLOR_SETTING), ArgsManager::ALLOW_STRING, OptionsCategory::OPTIONS); + argsman.AddArg("-color=<when>", strprintf("Color setting for CLI output (default: %s). Valid values: always, auto (add color codes when standard output is connected to a terminal and OS is not WIN32), never.", DEFAULT_COLOR_SETTING), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-named", strprintf("Pass named instead of positional arguments (default: %s)", DEFAULT_NAMED), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcclienttimeout=<n>", strprintf("Timeout in seconds during HTTP requests, or 0 for no timeout. (default: %d)", DEFAULT_HTTP_CLIENT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcconnect=<ip>", strprintf("Send commands to node running on <ip> (default: %s)", DEFAULT_RPCCONNECT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -83,7 +83,7 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-rpcport=<port>", strprintf("Connect to JSON-RPC on <port> (default: %u, testnet: %u, signet: %u, regtest: %u)", defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort(), signetBaseParams->RPCPort(), regtestBaseParams->RPCPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcuser=<user>", "Username for JSON-RPC connections", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcwait", "Wait for RPC server to start", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-rpcwaittimeout=<n>", strprintf("Timeout in seconds to wait for the RPC server to start, or 0 for no timeout. (default: %d)", DEFAULT_WAIT_CLIENT_TIMEOUT), ArgsManager::ALLOW_INT, OptionsCategory::OPTIONS); + argsman.AddArg("-rpcwaittimeout=<n>", strprintf("Timeout in seconds to wait for the RPC server to start, or 0 for no timeout. (default: %d)", DEFAULT_WAIT_CLIENT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-rpcwallet=<walletname>", "Send RPC for non-default wallet on RPC server (needs to exactly match corresponding -wallet option passed to bitcoind). This changes the RPC endpoint used, e.g. http://127.0.0.1:8332/wallet/<walletname>", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-stdin", "Read extra arguments from standard input, one per line until EOF/Ctrl-D (recommended for sensitive information such as passphrases). When combined with -stdinrpcpass, the first line from standard input is used for the RPC password.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-stdinrpcpass", "Read RPC password from standard input as a single line. When combined with -stdin, the first line from standard input is used for the RPC password. When combined with -stdinwalletpassphrase, -stdinrpcpass consumes the first line, and -stdinwalletpassphrase consumes the second.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -337,7 +337,7 @@ public: connections.pushKV("total", batch[ID_NETWORKINFO]["result"]["connections"]); result.pushKV("connections", connections); - result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); + result.pushKV("networks", batch[ID_NETWORKINFO]["result"]["networks"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO]["result"].isNull()) { @@ -551,15 +551,26 @@ public: } // Report peer connection totals by type. - result += " ipv4 ipv6 onion"; - const bool any_i2p_peers = m_counts.at(2).at(3); // false if total i2p peers count is 0, otherwise true - if (any_i2p_peers) result += " i2p"; + result += " "; + std::vector<int8_t> reachable_networks; + for (const UniValue& network : networkinfo["networks"].getValues()) { + if (network["reachable"].get_bool()) { + const std::string& network_name{network["name"].get_str()}; + const int8_t network_id{NetworkStringToId(network_name)}; + if (network_id == UNKNOWN_NETWORK) continue; + result += strprintf("%8s", network_name); // column header + reachable_networks.push_back(network_id); + } + }; result += " total block"; if (m_manual_peers_count) result += " manual"; + const std::array rows{"in", "out", "total"}; - for (uint8_t i = 0; i < 3; ++i) { - result += strprintf("\n%-5s %5i %5i %5i", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2)); // ipv4/ipv6/onion peers counts - if (any_i2p_peers) result += strprintf(" %5i", m_counts.at(i).at(3)); // i2p peers count + for (size_t i = 0; i < rows.size(); ++i) { + result += strprintf("\n%-5s", rows[i]); // row header + for (int8_t n : reachable_networks) { + result += strprintf("%8i", m_counts.at(i).at(n)); // network peers count + } result += strprintf(" %5i", m_counts.at(i).at(m_networks.size())); // total peers count if (i == 1) { // the outbound row has two extra columns for block relay and manual peer counts result += strprintf(" %5i", m_block_relay_peers_count); @@ -702,7 +713,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co // 3. default port for chain uint16_t port{BaseParams().RPCPort()}; SplitHostPort(gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT), port, host); - port = static_cast<uint16_t>(gArgs.GetArg("-rpcport", port)); + port = static_cast<uint16_t>(gArgs.GetIntArg("-rpcport", port)); // Obtain event base raii_event_base base = obtain_event_base(); @@ -712,7 +723,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co // Set connection timeout { - const int timeout = gArgs.GetArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT); + const int timeout = gArgs.GetIntArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT); if (timeout > 0) { evhttp_connection_set_timeout(evcon.get(), timeout); } else { @@ -786,7 +797,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( "Could not locate RPC credentials. No authentication cookie could be found, and RPC password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)", - GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)).string())); + fs::PathToString(GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME))))); } else { throw std::runtime_error("Authorization failed: Incorrect rpcuser or rpcpassword"); } @@ -822,7 +833,7 @@ static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& str UniValue response(UniValue::VOBJ); // Execute and handle connection failures with -rpcwait. const bool fWait = gArgs.GetBoolArg("-rpcwait", false); - const int timeout = gArgs.GetArg("-rpcwaittimeout", DEFAULT_WAIT_CLIENT_TIMEOUT); + const int timeout = gArgs.GetIntArg("-rpcwaittimeout", DEFAULT_WAIT_CLIENT_TIMEOUT); const auto deadline{GetTime<std::chrono::microseconds>() + 1s * timeout}; do { @@ -986,8 +997,26 @@ static void ParseGetInfoResult(UniValue& result) RESET); result_string += strprintf("Version: %s\n", result["version"].getValStr()); result_string += strprintf("Time offset (s): %s\n", result["timeoffset"].getValStr()); - const std::string proxy = result["proxy"].getValStr(); - result_string += strprintf("Proxy: %s\n", proxy.empty() ? "N/A" : proxy); + + // proxies + std::map<std::string, std::vector<std::string>> proxy_networks; + std::vector<std::string> ordered_proxies; + + for (const UniValue& network : result["networks"].getValues()) { + const std::string proxy = network["proxy"].getValStr(); + if (proxy.empty()) continue; + // Add proxy to ordered_proxy if has not been processed + if (proxy_networks.find(proxy) == proxy_networks.end()) ordered_proxies.push_back(proxy); + + proxy_networks[proxy].push_back(network["name"].getValStr()); + } + + std::vector<std::string> formatted_proxies; + for (const std::string& proxy : ordered_proxies) { + formatted_proxies.emplace_back(strprintf("%s (%s)", proxy, Join(proxy_networks.find(proxy)->second, ", "))); + } + result_string += strprintf("Proxies: %s\n", formatted_proxies.empty() ? "n/a" : Join(formatted_proxies, ", ")); + result_string += strprintf("Min tx relay fee rate (%s/kvB): %s\n\n", CURRENCY_UNIT, result["relayfee"].getValStr()); if (!result["has_wallet"].isNull()) { diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 58c51bd8e0..eb97cfc6f6 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -8,6 +8,7 @@ #include <clientversion.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <core_io.h> #include <key_io.h> @@ -234,6 +235,16 @@ static void MutateTxRBFOptIn(CMutableTransaction& tx, const std::string& strInId } } +template <typename T> +static T TrimAndParse(const std::string& int_str, const std::string& err) +{ + const auto parsed{ToIntegral<T>(TrimString(int_str))}; + if (!parsed.has_value()) { + throw std::runtime_error(err + " '" + int_str + "'"); + } + return parsed.value(); +} + static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInput) { std::vector<std::string> vStrInputParts; @@ -260,8 +271,9 @@ static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInpu // extract the optional sequence number uint32_t nSequenceIn = CTxIn::SEQUENCE_FINAL; - if (vStrInputParts.size() > 2) - nSequenceIn = std::stoul(vStrInputParts[2]); + if (vStrInputParts.size() > 2) { + nSequenceIn = TrimAndParse<uint32_t>(vStrInputParts.at(2), "invalid TX sequence id"); + } // append to transaction input list CTxIn txin(txid, vout, CScript(), nSequenceIn); @@ -351,10 +363,10 @@ static void MutateTxAddOutMultiSig(CMutableTransaction& tx, const std::string& s CAmount value = ExtractAndValidateValue(vStrInputParts[0]); // Extract REQUIRED - uint32_t required = stoul(vStrInputParts[1]); + const uint32_t required{TrimAndParse<uint32_t>(vStrInputParts.at(1), "invalid multisig required number")}; // Extract NUMKEYS - uint32_t numkeys = stoul(vStrInputParts[2]); + const uint32_t numkeys{TrimAndParse<uint32_t>(vStrInputParts.at(2), "invalid multisig total number")}; // Validate there are the correct number of pubkeys if (vStrInputParts.size() < numkeys + 3) @@ -727,7 +739,7 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, static void OutputTxJSON(const CTransaction& tx) { UniValue entry(UniValue::VOBJ); - TxToUniv(tx, uint256(), /* include_addresses */ false, entry); + TxToUniv(tx, uint256(), entry); std::string jsonOutput = entry.write(4); tfm::format(std::cout, "%s\n", jsonOutput); diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 765954c92e..afa5b99549 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -8,6 +8,7 @@ #include <chainparams.h> #include <chainparamsbase.h> +#include <interfaces/init.h> #include <logging.h> #include <util/system.h> #include <util/translation.h> @@ -27,9 +28,10 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-wallet=<wallet-name>", "Specify wallet name", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::OPTIONS); - argsman.AddArg("-dumpfile=<file name>", "When used with 'dump', writes out the records to this file. When used with 'createfromdump', loads the records into a new wallet.", ArgsManager::ALLOW_STRING, OptionsCategory::OPTIONS); + argsman.AddArg("-dumpfile=<file name>", "When used with 'dump', writes out the records to this file. When used with 'createfromdump', loads the records into a new wallet.", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-debug=<category>", "Output debugging information (default: 0).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-descriptors", "Create descriptors wallet. Only for 'create'", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); + argsman.AddArg("-descriptors", "Create descriptors wallet. Only for 'create'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-legacy", "Create legacy wallet. Only for 'create'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); @@ -83,6 +85,13 @@ int main(int argc, char* argv[]) util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); #endif + + int exit_status; + std::unique_ptr<interfaces::Init> init = interfaces::MakeWalletInit(argc, argv, exit_status); + if (!init) { + return exit_status; + } + SetupEnvironment(); RandomInit(); try { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 654679af27..25ec2809e9 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -19,6 +19,7 @@ #include <shutdown.h> #include <util/check.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/threadnames.h> #include <util/tokenpipe.h> @@ -238,6 +239,7 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) daemon_ep.Close(); } #endif + SetSyscallSandboxPolicy(SyscallSandboxPolicy::SHUTOFF); if (fRet) { WaitForShutdown(); } diff --git a/src/chain.h b/src/chain.h index 84a3a4e1e7..15ca8f8750 100644 --- a/src/chain.h +++ b/src/chain.h @@ -126,7 +126,15 @@ enum BlockStatus: uint32_t { BLOCK_FAILED_CHILD = 64, //!< descends from failed block BLOCK_FAILED_MASK = BLOCK_FAILED_VALID | BLOCK_FAILED_CHILD, - BLOCK_OPT_WITNESS = 128, //!< block data in blk*.data was received with a witness-enforcing client + BLOCK_OPT_WITNESS = 128, //!< block data in blk*.dat was received with a witness-enforcing client + + /** + * If set, this indicates that the block index entry is assumed-valid. + * Certain diagnostics will be skipped in e.g. CheckBlockIndex(). + * It almost certainly means that the block's full validation is pending + * on a background chainstate. See `doc/assumeutxo.md`. + */ + BLOCK_ASSUMED_VALID = 256, }; /** The block chain is a tree shaped structure starting with the @@ -170,7 +178,7 @@ public: //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero only if and only if transactions for this block and all its parents are available. - //! Change to 64-bit type when necessary; won't happen before 2030 + //! Change to 64-bit type before 2024 (assuming worst case of 60 byte transactions). //! //! Note: this value is faked during use of a UTXO snapshot because we don't //! have the underlying block data available during snapshot load. @@ -300,14 +308,24 @@ public: return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } + //! @returns true if the block is assumed-valid; this means it is queued to be + //! validated by a background chainstate. + bool IsAssumedValid() const { return nStatus & BLOCK_ASSUMED_VALID; } + //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. bool RaiseValidity(enum BlockStatus nUpTo) { assert(!(nUpTo & ~BLOCK_VALID_MASK)); // Only validity flags allowed. - if (nStatus & BLOCK_FAILED_MASK) - return false; + if (nStatus & BLOCK_FAILED_MASK) return false; + if ((nStatus & BLOCK_VALID_MASK) < nUpTo) { + // If this block had been marked assumed-valid and we're raising + // its validity to a certain point, there is no longer an assumption. + if (nStatus & BLOCK_ASSUMED_VALID && nUpTo >= BLOCK_VALID_SCRIPTS) { + nStatus &= ~BLOCK_ASSUMED_VALID; + } + nStatus = (nStatus & ~BLOCK_VALID_MASK) | nUpTo; return true; } diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 4cc37560a3..2e823c1211 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -118,15 +118,15 @@ public: // This is fine at runtime as we'll fall back to using them as an addrfetch if they don't support the // service bits we want, but we should get them updated to support all service bits wanted by any // release ASAP to avoid it where possible. - vSeeds.emplace_back("seed.bitcoin.sipa.be"); // Pieter Wuille, only supports x1, x5, x9, and xd - vSeeds.emplace_back("dnsseed.bluematt.me"); // Matt Corallo, only supports x9 - vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org"); // Luke Dashjr - vSeeds.emplace_back("seed.bitcoinstats.com"); // Christian Decker, supports x1 - xf - vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch"); // Jonas Schnelli, only supports x1, x5, x9, and xd - vSeeds.emplace_back("seed.btc.petertodd.org"); // Peter Todd, only supports x1, x5, x9, and xd - vSeeds.emplace_back("seed.bitcoin.sprovoost.nl"); // Sjors Provoost - vSeeds.emplace_back("dnsseed.emzy.de"); // Stephan Oeste - vSeeds.emplace_back("seed.bitcoin.wiz.biz"); // Jason Maurice + vSeeds.emplace_back("seed.bitcoin.sipa.be."); // Pieter Wuille, only supports x1, x5, x9, and xd + vSeeds.emplace_back("dnsseed.bluematt.me."); // Matt Corallo, only supports x9 + vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org."); // Luke Dashjr + vSeeds.emplace_back("seed.bitcoinstats.com."); // Christian Decker, supports x1 - xf + vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch."); // Jonas Schnelli, only supports x1, x5, x9, and xd + vSeeds.emplace_back("seed.btc.petertodd.org."); // Peter Todd, only supports x1, x5, x9, and xd + vSeeds.emplace_back("seed.bitcoin.sprovoost.nl."); // Sjors Provoost + vSeeds.emplace_back("dnsseed.emzy.de."); // Stephan Oeste + vSeeds.emplace_back("seed.bitcoin.wiz.biz."); // Jason Maurice base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,0); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,5); @@ -230,10 +230,10 @@ public: vFixedSeeds.clear(); vSeeds.clear(); // nodes with support for servicebits filtering should be at the top - vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch"); - vSeeds.emplace_back("seed.tbtc.petertodd.org"); - vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl"); - vSeeds.emplace_back("testnet-seed.bluematt.me"); // Just a static list of stable node(s), only supports x9 + vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch."); + vSeeds.emplace_back("seed.tbtc.petertodd.org."); + vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl."); + vSeeds.emplace_back("testnet-seed.bluematt.me."); // Just a static list of stable node(s), only supports x9 base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,196); @@ -280,8 +280,10 @@ public: if (!args.IsArgSet("-signetchallenge")) { bin = ParseHex("512103ad5e0edad18cb1f0fc0d28a3d4f1f3e445640337489abb10404f2d1e086be430210359ef5021964fe22d6f8e05b2463c9540ce96883fe3b278760f048f5189f2e6c452ae"); + vSeeds.emplace_back("seed.signet.bitcoin.sprovoost.nl."); + + // Hardcoded nodes can be removed once there are more DNS seeds vSeeds.emplace_back("178.128.221.177"); - vSeeds.emplace_back("2a01:7c8:d005:390::5"); vSeeds.emplace_back("v7ajjeirttkbnt32wpy3c6w3emwnfr3fkla7hpxcfokr3ysd3kqtzmqd.onion:38333"); consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000000000000000008546553c03"); @@ -390,12 +392,12 @@ public: consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 150; consensus.BIP16Exception = uint256(); - consensus.BIP34Height = 2; // BIP34 activated on regtest (Block at height 1 not enforced for testing purposes) + consensus.BIP34Height = 1; // Always active unless overridden consensus.BIP34Hash = uint256(); - consensus.BIP65Height = 111; // BIP65 activated on regtest (Block at height 110 and earlier not enforced for testing purposes) - consensus.BIP66Height = 102; // BIP66 activated on regtest (Block at height 101 and earlier not enforced for testing purposes) - consensus.CSVHeight = 432; // CSV activated on regtest (Used in rpc activation tests) - consensus.SegwitHeight = 0; // SEGWIT is always activated on regtest unless overridden + consensus.BIP65Height = 1; // Always active unless overridden + consensus.BIP66Height = 1; // Always active unless overridden + consensus.CSVHeight = 1; // Always active unless overridden + consensus.SegwitHeight = 1; // Always active unless overridden consensus.MinBIP9WarningHeight = 0; consensus.powLimit = uint256S("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks @@ -487,15 +489,38 @@ public: void UpdateActivationParametersFromArgs(const ArgsManager& args); }; -void CRegTestParams::UpdateActivationParametersFromArgs(const ArgsManager& args) +static void MaybeUpdateHeights(const ArgsManager& args, Consensus::Params& consensus) { - if (args.IsArgSet("-segwitheight")) { - int64_t height = args.GetArg("-segwitheight", consensus.SegwitHeight); - if (height < 0 || height >= std::numeric_limits<int>::max()) { - throw std::runtime_error(strprintf("Activation height %ld for segwit is out of valid range.", height)); + for (const std::string& arg : args.GetArgs("-testactivationheight")) { + const auto found{arg.find('@')}; + if (found == std::string::npos) { + throw std::runtime_error(strprintf("Invalid format (%s) for -testactivationheight=name@height.", arg)); + } + const auto name{arg.substr(0, found)}; + const auto value{arg.substr(found + 1)}; + int32_t height; + if (!ParseInt32(value, &height) || height < 0 || height >= std::numeric_limits<int>::max()) { + throw std::runtime_error(strprintf("Invalid height value (%s) for -testactivationheight=name@height.", arg)); + } + if (name == "segwit") { + consensus.SegwitHeight = int{height}; + } else if (name == "bip34") { + consensus.BIP34Height = int{height}; + } else if (name == "dersig") { + consensus.BIP66Height = int{height}; + } else if (name == "cltv") { + consensus.BIP65Height = int{height}; + } else if (name == "csv") { + consensus.CSVHeight = int{height}; + } else { + throw std::runtime_error(strprintf("Invalid name (%s) for -testactivationheight=name@height.", arg)); } - consensus.SegwitHeight = static_cast<int>(height); } +} + +void CRegTestParams::UpdateActivationParametersFromArgs(const ArgsManager& args) +{ + MaybeUpdateHeights(args, consensus); if (!args.IsArgSet("-vbparams")) return; diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index 79c1bc25bc..f24863f419 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -20,12 +20,12 @@ void SetupChainParamsBaseOptions(ArgsManager& argsman) argsman.AddArg("-chain=<chain>", "Use the chain <chain> (default: main). Allowed values: main, test, signet, regtest", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); argsman.AddArg("-regtest", "Enter regression test mode, which uses a special chain in which blocks can be solved instantly. " "This is intended for regression testing tools and app development. Equivalent to -chain=regtest.", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CHAINPARAMS); - argsman.AddArg("-segwitheight=<n>", "Set the activation height of segwit. (regtest-only)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-testactivationheight=name@height.", "Set the activation height of 'name' (segwit, bip34, dersig, cltv, csv). (regtest-only)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-testnet", "Use the test chain. Equivalent to -chain=test.", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); argsman.AddArg("-vbparams=deployment:start:end[:min_activation_height]", "Use given start/end times and min_activation_height for specified version bits deployment (regtest-only)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CHAINPARAMS); argsman.AddArg("-signet", "Use the signet chain. Equivalent to -chain=signet. Note that the network is defined by the -signetchallenge parameter", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); - argsman.AddArg("-signetchallenge", "Blocks must satisfy the given script to be considered valid (only for signet networks; defaults to the global default signet test network challenge)", ArgsManager::ALLOW_STRING, OptionsCategory::CHAINPARAMS); - argsman.AddArg("-signetseednode", "Specify a seed node for the signet network, in the hostname[:port] format, e.g. sig.net:1234 (may be used multiple times to specify multiple seed nodes; defaults to the global default signet test network seed node(s))", ArgsManager::ALLOW_STRING, OptionsCategory::CHAINPARAMS); + argsman.AddArg("-signetchallenge", "Blocks must satisfy the given script to be considered valid (only for signet networks; defaults to the global default signet test network challenge)", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::CHAINPARAMS); + argsman.AddArg("-signetseednode", "Specify a seed node for the signet network, in the hostname[:port] format, e.g. sig.net:1234 (may be used multiple times to specify multiple seed nodes; defaults to the global default signet test network seed node(s))", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::CHAINPARAMS); } static std::unique_ptr<CBaseChainParams> globalChainBaseParams; diff --git a/src/checkqueue.h b/src/checkqueue.h index 4ceeb3600a..7c20e2013c 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -7,6 +7,7 @@ #include <sync.h> #include <tinyformat.h> +#include <util/syscall_sandbox.h> #include <util/threadnames.h> #include <algorithm> @@ -151,6 +152,7 @@ public: for (int n = 0; n < threads_num; ++n) { m_worker_threads.emplace_back([this, n]() { util::ThreadRename(strprintf("scriptch.%i", n)); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::VALIDATION_SCRIPT_CHECK); Loop(false /* worker thread */); }); } diff --git a/src/bloom.cpp b/src/common/bloom.cpp index d0128a26d7..26b70b4d14 100644 --- a/src/bloom.cpp +++ b/src/common/bloom.cpp @@ -2,22 +2,24 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <bloom.h> +#include <common/bloom.h> -#include <primitives/transaction.h> #include <hash.h> +#include <primitives/transaction.h> +#include <random.h> #include <script/script.h> #include <script/standard.h> -#include <random.h> +#include <span.h> #include <streams.h> -#include <math.h> -#include <stdlib.h> - #include <algorithm> +#include <cmath> +#include <cstdlib> +#include <limits> +#include <vector> -#define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 -#define LN2 0.6931471805599453094172321214581765680755001343602552 +static constexpr double LN2SQUARED = 0.4804530139182014246671025263266649717305529515945455; +static constexpr double LN2 = 0.6931471805599453094172321214581765680755001343602552; CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn, unsigned char nFlagsIn) : /** @@ -37,13 +39,13 @@ CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, c { } -inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const +inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, Span<const unsigned char> vDataToHash) const { // 0xFBA4C795 chosen as it guarantees a reasonable bit difference between nHashNum values. return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (vData.size() * 8); } -void CBloomFilter::insert(const std::vector<unsigned char>& vKey) +void CBloomFilter::insert(Span<const unsigned char> vKey) { if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700) return; @@ -59,17 +61,10 @@ void CBloomFilter::insert(const COutPoint& outpoint) { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << outpoint; - std::vector<unsigned char> data(stream.begin(), stream.end()); - insert(data); + insert(stream); } -void CBloomFilter::insert(const uint256& hash) -{ - std::vector<unsigned char> data(hash.begin(), hash.end()); - insert(data); -} - -bool CBloomFilter::contains(const std::vector<unsigned char>& vKey) const +bool CBloomFilter::contains(Span<const unsigned char> vKey) const { if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700) return true; @@ -87,14 +82,7 @@ bool CBloomFilter::contains(const COutPoint& outpoint) const { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << outpoint; - std::vector<unsigned char> data(stream.begin(), stream.end()); - return contains(data); -} - -bool CBloomFilter::contains(const uint256& hash) const -{ - std::vector<unsigned char> data(hash.begin(), hash.end()); - return contains(data); + return contains(stream); } bool CBloomFilter::IsWithinSizeConstraints() const @@ -198,7 +186,8 @@ CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const dou } /* Similar to CBloomFilter::Hash */ -static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak, const std::vector<unsigned char>& vDataToHash) { +static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak, Span<const unsigned char> vDataToHash) +{ return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash); } @@ -210,7 +199,7 @@ static inline uint32_t FastMod(uint32_t x, size_t n) { return ((uint64_t)x * (uint64_t)n) >> 32; } -void CRollingBloomFilter::insert(const std::vector<unsigned char>& vKey) +void CRollingBloomFilter::insert(Span<const unsigned char> vKey) { if (nEntriesThisGeneration == nEntriesPerGeneration) { nEntriesThisGeneration = 0; @@ -241,13 +230,7 @@ void CRollingBloomFilter::insert(const std::vector<unsigned char>& vKey) } } -void CRollingBloomFilter::insert(const uint256& hash) -{ - std::vector<unsigned char> vData(hash.begin(), hash.end()); - insert(vData); -} - -bool CRollingBloomFilter::contains(const std::vector<unsigned char>& vKey) const +bool CRollingBloomFilter::contains(Span<const unsigned char> vKey) const { for (int n = 0; n < nHashFuncs; n++) { uint32_t h = RollingBloomHash(n, nTweak, vKey); @@ -261,12 +244,6 @@ bool CRollingBloomFilter::contains(const std::vector<unsigned char>& vKey) const return true; } -bool CRollingBloomFilter::contains(const uint256& hash) const -{ - std::vector<unsigned char> vData(hash.begin(), hash.end()); - return contains(vData); -} - void CRollingBloomFilter::reset() { nTweak = GetRand(std::numeric_limits<unsigned int>::max()); diff --git a/src/bloom.h b/src/common/bloom.h index fdaa8abfb2..25c16fbfe2 100644 --- a/src/bloom.h +++ b/src/common/bloom.h @@ -2,20 +2,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_BLOOM_H -#define BITCOIN_BLOOM_H +#ifndef BITCOIN_COMMON_BLOOM_H +#define BITCOIN_COMMON_BLOOM_H #include <serialize.h> +#include <span.h> #include <vector> class COutPoint; class CTransaction; -class uint256; //! 20,000 items with fp rate < 0.1% or 10,000 items and <0.0001% -static const unsigned int MAX_BLOOM_FILTER_SIZE = 36000; // bytes -static const unsigned int MAX_HASH_FUNCS = 50; +static constexpr unsigned int MAX_BLOOM_FILTER_SIZE = 36000; // bytes +static constexpr unsigned int MAX_HASH_FUNCS = 50; /** * First two bits of nFlags control how much IsRelevantAndUpdate actually updates @@ -49,7 +49,7 @@ private: unsigned int nTweak; unsigned char nFlags; - unsigned int Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const; + unsigned int Hash(unsigned int nHashNum, Span<const unsigned char> vDataToHash) const; public: /** @@ -66,13 +66,11 @@ public: SERIALIZE_METHODS(CBloomFilter, obj) { READWRITE(obj.vData, obj.nHashFuncs, obj.nTweak, obj.nFlags); } - void insert(const std::vector<unsigned char>& vKey); + void insert(Span<const unsigned char> vKey); void insert(const COutPoint& outpoint); - void insert(const uint256& hash); - bool contains(const std::vector<unsigned char>& vKey) const; + bool contains(Span<const unsigned char> vKey) const; bool contains(const COutPoint& outpoint) const; - bool contains(const uint256& hash) const; //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) @@ -112,10 +110,8 @@ class CRollingBloomFilter public: CRollingBloomFilter(const unsigned int nElements, const double nFPRate); - void insert(const std::vector<unsigned char>& vKey); - void insert(const uint256& hash); - bool contains(const std::vector<unsigned char>& vKey) const; - bool contains(const uint256& hash) const; + void insert(Span<const unsigned char> vKey); + bool contains(Span<const unsigned char> vKey) const; void reset(); @@ -128,4 +124,4 @@ private: int nHashFuncs; }; -#endif // BITCOIN_BLOOM_H +#endif // BITCOIN_COMMON_BLOOM_H diff --git a/src/amount.h b/src/consensus/amount.h index 47968e80b1..96566ea13f 100644 --- a/src/amount.h +++ b/src/consensus/amount.h @@ -3,15 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_AMOUNT_H -#define BITCOIN_AMOUNT_H +#ifndef BITCOIN_CONSENSUS_AMOUNT_H +#define BITCOIN_CONSENSUS_AMOUNT_H -#include <stdint.h> +#include <cstdint> /** Amount in satoshis (Can be negative) */ typedef int64_t CAmount; -static const CAmount COIN = 100000000; +/** The amount of satoshis in one BTC. */ +static constexpr CAmount COIN = 100000000; /** No amount larger than this (in satoshi) is valid. * @@ -22,7 +23,7 @@ static const CAmount COIN = 100000000; * critical; in unusual circumstances like a(nother) overflow bug that allowed * for the creation of coins out of thin air modification could lead to a fork. * */ -static const CAmount MAX_MONEY = 21000000 * COIN; +static constexpr CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -#endif // BITCOIN_AMOUNT_H +#endif // BITCOIN_CONSENSUS_AMOUNT_H diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index bb8cd10c63..de4824fadc 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -4,6 +4,7 @@ #include <consensus/tx_check.h> +#include <consensus/amount.h> #include <primitives/transaction.h> #include <consensus/validation.h> diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 0ab790ccdc..a07adae536 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -4,6 +4,7 @@ #include <consensus/tx_verify.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <primitives/transaction.h> #include <script/interpreter.h> diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index 264433c33d..777556808a 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CONSENSUS_TX_VERIFY_H #define BITCOIN_CONSENSUS_TX_VERIFY_H -#include <amount.h> +#include <consensus/amount.h> #include <stdint.h> #include <vector> diff --git a/src/consensus/validation.h b/src/consensus/validation.h index c4d305434a..05416d0aca 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -53,6 +53,7 @@ enum class TxValidationResult { */ TX_CONFLICT, TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits + TX_NO_MEMPOOL, //!< this node does not have a mempool so can't validate the transaction }; /** A "reason" why a block was invalid, suitable for determining whether the diff --git a/src/core_io.h b/src/core_io.h index 3b9b66574c..4d7199ab12 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CORE_IO_H #define BITCOIN_CORE_IO_H -#include <amount.h> +#include <consensus/amount.h> #include <attributes.h> #include <string> @@ -20,6 +20,15 @@ class uint256; class UniValue; class CTxUndo; +/** + * Verbose level for block's transaction + */ +enum class TxVerbosity { + SHOW_TXID, //!< Only TXID for each block's transaction + SHOW_DETAILS, //!< Include TXID, inputs, outputs, and other common block's transaction information + SHOW_DETAILS_AND_PREVOUT //!< The same as previous option with information about prevouts if available +}; + // core_read.cpp CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); @@ -44,8 +53,8 @@ UniValue ValueFromAmount(const CAmount amount); std::string FormatScript(const CScript& script); std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); std::string SighashToStr(unsigned char sighash_type); -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex, bool include_addresses); -void ScriptToUniv(const CScript& script, UniValue& out, bool include_address); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_addresses, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); +void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address = true); +void ScriptToUniv(const CScript& script, UniValue& out); +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_read.cpp b/src/core_read.cpp index 6108961010..2149b428d2 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -26,20 +26,20 @@ opcodetype ParseOpCode(const std::string& s) { static std::map<std::string, opcodetype> mapOpNames; - if (mapOpNames.empty()) - { - for (unsigned int op = 0; op <= MAX_OPCODE; op++) - { + if (mapOpNames.empty()) { + for (unsigned int op = 0; op <= MAX_OPCODE; op++) { // Allow OP_RESERVED to get into mapOpNames - if (op < OP_NOP && op != OP_RESERVED) + if (op < OP_NOP && op != OP_RESERVED) { continue; + } std::string strName = GetOpName(static_cast<opcodetype>(op)); - if (strName == "OP_UNKNOWN") + if (strName == "OP_UNKNOWN") { continue; + } mapOpNames[strName] = static_cast<opcodetype>(op); // Convenience: OP_ADD and just ADD are both recognized: - if (strName.compare(0, 3, "OP_") == 0) { // strName starts with "OP_" + if (strName.compare(0, 3, "OP_") == 0) { // strName starts with "OP_" mapOpNames[strName.substr(3)] = static_cast<opcodetype>(op); } } @@ -59,44 +59,35 @@ CScript ParseScript(const std::string& s) std::vector<std::string> words; boost::algorithm::split(words, s, boost::algorithm::is_any_of(" \t\n"), boost::algorithm::token_compress_on); - for (std::vector<std::string>::const_iterator w = words.begin(); w != words.end(); ++w) - { - if (w->empty()) - { + for (const std::string& w : words) { + if (w.empty()) { // Empty string, ignore. (boost::split given '' will return one word) - } - else if (std::all_of(w->begin(), w->end(), ::IsDigit) || - (w->front() == '-' && w->size() > 1 && std::all_of(w->begin()+1, w->end(), ::IsDigit))) + } else if (std::all_of(w.begin(), w.end(), ::IsDigit) || + (w.front() == '-' && w.size() > 1 && std::all_of(w.begin() + 1, w.end(), ::IsDigit))) { // Number - int64_t n = atoi64(*w); + const auto num{ToIntegral<int64_t>(w)}; - //limit the range of numbers ParseScript accepts in decimal - //since numbers outside -0xFFFFFFFF...0xFFFFFFFF are illegal in scripts - if (n > int64_t{0xffffffff} || n < -1 * int64_t{0xffffffff}) { + // limit the range of numbers ParseScript accepts in decimal + // since numbers outside -0xFFFFFFFF...0xFFFFFFFF are illegal in scripts + if (!num.has_value() || num > int64_t{0xffffffff} || num < -1 * int64_t{0xffffffff}) { throw std::runtime_error("script parse error: decimal numeric value only allowed in the " "range -0xFFFFFFFF...0xFFFFFFFF"); } - result << n; - } - else if (w->substr(0,2) == "0x" && w->size() > 2 && IsHex(std::string(w->begin()+2, w->end()))) - { + result << num.value(); + } else if (w.substr(0, 2) == "0x" && w.size() > 2 && IsHex(std::string(w.begin() + 2, w.end()))) { // Raw hex data, inserted NOT pushed onto stack: - std::vector<unsigned char> raw = ParseHex(std::string(w->begin()+2, w->end())); + std::vector<unsigned char> raw = ParseHex(std::string(w.begin() + 2, w.end())); result.insert(result.end(), raw.begin(), raw.end()); - } - else if (w->size() >= 2 && w->front() == '\'' && w->back() == '\'') - { + } else if (w.size() >= 2 && w.front() == '\'' && w.back() == '\'') { // Single-quoted string, pushed as data. NOTE: this is poor-man's // parsing, spaces/tabs/newlines in single-quoted strings won't work. - std::vector<unsigned char> value(w->begin()+1, w->end()-1); + std::vector<unsigned char> value(w.begin() + 1, w.end() - 1); result << value; - } - else - { + } else { // opcode, e.g. OP_ADD or ADD: - result << ParseOpCode(*w); + result << ParseOpCode(w); } } diff --git a/src/core_write.cpp b/src/core_write.cpp index b35f835f42..468694b011 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -4,6 +4,7 @@ #include <core_io.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <key_io.h> @@ -141,56 +142,28 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) return HexStr(ssTx); } -void ScriptToUniv(const CScript& script, UniValue& out, bool include_address) +void ScriptToUniv(const CScript& script, UniValue& out) { - out.pushKV("asm", ScriptToAsmStr(script)); - out.pushKV("hex", HexStr(script)); - - std::vector<std::vector<unsigned char>> solns; - TxoutType type = Solver(script, solns); - out.pushKV("type", GetTxnOutputType(type)); - - CTxDestination address; - if (include_address && ExtractDestination(script, address) && type != TxoutType::PUBKEY) { - out.pushKV("address", EncodeDestination(address)); - } + ScriptPubKeyToUniv(script, out, /* include_hex */ true, /* include_address */ false); } -// TODO: from v23 ("addresses" and "reqSigs" deprecated) this method should be refactored to remove the `include_addresses` option -// this method can also be combined with `ScriptToUniv` as they will overlap -void ScriptPubKeyToUniv(const CScript& scriptPubKey, - UniValue& out, bool fIncludeHex, bool include_addresses) +void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address) { - TxoutType type; CTxDestination address; - std::vector<CTxDestination> addresses; - int nRequired; out.pushKV("asm", ScriptToAsmStr(scriptPubKey)); - if (fIncludeHex) - out.pushKV("hex", HexStr(scriptPubKey)); + if (include_hex) out.pushKV("hex", HexStr(scriptPubKey)); - if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired) || type == TxoutType::PUBKEY) { - out.pushKV("type", GetTxnOutputType(type)); - return; - } + std::vector<std::vector<unsigned char>> solns; + const TxoutType type{Solver(scriptPubKey, solns)}; - if (ExtractDestination(scriptPubKey, address)) { + if (include_address && ExtractDestination(scriptPubKey, address) && type != TxoutType::PUBKEY) { out.pushKV("address", EncodeDestination(address)); } out.pushKV("type", GetTxnOutputType(type)); - - if (include_addresses) { - UniValue a(UniValue::VARR); - for (const CTxDestination& addr : addresses) { - a.push_back(EncodeDestination(addr)); - } - out.pushKV("addresses", a); - out.pushKV("reqSigs", nRequired); - } } -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_addresses, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo) +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) { entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); @@ -206,7 +179,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_add // If available, use Undo data to calculate the fee. Note that txundo == nullptr // for coinbase transactions and for transactions where undo data is unavailable. - const bool calculate_fee = txundo != nullptr; + const bool have_undo = txundo != nullptr; CAmount amt_total_in = 0; CAmount amt_total_out = 0; @@ -230,9 +203,28 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_add } in.pushKV("txinwitness", txinwitness); } - if (calculate_fee) { - const CTxOut& prev_txout = txundo->vprevout[i].out; + if (have_undo) { + const Coin& prev_coin = txundo->vprevout[i]; + const CTxOut& prev_txout = prev_coin.out; + amt_total_in += prev_txout.nValue; + switch (verbosity) { + case TxVerbosity::SHOW_TXID: + case TxVerbosity::SHOW_DETAILS: + break; + + case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: + UniValue o_script_pub_key(UniValue::VOBJ); + ScriptPubKeyToUniv(prev_txout.scriptPubKey, o_script_pub_key, /* includeHex */ true); + + UniValue p(UniValue::VOBJ); + p.pushKV("generated", bool(prev_coin.fCoinBase)); + p.pushKV("height", uint64_t(prev_coin.nHeight)); + p.pushKV("value", ValueFromAmount(prev_txout.nValue)); + p.pushKV("scriptPubKey", o_script_pub_key); + in.pushKV("prevout", p); + break; + } } in.pushKV("sequence", (int64_t)txin.nSequence); vin.push_back(in); @@ -249,17 +241,17 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_add out.pushKV("n", (int64_t)i); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(txout.scriptPubKey, o, true, include_addresses); + ScriptPubKeyToUniv(txout.scriptPubKey, o, true); out.pushKV("scriptPubKey", o); vout.push_back(out); - if (calculate_fee) { + if (have_undo) { amt_total_out += txout.nValue; } } entry.pushKV("vout", vout); - if (calculate_fee) { + if (have_undo) { const CAmount fee = amt_total_in - amt_total_out; CHECK_NONFATAL(MoneyRange(fee)); entry.pushKV("fee", ValueFromAmount(fee)); diff --git a/src/crc32c/.travis.yml b/src/crc32c/.travis.yml index d990a89f07..183a5fba45 100644 --- a/src/crc32c/.travis.yml +++ b/src/crc32c/.travis.yml @@ -4,7 +4,7 @@ language: cpp dist: bionic -osx_image: xcode10.3 +osx_image: xcode12.5 compiler: - gcc @@ -24,20 +24,20 @@ env: addons: apt: sources: - - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main' + - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main' key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - sourceline: 'ppa:ubuntu-toolchain-r/test' packages: - - clang-9 + - clang-12 - cmake - - gcc-9 - - g++-9 + - gcc-11 + - g++-11 - ninja-build homebrew: packages: - cmake - - gcc@9 - - llvm@9 + - gcc@11 + - llvm@12 - ninja update: true @@ -48,14 +48,14 @@ install: export PATH="$(brew --prefix llvm)/bin:$PATH"; fi # /usr/bin/gcc points to an older compiler on both Linux and macOS. -- if [ "$CXX" = "g++" ]; then export CXX="g++-9" CC="gcc-9"; fi +- if [ "$CXX" = "g++" ]; then export CXX="g++-11" CC="gcc-11"; fi # /usr/bin/clang points to an older compiler on both Linux and macOS. # # Homebrew's llvm package doesn't ship a versioned clang++ binary, so the values # below don't work on macOS. Fortunately, the path change above makes the # default values (clang and clang++) resolve to the correct compiler on macOS. - if [ "$TRAVIS_OS_NAME" = "linux" ]; then - if [ "$CXX" = "clang++" ]; then export CXX="clang++-9" CC="clang-9"; fi; + if [ "$CXX" = "clang++" ]; then export CXX="clang++-12" CC="clang-12"; fi; fi - echo ${CC} - echo ${CXX} diff --git a/src/crc32c/.ycm_extra_conf.py b/src/crc32c/.ycm_extra_conf.py index 536aadcec8..62daa8a4ac 100644 --- a/src/crc32c/.ycm_extra_conf.py +++ b/src/crc32c/.ycm_extra_conf.py @@ -4,10 +4,10 @@ """YouCompleteMe configuration that interprets a .clang_complete file. This module implementes the YouCompleteMe configuration API documented at: -https://github.com/Valloric/ycmd#ycm_extra_confpy-specification +https://github.com/ycm-core/ycmd#ycm_extra_confpy-specification The implementation loads and processes a .clang_complete file, documented at: -https://github.com/Rip-Rip/clang_complete/blob/master/README.md +https://github.com/xavierd/clang_complete/blob/master/README.md """ import os diff --git a/src/crc32c/README.md b/src/crc32c/README.md index 0bd69f7f09..58ba38e611 100644 --- a/src/crc32c/README.md +++ b/src/crc32c/README.md @@ -65,7 +65,7 @@ apm install autocomplete-clang build build-cmake clang-format language-cmake \ If you don't mind more setup in return for more speed, replace `autocomplete-clang` and `linter-clang` with `you-complete-me`. This requires -[setting up ycmd](https://github.com/Valloric/ycmd#building). +[setting up ycmd](https://github.com/ycm-core/ycmd#building). ```bash apm install autocomplete-plus build build-cmake clang-format language-cmake \ diff --git a/src/crc32c/src/crc32c_arm64_check.h b/src/crc32c/src/crc32c_arm64_check.h index 62a07aba09..6b80f70037 100644 --- a/src/crc32c/src/crc32c_arm64_check.h +++ b/src/crc32c/src/crc32c_arm64_check.h @@ -40,7 +40,15 @@ inline bool CanUseArm64Crc32() { // From 'arch/arm64/include/uapi/asm/hwcap.h' in Linux kernel source code. constexpr unsigned long kHWCAP_PMULL = 1 << 4; constexpr unsigned long kHWCAP_CRC32 = 1 << 7; - unsigned long hwcap = (&getauxval != nullptr) ? getauxval(AT_HWCAP) : 0; + unsigned long hwcap = +#if HAVE_STRONG_GETAUXVAL + // Some compilers warn on (&getauxval != nullptr) in the block below. + getauxval(AT_HWCAP); +#elif HAVE_WEAK_GETAUXVAL + (&getauxval != nullptr) ? getauxval(AT_HWCAP) : 0; +#else +#error This is supposed to be nested inside a check for HAVE_*_GETAUXVAL. +#endif // HAVE_STRONG_GETAUXVAL return (hwcap & (kHWCAP_PMULL | kHWCAP_CRC32)) == (kHWCAP_PMULL | kHWCAP_CRC32); #elif defined(__APPLE__) diff --git a/src/crypto/chacha_poly_aead.h b/src/crypto/chacha_poly_aead.h index 0afe8fcc14..6a7998335d 100644 --- a/src/crypto/chacha_poly_aead.h +++ b/src/crypto/chacha_poly_aead.h @@ -117,8 +117,8 @@ static constexpr int AAD_PACKAGES_PER_ROUND = 21; /* 64 / 3 round down*/ class ChaCha20Poly1305AEAD { private: - ChaCha20 m_chacha_main; // payload and poly1305 key-derivation cipher instance - ChaCha20 m_chacha_header; // AAD cipher instance (encrypted length) + ChaCha20 m_chacha_header; // AAD cipher instance (encrypted length) and poly1305 key-derivation cipher instance + ChaCha20 m_chacha_main; // payload unsigned char m_aad_keystream_buffer[CHACHA20_ROUND_OUTPUT]; // aad keystream cache uint64_t m_cached_aad_seqnr; // aad keystream cache hint diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 3a1086bf4c..2fdc54464a 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -115,7 +115,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) } CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) - : m_name{path.stem().string()} + : m_name{fs::PathToString(path.stem())} { penv = nullptr; readoptions.verify_checksums = true; @@ -129,21 +129,21 @@ CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bo options.env = penv; } else { if (fWipe) { - LogPrintf("Wiping LevelDB in %s\n", path.string()); - leveldb::Status result = leveldb::DestroyDB(path.string(), options); + LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(path)); + leveldb::Status result = leveldb::DestroyDB(fs::PathToString(path), options); dbwrapper_private::HandleError(result); } TryCreateDirectories(path); - LogPrintf("Opening LevelDB in %s\n", path.string()); + LogPrintf("Opening LevelDB in %s\n", fs::PathToString(path)); } - leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); + leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(path), &pdb); dbwrapper_private::HandleError(status); LogPrintf("Opened LevelDB successfully\n"); if (gArgs.GetBoolArg("-forcecompactdb", false)) { - LogPrintf("Starting database compaction of %s\n", path.string()); + LogPrintf("Starting database compaction of %s\n", fs::PathToString(path)); pdb->CompactRange(nullptr, nullptr); - LogPrintf("Finished database compaction of %s\n", path.string()); + LogPrintf("Finished database compaction of %s\n", fs::PathToString(path)); } // The base-case obfuscation key, which is a noop. @@ -160,10 +160,10 @@ CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bo Write(OBFUSCATE_KEY_KEY, new_key); obfuscate_key = new_key; - LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), HexStr(obfuscate_key)); + LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key)); } - LogPrintf("Using obfuscation key for %s: %s\n", path.string(), HexStr(obfuscate_key)); + LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key)); } CDBWrapper::~CDBWrapper() @@ -197,13 +197,15 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) return true; } -size_t CDBWrapper::DynamicMemoryUsage() const { +size_t CDBWrapper::DynamicMemoryUsage() const +{ std::string memory; - if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) { + std::optional<size_t> parsed; + if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) { LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n"); return 0; } - return stoul(memory); + return parsed.value(); } // Prefixed with null character to avoid collisions with other keys diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 8caeb32627..7f6471740f 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -39,8 +39,6 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-maxtxfee=<amt>", "-mintxfee=<amt>", "-paytxfee=<amt>", - "-rescan", - "-salvagewallet", "-signer=<cmd>", "-spendzeroconfchange", "-txconfirmtarget=<n>", diff --git a/src/external_signer.cpp b/src/external_signer.cpp index d6388b759a..75070899c6 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -9,6 +9,7 @@ #include <util/system.h> #include <external_signer.h> +#include <algorithm> #include <stdexcept> #include <string> #include <vector> @@ -75,15 +76,14 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str ssTx << psbtx; // Check if signer fingerprint matches any input master key fingerprint - bool match = false; - for (unsigned int i = 0; i < psbtx.inputs.size(); ++i) { - const PSBTInput& input = psbtx.inputs[i]; + auto matches_signer_fingerprint = [&](const PSBTInput& input) { for (const auto& entry : input.hd_keypaths) { - if (m_fingerprint == strprintf("%08x", ReadBE32(entry.second.fingerprint))) match = true; + if (m_fingerprint == strprintf("%08x", ReadBE32(entry.second.fingerprint))) return true; } - } + return false; + }; - if (!match) { + if (!std::any_of(psbtx.inputs.begin(), psbtx.inputs.end(), matches_signer_fingerprint)) { error = "Signer fingerprint " + m_fingerprint + " does not match any of the inputs:\n" + EncodeBase64(ssTx.str()); return false; } diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 151f1a38f1..929808c7fa 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -41,11 +41,11 @@ FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) if (!file && !read_only) file = fsbridge::fopen(path, "wb+"); if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); + LogPrintf("Unable to open file %s\n", fs::PathToString(path)); return nullptr; } if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, fs::PathToString(path)); fclose(file); return nullptr; } diff --git a/src/fs.cpp b/src/fs.cpp index b9b3c46d8d..7a99444eef 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -16,6 +16,7 @@ #define NOMINMAX #endif #include <codecvt> +#include <limits> #include <windows.h> #endif @@ -24,7 +25,7 @@ namespace fsbridge { FILE *fopen(const fs::path& p, const char *mode) { #ifndef WIN32 - return ::fopen(p.string().c_str(), mode); + return ::fopen(p.c_str(), mode); #else std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t> utf8_cvt; return ::_wfopen(p.wstring().c_str(), utf8_cvt.from_bytes(mode).c_str()); @@ -46,7 +47,7 @@ static std::string GetErrorReason() FileLock::FileLock(const fs::path& file) { - fd = open(file.string().c_str(), O_RDWR); + fd = open(file.c_str(), O_RDWR); if (fd == -1) { reason = GetErrorReason(); } @@ -249,9 +250,9 @@ void ofstream::close() #else // __GLIBCXX__ #if BOOST_VERSION >= 107700 -static_assert(sizeof(*BOOST_FILESYSTEM_C_STR(fs::path())) == sizeof(wchar_t), +static_assert(sizeof(*BOOST_FILESYSTEM_C_STR(boost::filesystem::path())) == sizeof(wchar_t), #else -static_assert(sizeof(*fs::path().BOOST_FILESYSTEM_C_STR) == sizeof(wchar_t), +static_assert(sizeof(*boost::filesystem::path().BOOST_FILESYSTEM_C_STR) == sizeof(wchar_t), #endif // BOOST_VERSION >= 107700 "Warning: This build is using boost::filesystem ofstream and ifstream " "implementations which will fail to open paths containing multibyte " @@ -13,9 +13,132 @@ #include <boost/filesystem.hpp> #include <boost/filesystem/fstream.hpp> +#include <tinyformat.h> /** Filesystem operations and types */ -namespace fs = boost::filesystem; +namespace fs { + +using namespace boost::filesystem; + +/** + * Path class wrapper to prepare application code for transition from + * boost::filesystem library to std::filesystem implementation. The main + * purpose of the class is to define fs::path::u8string() and fs::u8path() + * functions not present in boost. It also blocks calls to the + * fs::path(std::string) implicit constructor and the fs::path::string() + * method, which worked well in the boost::filesystem implementation, but have + * unsafe and unpredictable behavior on Windows in the std::filesystem + * implementation (see implementation note in \ref PathToString for details). + */ +class path : public boost::filesystem::path +{ +public: + using boost::filesystem::path::path; + + // Allow path objects arguments for compatibility. + path(boost::filesystem::path path) : boost::filesystem::path::path(std::move(path)) {} + path& operator=(boost::filesystem::path path) { boost::filesystem::path::operator=(std::move(path)); return *this; } + path& operator/=(boost::filesystem::path path) { boost::filesystem::path::operator/=(std::move(path)); return *this; } + + // Allow literal string arguments, which are safe as long as the literals are ASCII. + path(const char* c) : boost::filesystem::path(c) {} + path& operator=(const char* c) { boost::filesystem::path::operator=(c); return *this; } + path& operator/=(const char* c) { boost::filesystem::path::operator/=(c); return *this; } + path& append(const char* c) { boost::filesystem::path::append(c); return *this; } + + // Disallow std::string arguments to avoid locale-dependent decoding on windows. + path(std::string) = delete; + path& operator=(std::string) = delete; + path& operator/=(std::string) = delete; + path& append(std::string) = delete; + + // Disallow std::string conversion method to avoid locale-dependent encoding on windows. + std::string string() const = delete; + + // Define UTF-8 string conversion method not present in boost::filesystem but present in std::filesystem. + std::string u8string() const { return boost::filesystem::path::string(); } +}; + +// Define UTF-8 string conversion function not present in boost::filesystem but present in std::filesystem. +static inline path u8path(const std::string& string) +{ + return boost::filesystem::path(string); +} + +// Disallow implicit std::string conversion for system_complete to avoid +// locale-dependent encoding on windows. +static inline path system_complete(const path& p) +{ + return boost::filesystem::system_complete(p); +} + +// Disallow implicit std::string conversion for exists to avoid +// locale-dependent encoding on windows. +static inline bool exists(const path& p) +{ + return boost::filesystem::exists(p); +} + +// Allow explicit quoted stream I/O. +static inline auto quoted(const std::string& s) +{ + return boost::io::quoted(s, '&'); +} + +// Allow safe path append operations. +static inline path operator+(path p1, path p2) +{ + p1 += std::move(p2); + return p1; +} + +/** + * Convert path object to byte string. On POSIX, paths natively are byte + * strings so this is trivial. On Windows, paths natively are Unicode, so an + * encoding step is necessary. + * + * The inverse of \ref PathToString is \ref PathFromString. The strings + * returned and parsed by these functions can be used to call POSIX APIs, and + * for roundtrip conversion, logging, and debugging. But they are not + * guaranteed to be valid UTF-8, and are generally meant to be used internally, + * not externally. When communicating with external programs and libraries that + * require UTF-8, fs::path::u8string() and fs::u8path() methods can be used. + * For other applications, if support for non UTF-8 paths is required, or if + * higher-level JSON or XML or URI or C-style escapes are preferred, it may be + * also be appropriate to use different path encoding functions. + * + * Implementation note: On Windows, the std::filesystem::path(string) + * constructor and std::filesystem::path::string() method are not safe to use + * here, because these methods encode the path using C++'s narrow multibyte + * encoding, which on Windows corresponds to the current "code page", which is + * unpredictable and typically not able to represent all valid paths. So + * std::filesystem::path::u8string() and std::filesystem::u8path() functions + * are used instead on Windows. On POSIX, u8string/u8path functions are not + * safe to use because paths are not always valid UTF-8, so plain string + * methods which do not transform the path there are used. + */ +static inline std::string PathToString(const path& path) +{ +#ifdef WIN32 + return path.u8string(); +#else + static_assert(std::is_same<path::string_type, std::string>::value, "PathToString not implemented on this platform"); + return path.boost::filesystem::path::string(); +#endif +} + +/** + * Convert byte string to path object. Inverse of \ref PathToString. + */ +static inline path PathFromString(const std::string& string) +{ +#ifdef WIN32 + return u8path(string); +#else + return boost::filesystem::path(string); +#endif +} +} // namespace fs /** Bridge operations to C stdio */ namespace fsbridge { @@ -103,4 +226,11 @@ namespace fsbridge { #endif // WIN32 && __GLIBCXX__ }; +// Disallow path operator<< formatting in tinyformat to avoid locale-dependent +// encoding on windows. +namespace tinyformat { +template<> inline void formatValue(std::ostream&, const char*, const char*, int, const boost::filesystem::path&) = delete; +template<> inline void formatValue(std::ostream&, const char*, const char*, int, const fs::path&) = delete; +} // namespace tinyformat + #endif // BITCOIN_FS_H diff --git a/src/hash.cpp b/src/hash.cpp index 3465caa3a9..0e5bd975e4 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <hash.h> +#include <span.h> #include <crypto/common.h> #include <crypto/hmac_sha512.h> @@ -74,10 +75,7 @@ unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vData void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]) { unsigned char num[4]; - num[0] = (nChild >> 24) & 0xFF; - num[1] = (nChild >> 16) & 0xFF; - num[2] = (nChild >> 8) & 0xFF; - num[3] = (nChild >> 0) & 0xFF; + WriteBE32(num, nChild); CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output); } diff --git a/src/httpserver.cpp b/src/httpserver.cpp index fa0379f612..6e75e28596 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -12,6 +12,7 @@ #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> @@ -279,6 +280,7 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) static bool ThreadHTTP(struct event_base* base) { util::ThreadRename("http"); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER); LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() @@ -289,7 +291,7 @@ static bool ThreadHTTP(struct event_base* base) /** Bind HTTP server to specified addresses */ static bool HTTPBindAddresses(struct evhttp* http) { - uint16_t http_port{static_cast<uint16_t>(gArgs.GetArg("-rpcport", BaseParams().RPCPort()))}; + uint16_t http_port{static_cast<uint16_t>(gArgs.GetIntArg("-rpcport", BaseParams().RPCPort()))}; std::vector<std::pair<std::string, uint16_t>> endpoints; // Determine what addresses to bind to @@ -332,6 +334,7 @@ static bool HTTPBindAddresses(struct evhttp* http) static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue, int worker_num) { util::ThreadRename(strprintf("httpworker.%i", worker_num)); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER_WORKER); queue->Run(); } @@ -374,7 +377,7 @@ bool InitHTTPServer() return false; } - evhttp_set_timeout(http, gArgs.GetArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); + evhttp_set_timeout(http, gArgs.GetIntArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); evhttp_set_max_headers_size(http, MAX_HEADERS_SIZE); evhttp_set_max_body_size(http, MAX_SIZE); evhttp_set_gencb(http, http_request_cb, nullptr); @@ -385,7 +388,7 @@ bool InitHTTPServer() } LogPrint(BCLog::HTTP, "Initialized HTTP server\n"); - int workQueueDepth = std::max((long)gArgs.GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); + int workQueueDepth = std::max((long)gArgs.GetIntArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); LogPrintf("HTTP: creating work queue of depth %d\n", workQueueDepth); g_work_queue = std::make_unique<WorkQueue<HTTPClosure>>(workQueueDepth); @@ -415,7 +418,7 @@ static std::vector<std::thread> g_thread_http_workers; void StartHTTPServer() { LogPrint(BCLog::HTTP, "Starting HTTP server\n"); - int rpcThreads = std::max((long)gArgs.GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); + int rpcThreads = std::max((long)gArgs.GetIntArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); g_thread_http = std::thread(ThreadHTTP, eventBase); diff --git a/src/i2p.cpp b/src/i2p.cpp index 5e7e42fb77..35ac8731f2 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -328,7 +328,7 @@ void Session::GenerateAndSavePrivateKey(const Sock& sock) if (!WriteBinaryFile(m_private_key_file, std::string(m_private_key.begin(), m_private_key.end()))) { throw std::runtime_error( - strprintf("Cannot save I2P private key to %s", m_private_key_file)); + strprintf("Cannot save I2P private key to %s", fs::quoted(fs::PathToString(m_private_key_file)))); } } diff --git a/src/index/base.cpp b/src/index/base.cpp index 6fd2701e2e..fc6dd77a72 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -8,6 +8,7 @@ #include <node/ui_interface.h> #include <shutdown.h> #include <tinyformat.h> +#include <util/syscall_sandbox.h> #include <util/thread.h> #include <util/translation.h> #include <validation.h> // For g_chainman @@ -123,6 +124,7 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& void BaseIndex::ThreadSync() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::TX_INDEX); const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { auto& consensus_params = Params().GetConsensus(); @@ -319,7 +321,7 @@ bool BaseIndex::BlockUntilSyncedToCurrentChain() const { // Skip the queue-draining stuff if we know we're caught up with - // ::ChainActive().Tip(). + // m_chain.Tip(). LOCK(cs_main); const CBlockIndex* chain_tip = m_chainstate->m_chain.Tip(); const CBlockIndex* best_block_index = m_best_block_index.load(); diff --git a/src/init.cpp b/src/init.cpp index 017c598ad5..f659de3a02 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -10,12 +10,12 @@ #include <init.h> #include <addrman.h> -#include <amount.h> #include <banman.h> #include <blockfilter.h> #include <chain.h> #include <chainparams.h> #include <compat/sanity.h> +#include <consensus/amount.h> #include <deploymentstatus.h> #include <fs.h> #include <hash.h> @@ -60,6 +60,7 @@ #include <util/check.h> #include <util/moneystr.h> #include <util/string.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> @@ -112,7 +113,7 @@ static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; static fs::path GetPidFile(const ArgsManager& args) { - return AbsPathForConfigVal(fs::path(args.GetArg("-pid", BITCOIN_PID_FILENAME))); + return AbsPathForConfigVal(fs::PathFromString(args.GetArg("-pid", BITCOIN_PID_FILENAME))); } [[nodiscard]] static bool CreatePidFile(const ArgsManager& args) @@ -126,7 +127,7 @@ static fs::path GetPidFile(const ArgsManager& args) #endif return true; } else { - return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile(args).string(), std::strerror(errno))); + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), fs::PathToString(GetPidFile(args)), std::strerror(errno))); } } @@ -218,7 +219,7 @@ void Shutdown(NodeContext& node) node.banman.reset(); node.addrman.reset(); - if (node.mempool && node.mempool->IsLoaded() && node.args->GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + if (node.mempool && node.mempool->IsLoaded() && node.args->GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(*node.mempool); } @@ -397,7 +398,7 @@ void SetupServerArgs(ArgsManager& argsman) -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -coinstatsindex and -rescan. " + argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -coinstatsindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -421,13 +422,14 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bind=<addr>[:<port>][=onion]", strprintf("Bind to given address and always listen on it (default: 0.0.0.0). Use [host]:port notation for IPv6. Append =onion to tag any incoming connections to that address and port as incoming Tor connections (default: 127.0.0.1:%u=onion, testnet: 127.0.0.1:%u=onion, signet: 127.0.0.1:%u=onion, regtest: 127.0.0.1:%u=onion)", defaultBaseParams->OnionServiceTargetPort(), testnetBaseParams->OnionServiceTargetPort(), signetBaseParams->OnionServiceTargetPort(), regtestBaseParams->OnionServiceTargetPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-cjdnsreachable", "If set then this host is configured for CJDNS (connecting to fc00::/8 addresses would lead us to the CJDNS network) (default: 0)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-dns", strprintf("Allow DNS lookups for -addnode, -seednode and -connect (default: %u)", DEFAULT_NAME_LOOKUP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-dnsseed", strprintf("Query for peer addresses via DNS lookup, if low on addresses (default: %u unless -connect used)", DEFAULT_DNSSEED), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); + argsman.AddArg("-dnsseed", strprintf("Query for peer addresses via DNS lookup, if low on addresses (default: %u unless -connect used)", DEFAULT_DNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-externalip=<ip>", "Specify your own public address", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-fixedseeds", strprintf("Allow fixed seeds if DNS seeds don't provide peers (default: %u)", DEFAULT_FIXEDSEEDS), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); - argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); + argsman.AddArg("-fixedseeds", strprintf("Allow fixed seeds if DNS seeds don't provide peers (default: %u)", DEFAULT_FIXEDSEEDS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listen", "Accept connections from outside (default: 1 if no -proxy or -connect)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -437,7 +439,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2psam=<ip:port>", "I2P SAM proxy to reach I2P peers and accept I2P connections (default: none)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-i2pacceptincoming", "If set and -i2psam is also set then incoming I2P connections are accepted via the SAM proxy. If this is not set but -i2psam is set then only outgoing connections will be made to the I2P network. Ignored if -i2psam is not set. Listening for incoming I2P connections is done through the SAM proxy, not by binding to a local address and port (default: 1)", ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); + argsman.AddArg("-i2pacceptincoming", "If set and -i2psam is also set then incoming I2P connections are accepted via the SAM proxy. If this is not set but -i2psam is set then only outgoing connections will be made to the I2P network. Ignored if -i2psam is not set. Listening for incoming I2P connections is done through the SAM proxy, not by binding to a local address and port (default: 1)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (" + Join(GetNetworkNames(), ", ") + "). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks. Warning: if it is used with non-onion networks and the -onion or -proxy option is set, then outbound onion connections will still be made; use -noonion or -onion=0 to disable outbound onion connections in this case.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -446,7 +448,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); + argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -461,7 +463,7 @@ void SetupServerArgs(ArgsManager& argsman) hidden_args.emplace_back("-upnp"); #endif #ifdef USE_NATPMP - argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %s)", DEFAULT_NATPMP ? "1 when listening and no -proxy" : "0"), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); + argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %s)", DEFAULT_NATPMP ? "1 when listening and no -proxy" : "0"), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #else hidden_args.emplace_back("-natpmp"); #endif // USE_NATPMP @@ -513,7 +515,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_BOOL | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -550,18 +552,22 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-rpcthreads=<n>", strprintf("Set the number of threads to service RPC calls (default: %d)", DEFAULT_HTTP_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcuser=<user>", "Username for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); argsman.AddArg("-rpcwhitelist=<whitelist>", "Set a whitelist to filter incoming RPC calls for a specific user. The field <whitelist> comes in the format: <USERNAME>:<rpc 1>,<rpc 2>,...,<rpc n>. If multiple whitelists are set for a given user, they are set-intersected. See -rpcwhitelistdefault documentation for information on default whitelist behavior.", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - argsman.AddArg("-rpcwhitelistdefault", "Sets default behavior for rpc whitelisting. Unless rpcwhitelistdefault is set to 0, if any -rpcwhitelist is set, the rpc server acts as if all rpc users are subject to empty-unless-otherwise-specified whitelists. If rpcwhitelistdefault is set to 1 and no -rpcwhitelist is set, rpc server acts as if all rpc users are subject to empty whitelists.", ArgsManager::ALLOW_BOOL, OptionsCategory::RPC); + argsman.AddArg("-rpcwhitelistdefault", "Sets default behavior for rpc whitelisting. Unless rpcwhitelistdefault is set to 0, if any -rpcwhitelist is set, the rpc server acts as if all rpc users are subject to empty-unless-otherwise-specified whitelists. If rpcwhitelistdefault is set to 1 and no -rpcwhitelist is set, rpc server acts as if all rpc users are subject to empty whitelists.", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcworkqueue=<n>", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); argsman.AddArg("-server", "Accept command line and JSON-RPC commands", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); #if HAVE_DECL_FORK - argsman.AddArg("-daemon", strprintf("Run in the background as a daemon and accept commands (default: %d)", DEFAULT_DAEMON), ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); - argsman.AddArg("-daemonwait", strprintf("Wait for initialization to be finished before exiting. This implies -daemon (default: %d)", DEFAULT_DAEMONWAIT), ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); + argsman.AddArg("-daemon", strprintf("Run in the background as a daemon and accept commands (default: %d)", DEFAULT_DAEMON), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-daemonwait", strprintf("Wait for initialization to be finished before exiting. This implies -daemon (default: %d)", DEFAULT_DAEMONWAIT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #else hidden_args.emplace_back("-daemon"); hidden_args.emplace_back("-daemonwait"); #endif +#if defined(USE_SYSCALL_SANDBOX) + argsman.AddArg("-sandbox=<mode>", "Use the experimental syscall sandbox in the specified mode (-sandbox=log-and-abort or -sandbox=abort). Allow only expected syscalls to be used by bitcoind. Note that this is an experimental new feature that may cause bitcoind to exit or crash unexpectedly: use with caution. In the \"log-and-abort\" mode the invocation of an unexpected syscall results in a debug handler being invoked which will log the incident and terminate the program (without executing the unexpected syscall). In the \"abort\" mode the invocation of an unexpected syscall results in the entire process being killed immediately by the kernel without executing the unexpected syscall.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); +#endif // USE_SYSCALL_SANDBOX + // Add the hidden options argsman.AddHiddenArgs(hidden_args); } @@ -843,7 +849,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) } // if using block pruning, then disallow txindex and coinstatsindex - if (args.GetArg("-prune", 0)) { + if (args.GetIntArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) @@ -868,11 +874,9 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Make sure enough file descriptors are available int nBind = std::max(nUserBind, size_t(1)); - nUserMaxConnections = args.GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); + nUserMaxConnections = args.GetIntArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); - // Trim requested connection counts, to fit into system limitations - // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind + NUM_FDS_MESSAGE_CAPTURE); #ifdef USE_POLL @@ -880,6 +884,8 @@ bool AppInitParameterInteraction(const ArgsManager& args) #else int fd_max = FD_SETSIZE; #endif + // Trim requested connection counts, to fit into system limitations + // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE), 0); if (nFD < MIN_CORE_FILEDESCRIPTORS) return InitError(_("Not enough file descriptors available.")); @@ -915,8 +921,8 @@ bool AppInitParameterInteraction(const ArgsManager& args) } // mempool limits - int64_t nMempoolSizeMax = args.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t nMempoolSizeMin = args.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; + int64_t nMempoolSizeMax = args.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolSizeMin = args.GetIntArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; if (nMempoolSizeMax < 0 || nMempoolSizeMax < nMempoolSizeMin) return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); // incremental relay fee sets the minimum feerate increase necessary for BIP 125 replacement in the mempool @@ -930,7 +936,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) } // block pruning; get the amount of disk space (in MiB) to allot for block & undo files - int64_t nPruneArg = args.GetArg("-prune", 0); + int64_t nPruneArg = args.GetIntArg("-prune", 0); if (nPruneArg < 0) { return InitError(_("Prune cannot be configured with a negative value.")); } @@ -947,12 +953,12 @@ bool AppInitParameterInteraction(const ArgsManager& args) fPruneMode = true; } - nConnectTimeout = args.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); + nConnectTimeout = args.GetIntArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; } - peer_connect_timeout = args.GetArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); + peer_connect_timeout = args.GetIntArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); } @@ -992,32 +998,63 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (!chainparams.IsTestChain() && !fRequireStandard) { return InitError(strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.NetworkIDString())); } - nBytesPerSigOp = args.GetArg("-bytespersigop", nBytesPerSigOp); + nBytesPerSigOp = args.GetIntArg("-bytespersigop", nBytesPerSigOp); if (!g_wallet_init_interface.ParameterInteraction()) return false; fIsBareMultisigStd = args.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG); fAcceptDatacarrier = args.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); - nMaxDatacarrierBytes = args.GetArg("-datacarriersize", nMaxDatacarrierBytes); + nMaxDatacarrierBytes = args.GetIntArg("-datacarriersize", nMaxDatacarrierBytes); // Option to startup with mocktime set (used for regression testing): - SetMockTime(args.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op + SetMockTime(args.GetIntArg("-mocktime", 0)); // SetMockTime(0) is a no-op if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); - if (args.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) + if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) return InitError(Untranslated("rpcserialversion must be non-negative.")); - if (args.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) + if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - nMaxTipAge = args.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); + nMaxTipAge = args.GetIntArg("-maxtipage", DEFAULT_MAX_TIP_AGE); if (args.IsArgSet("-proxy") && args.GetArg("-proxy", "").empty()) { return InitError(_("No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.")); } +#if defined(USE_SYSCALL_SANDBOX) + if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { + const std::string sandbox_arg{args.GetArg("-sandbox", "")}; + bool log_syscall_violation_before_terminating{false}; + if (sandbox_arg == "log-and-abort") { + log_syscall_violation_before_terminating = true; + } else if (sandbox_arg == "abort") { + // log_syscall_violation_before_terminating is false by default. + } else { + return InitError(Untranslated("Unknown syscall sandbox mode (-sandbox=<mode>). Available modes are \"log-and-abort\" and \"abort\".")); + } + // execve(...) is not allowed by the syscall sandbox. + const std::vector<std::string> features_using_execve{ + "-alertnotify", + "-blocknotify", + "-signer", + "-startupnotify", + "-walletnotify", + }; + for (const std::string& feature_using_execve : features_using_execve) { + if (!args.GetArg(feature_using_execve, "").empty()) { + return InitError(Untranslated(strprintf("The experimental syscall sandbox feature (-sandbox=<mode>) is incompatible with %s (which uses execve).", feature_using_execve))); + } + } + if (!SetupSyscallSandbox(log_syscall_violation_before_terminating)) { + return InitError(Untranslated("Installation of the syscall sandbox failed.")); + } + LogPrintf("Experimental syscall sandbox enabled (-sandbox=%s): bitcoind will terminate if an unexpected (not allowlisted) syscall is invoked.\n", sandbox_arg); + } +#endif // USE_SYSCALL_SANDBOX + return true; } @@ -1026,10 +1063,10 @@ static bool LockDataDirectory(bool probeOnly) // Make sure only a single Bitcoin process is using the data directory. fs::path datadir = gArgs.GetDataDirNet(); if (!DirIsWritable(datadir)) { - return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions."), datadir.string())); + return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions."), fs::PathToString(datadir))); } if (!LockDirectory(datadir, ".lock", probeOnly)) { - return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), datadir.string(), PACKAGE_NAME)); + return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), fs::PathToString(datadir), PACKAGE_NAME)); } return true; } @@ -1085,18 +1122,18 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. - if (args.IsArgSet("-datadir") && !fs::path(args.GetArg("-datadir", "")).is_absolute()) { + if (args.IsArgSet("-datadir") && !fs::PathFromString(args.GetArg("-datadir", "")).is_absolute()) { LogPrintf("Warning: relative datadir option '%s' specified, which will be interpreted relative to the " /* Continued */ "current working directory '%s'. This is fragile, because if bitcoin is started in the future " "from a different location, it will be unable to locate the current data files. There could " "also be data loss if bitcoin is started while in a temporary directory.\n", - args.GetArg("-datadir", ""), fs::current_path().string()); + args.GetArg("-datadir", ""), fs::PathToString(fs::current_path())); } InitSignatureCache(); InitScriptExecutionCache(); - int script_threads = args.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); + int script_threads = args.GetIntArg("-par", DEFAULT_SCRIPTCHECK_THREADS); if (script_threads <= 0) { // -par=0 means autodetect (number of cores - 1 script threads) // -par=-n means "leave n cores free" (number of cores - n - 1 script threads) @@ -1181,20 +1218,20 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Read asmap file if configured std::vector<bool> asmap; if (args.IsArgSet("-asmap")) { - fs::path asmap_path = fs::path(args.GetArg("-asmap", "")); + fs::path asmap_path = fs::PathFromString(args.GetArg("-asmap", "")); if (asmap_path.empty()) { - asmap_path = DEFAULT_ASMAP_FILENAME; + asmap_path = fs::PathFromString(DEFAULT_ASMAP_FILENAME); } if (!asmap_path.is_absolute()) { asmap_path = gArgs.GetDataDirNet() / asmap_path; } if (!fs::exists(asmap_path)) { - InitError(strprintf(_("Could not find asmap file %s"), asmap_path)); + InitError(strprintf(_("Could not find asmap file %s"), fs::quoted(fs::PathToString(asmap_path)))); return false; } asmap = DecodeAsmap(asmap_path); if (asmap.size() == 0) { - InitError(strprintf(_("Could not parse asmap file %s"), asmap_path)); + InitError(strprintf(_("Could not parse asmap file %s"), fs::quoted(fs::PathToString(asmap_path)))); return false; } const uint256 asmap_version = SerializeHash(asmap); @@ -1210,7 +1247,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } assert(!node.banman); - node.banman = std::make_unique<BanMan>(gArgs.GetDataDirNet() / "banlist", &uiInterface, args.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); + node.banman = std::make_unique<BanMan>(gArgs.GetDataDirNet() / "banlist", &uiInterface, args.GetIntArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); assert(!node.connman); node.connman = std::make_unique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), *node.addrman, args.GetBoolArg("-networkactive", true)); @@ -1220,7 +1257,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(); assert(!node.mempool); - int check_ratio = std::min<int>(std::max<int>(args.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); + int check_ratio = std::min<int>(std::max<int>(args.GetIntArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); node.mempool = std::make_unique<CTxMemPool>(node.fee_estimator.get(), check_ratio); assert(!node.chainman); @@ -1260,6 +1297,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } + if (!args.IsArgSet("-cjdnsreachable")) { + SetReachable(NET_CJDNS, false); + } + // Now IsReachable(NET_CJDNS) is true if: + // 1. -cjdnsreachable is given and + // 2.1. -onlynet is not given or + // 2.2. -onlynet=cjdns is given + // Check for host lookup allowed before parsing any network related parameters fNameLookup = args.GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); @@ -1281,6 +1326,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) SetProxy(NET_IPV4, addrProxy); SetProxy(NET_IPV6, addrProxy); SetProxy(NET_ONION, addrProxy); + SetProxy(NET_CJDNS, addrProxy); SetNameProxy(addrProxy); SetReachable(NET_ONION, true); // by default, -proxy sets onion as reachable, unless -noonion later } @@ -1327,7 +1373,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); // cache size calculations - int64_t nTotalCache = (args.GetArg("-dbcache", nDefaultDbCache) << 20); + int64_t nTotalCache = (args.GetIntArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20); @@ -1345,7 +1391,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache - int64_t nMempoolSizeMax = args.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolSizeMax = args.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { @@ -1501,7 +1547,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) for (CChainState* chainstate : chainman.GetAll()) { if (!is_coinsview_empty(chainstate)) { uiInterface.InitMessage(_("Verifying blocks…").translated); - if (fHavePruned && args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { + if (fHavePruned && args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", MIN_BLOCKS_TO_KEEP); } @@ -1518,8 +1564,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!CVerifyDB().VerifyDB( *chainstate, chainparams, chainstate->CoinsDB(), - args.GetArg("-checklevel", DEFAULT_CHECKLEVEL), - args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { + args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL), + args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); failed_verification = true; break; @@ -1619,11 +1665,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 11: import blocks if (!CheckDiskSpace(gArgs.GetDataDirNet())) { - InitError(strprintf(_("Error: Disk space is low for %s"), gArgs.GetDataDirNet())); + InitError(strprintf(_("Error: Disk space is low for %s"), fs::quoted(fs::PathToString(gArgs.GetDataDirNet())))); return false; } if (!CheckDiskSpace(gArgs.GetBlocksDirPath())) { - InitError(strprintf(_("Error: Disk space is low for %s"), gArgs.GetBlocksDirPath())); + InitError(strprintf(_("Error: Disk space is low for %s"), fs::quoted(fs::PathToString(gArgs.GetBlocksDirPath())))); return false; } @@ -1651,7 +1697,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) std::vector<fs::path> vImportFiles; for (const std::string& strFile : args.GetArgs("-loadblock")) { - vImportFiles.push_back(strFile); + vImportFiles.push_back(fs::PathFromString(strFile)); } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { @@ -1711,11 +1757,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); connOptions.m_msgproc = node.peerman.get(); - connOptions.nSendBufferMaxSize = 1000 * args.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); - connOptions.nReceiveFloodSize = 1000 * args.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); + connOptions.nSendBufferMaxSize = 1000 * args.GetIntArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); + connOptions.nReceiveFloodSize = 1000 * args.GetIntArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); connOptions.m_added_nodes = args.GetArgs("-addnode"); - connOptions.nMaxOutboundLimit = 1024 * 1024 * args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET); + connOptions.nMaxOutboundLimit = 1024 * 1024 * args.GetIntArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET); connOptions.m_peer_connect_timeout = peer_connect_timeout; for (const std::string& bind_arg : args.GetArgs("-bind")) { diff --git a/src/init/bitcoin-gui.cpp b/src/init/bitcoin-gui.cpp new file mode 100644 index 0000000000..c549ed3cc0 --- /dev/null +++ b/src/init/bitcoin-gui.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/chain.h> +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/ipc.h> +#include <interfaces/node.h> +#include <interfaces/wallet.h> +#include <node/context.h> +#include <util/system.h> + +#include <memory> + +namespace init { +namespace { +const char* EXE_NAME = "bitcoin-gui"; + +class BitcoinGuiInit : public interfaces::Init +{ +public: + BitcoinGuiInit(const char* arg0) : m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this)) + { + m_node.args = &gArgs; + m_node.init = this; + } + std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } + std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr<interfaces::WalletClient> makeWalletClient(interfaces::Chain& chain) override + { + return MakeWalletClient(chain, *Assert(m_node.args)); + } + std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } + interfaces::Ipc* ipc() override { return m_ipc.get(); } + NodeContext m_node; + std::unique_ptr<interfaces::Ipc> m_ipc; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr<Init> MakeGuiInit(int argc, char* argv[]) +{ + return std::make_unique<init::BitcoinGuiInit>(argc > 0 ? argv[0] : ""); +} +} // namespace interfaces diff --git a/src/init/bitcoin-qt.cpp b/src/init/bitcoin-qt.cpp new file mode 100644 index 0000000000..d71177e885 --- /dev/null +++ b/src/init/bitcoin-qt.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/chain.h> +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/node.h> +#include <interfaces/wallet.h> +#include <node/context.h> +#include <util/system.h> + +#include <memory> + +namespace init { +namespace { +class BitcoinQtInit : public interfaces::Init +{ +public: + BitcoinQtInit() + { + m_node.args = &gArgs; + m_node.init = this; + } + std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } + std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr<interfaces::WalletClient> makeWalletClient(interfaces::Chain& chain) override + { + return MakeWalletClient(chain, *Assert(m_node.args)); + } + std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } + NodeContext m_node; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr<Init> MakeGuiInit(int argc, char* argv[]) +{ + return std::make_unique<init::BitcoinQtInit>(); +} +} // namespace interfaces diff --git a/src/init/bitcoin-wallet.cpp b/src/init/bitcoin-wallet.cpp new file mode 100644 index 0000000000..e9dcde72fe --- /dev/null +++ b/src/init/bitcoin-wallet.cpp @@ -0,0 +1,12 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/init.h> + +namespace interfaces { +std::unique_ptr<Init> MakeWalletInit(int argc, char* argv[], int& exit_status) +{ + return std::make_unique<Init>(); +} +} // namespace interfaces diff --git a/src/init/common.cpp b/src/init/common.cpp index 5c1f469081..8f9e0ebc87 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -81,7 +81,7 @@ void AddLoggingArgs(ArgsManager& argsman) void SetLoggingOptions(const ArgsManager& args) { LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); - LogInstance().m_file_path = AbsPathForConfigVal(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + LogInstance().m_file_path = AbsPathForConfigVal(fs::PathFromString(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE))); LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); @@ -128,24 +128,24 @@ bool StartLogging(const ArgsManager& args) } if (!LogInstance().StartLogging()) { return InitError(strprintf(Untranslated("Could not open debug log file %s"), - LogInstance().m_file_path.string())); + fs::PathToString(LogInstance().m_file_path))); } if (!LogInstance().m_log_timestamps) LogPrintf("Startup time: %s\n", FormatISO8601DateTime(GetTime())); - LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); - LogPrintf("Using data directory %s\n", gArgs.GetDataDirNet().string()); + LogPrintf("Default data directory %s\n", fs::PathToString(GetDefaultDataDir())); + LogPrintf("Using data directory %s\n", fs::PathToString(gArgs.GetDataDirNet())); // Only log conf file usage message if conf file actually exists. fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); if (fs::exists(config_file_path)) { - LogPrintf("Config file: %s\n", config_file_path.string()); + LogPrintf("Config file: %s\n", fs::PathToString(config_file_path)); } else if (args.IsArgSet("-conf")) { // Warn if no conf file exists at path provided by user - InitWarning(strprintf(_("The specified config file %s does not exist"), config_file_path.string())); + InitWarning(strprintf(_("The specified config file %s does not exist"), fs::PathToString(config_file_path))); } else { // Not categorizing as "Warning" because it's the default behavior - LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); + LogPrintf("Config file: %s (not found, skipping)\n", fs::PathToString(config_file_path)); } // Log the config arguments to debug.log diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index eceede3c8f..d4ceb517dd 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -177,7 +177,7 @@ public: std::string& err_string) = 0; //! Calculate mempool ancestor and descendant counts for the given transaction. - virtual void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) = 0; + virtual void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) = 0; //! Get the node's package limits. //! Currently only returns the ancestor and descendant count limits, but could be enhanced to @@ -289,7 +289,7 @@ public: virtual void requestMempoolTransactions(Notifications& notifications) = 0; //! Check if Taproot has activated - virtual bool isTaprootActive() const = 0; + virtual bool isTaprootActive() = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/node.h b/src/interfaces/node.h index ee941b4602..ea1cb5ed72 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -5,8 +5,7 @@ #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H -#include <amount.h> // For CAmount -#include <external_signer.h> +#include <consensus/amount.h> #include <net.h> // For NodeId #include <net_types.h> // For banmap_t #include <netaddress.h> // For Network @@ -50,6 +49,16 @@ struct BlockAndHeaderTipInfo double verification_progress; }; +//! External signer interface used by the GUI. +class ExternalSigner +{ +public: + virtual ~ExternalSigner() {}; + + //! Get signer display name + virtual std::string getName() = 0; +}; + //! Top-level interface for a bitcoin node (bitcoind process). class Node { @@ -111,8 +120,8 @@ public: //! Disconnect node by id. virtual bool disconnectById(NodeId id) = 0; - //! List external signers - virtual std::vector<ExternalSigner> externalSigners() = 0; + //! Return list of external signers (attached devices which can sign transactions). + virtual std::vector<std::unique_ptr<ExternalSigner>> listExternalSigners() = 0; //! Get total bytes recv. virtual int64_t getTotalBytesRecv() = 0; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index a85db04b8b..490563426c 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_INTERFACES_WALLET_H #define BITCOIN_INTERFACES_WALLET_H -#include <amount.h> // For CAmount +#include <consensus/amount.h> #include <interfaces/chain.h> // For ChainClient #include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) #include <script/standard.h> // For CTxDestination @@ -122,10 +122,10 @@ public: virtual bool displayAddress(const CTxDestination& dest) = 0; //! Lock coin. - virtual void lockCoin(const COutPoint& output) = 0; + virtual bool lockCoin(const COutPoint& output, const bool write_to_db) = 0; //! Unlock coin. - virtual void unlockCoin(const COutPoint& output) = 0; + virtual bool unlockCoin(const COutPoint& output) = 0; //! Return whether coin is locked. virtual bool isLockedCoin(const COutPoint& output) = 0; diff --git a/src/ipc/process.cpp b/src/ipc/process.cpp index 43ed1f1bae..9036b80c45 100644 --- a/src/ipc/process.cpp +++ b/src/ipc/process.cpp @@ -30,8 +30,8 @@ public: return mp::SpawnProcess(pid, [&](int fd) { fs::path path = argv0_path; path.remove_filename(); - path.append(new_exe_name); - return std::vector<std::string>{path.string(), "-ipcfd", strprintf("%i", fd)}; + path /= fs::PathFromString(new_exe_name); + return std::vector<std::string>{fs::PathToString(path), "-ipcfd", strprintf("%i", fd)}; }); } int waitSpawned(int pid) override { return mp::WaitProcess(pid); } diff --git a/src/key.cpp b/src/key.cpp index 40df248e02..7688254515 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -229,6 +229,12 @@ bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig, bool gr assert(ret); secp256k1_ecdsa_signature_serialize_der(secp256k1_context_sign, vchSig.data(), &nSigLen, &sig); vchSig.resize(nSigLen); + // Additional verification step to prevent using a potentially corrupted signature + secp256k1_pubkey pk; + ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pk, begin()); + assert(ret); + ret = secp256k1_ecdsa_verify(GetVerifyContext(), &sig, hash.begin(), &pk); + assert(ret); return true; } @@ -251,13 +257,21 @@ bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) return false; vchSig.resize(CPubKey::COMPACT_SIGNATURE_SIZE); int rec = -1; - secp256k1_ecdsa_recoverable_signature sig; - int ret = secp256k1_ecdsa_sign_recoverable(secp256k1_context_sign, &sig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, nullptr); + secp256k1_ecdsa_recoverable_signature rsig; + int ret = secp256k1_ecdsa_sign_recoverable(secp256k1_context_sign, &rsig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, nullptr); assert(ret); - ret = secp256k1_ecdsa_recoverable_signature_serialize_compact(secp256k1_context_sign, &vchSig[1], &rec, &sig); + ret = secp256k1_ecdsa_recoverable_signature_serialize_compact(secp256k1_context_sign, &vchSig[1], &rec, &rsig); assert(ret); assert(rec != -1); vchSig[0] = 27 + rec + (fCompressed ? 4 : 0); + // Additional verification step to prevent using a potentially corrupted signature + secp256k1_pubkey epk, rpk; + ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &epk, begin()); + assert(ret); + ret = secp256k1_ecdsa_recover(GetVerifyContext(), &rpk, &rsig, hash.begin()); + assert(ret); + ret = secp256k1_ec_pubkey_cmp(GetVerifyContext(), &epk, &rpk); + assert(ret == 0); return true; } @@ -275,6 +289,13 @@ bool CKey::SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint2 if (!secp256k1_keypair_xonly_tweak_add(GetVerifyContext(), &keypair, tweak.data())) return false; } bool ret = secp256k1_schnorrsig_sign(secp256k1_context_sign, sig.data(), hash.data(), &keypair, aux ? (unsigned char*)aux->data() : nullptr); + if (ret) { + // Additional verification step to prevent using a potentially corrupted signature + secp256k1_xonly_pubkey pubkey_verify; + ret = secp256k1_keypair_xonly_pub(GetVerifyContext(), &pubkey_verify, nullptr, &keypair); + ret &= secp256k1_schnorrsig_verify(GetVerifyContext(), sig.data(), hash.begin(), 32, &pubkey_verify); + } + if (!ret) memory_cleanse(sig.data(), sig.size()); memory_cleanse(&keypair, sizeof(keypair)); return ret; } @@ -319,10 +340,11 @@ bool CExtKey::Derive(CExtKey &out, unsigned int _nChild) const { return key.Derive(out.key, out.chaincode, _nChild, chaincode); } -void CExtKey::SetSeed(const unsigned char *seed, unsigned int nSeedLen) { +void CExtKey::SetSeed(Span<const uint8_t> seed) +{ static const unsigned char hashkey[] = {'B','i','t','c','o','i','n',' ','s','e','e','d'}; std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); - CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(vout.data()); + CHMAC_SHA512{hashkey, sizeof(hashkey)}.Write(seed.data(), seed.size()).Finalize(vout.data()); key.Set(vout.data(), vout.data() + 32, true); memcpy(chaincode.begin(), vout.data() + 32, 32); nDepth = 0; @@ -343,8 +365,7 @@ CExtPubKey CExtKey::Neuter() const { void CExtKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); - code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; - code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; + WriteBE32(code+5, nChild); memcpy(code+9, chaincode.begin(), 32); code[41] = 0; assert(key.size() == 32); @@ -354,7 +375,7 @@ void CExtKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { void CExtKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); - nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; + nChild = ReadBE32(code+5); memcpy(chaincode.begin(), code+9, 32); key.Set(code+42, code+BIP32_EXTKEY_SIZE, true); if ((nDepth == 0 && (nChild != 0 || ReadLE32(vchFingerprint) != 0)) || code[41] != 0) key = CKey(); @@ -85,6 +85,7 @@ public: //! Simple read-only vector-like interface. unsigned int size() const { return (fValid ? keydata.size() : 0); } + const unsigned char* data() const { return keydata.data(); } const unsigned char* begin() const { return keydata.data(); } const unsigned char* end() const { return keydata.data() + size(); } @@ -177,7 +178,7 @@ struct CExtKey { void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtKey& out, unsigned int nChild) const; CExtPubKey Neuter() const; - void SetSeed(const unsigned char* seed, unsigned int nSeedLen); + void SetSeed(Span<const uint8_t> seed); }; /** Initialize the elliptic curve support. May not be called twice without calling ECC_Stop first. */ diff --git a/src/logging.cpp b/src/logging.cpp index eb2c750296..1efce21bdb 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -160,6 +160,8 @@ const CLogCategoryDesc LogCategories[] = {BCLog::I2P, "i2p"}, {BCLog::IPC, "ipc"}, {BCLog::LOCK, "lock"}, + {BCLog::UTIL, "util"}, + {BCLog::BLOCKSTORE, "blockstorage"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; diff --git a/src/logging.h b/src/logging.h index 53a89d28bd..f46104364c 100644 --- a/src/logging.h +++ b/src/logging.h @@ -60,6 +60,8 @@ namespace BCLog { I2P = (1 << 22), IPC = (1 << 23), LOCK = (1 << 24), + UTIL = (1 << 25), + BLOCKSTORE = (1 << 26), ALL = ~(uint32_t)0, }; diff --git a/src/logging/timer.h b/src/logging/timer.h index 79627b1fe3..b77a0e17c7 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -27,10 +27,12 @@ public: Timer( std::string prefix, std::string end_msg, - BCLog::LogFlags log_category = BCLog::LogFlags::ALL) : + BCLog::LogFlags log_category = BCLog::LogFlags::ALL, + bool msg_on_completion = true) : m_prefix(std::move(prefix)), m_title(std::move(end_msg)), - m_log_category(log_category) + m_log_category(log_category), + m_message_on_completion(msg_on_completion) { this->Log(strprintf("%s started", m_title)); m_start_t = GetTime<std::chrono::microseconds>(); @@ -38,7 +40,11 @@ public: ~Timer() { - this->Log(strprintf("%s completed", m_title)); + if (m_message_on_completion) { + this->Log(strprintf("%s completed", m_title)); + } else { + this->Log("completed"); + } } void Log(const std::string& msg) @@ -74,14 +80,17 @@ private: std::chrono::microseconds m_start_t{}; //! Log prefix; usually the name of the function this was created in. - const std::string m_prefix{}; + const std::string m_prefix; //! A descriptive message of what is being timed. - const std::string m_title{}; + const std::string m_title; //! Forwarded on to LogPrint if specified - has the effect of only //! outputting the timing log when a particular debug= category is specified. - const BCLog::LogFlags m_log_category{}; + const BCLog::LogFlags m_log_category; + + //! Whether to output the message again on completion. + const bool m_message_on_completion; }; } // namespace BCLog @@ -91,6 +100,8 @@ private: BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) #define LOG_TIME_MILLIS_WITH_CATEGORY(end_msg, log_category) \ BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) +#define LOG_TIME_MILLIS_WITH_CATEGORY_MSG_ONCE(end_msg, log_category) \ + BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category, /* msg_on_completion=*/false) #define LOG_TIME_SECONDS(end_msg) \ BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg) diff --git a/src/mapport.cpp b/src/mapport.cpp index 135efb561e..a2d06c68b4 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -14,6 +14,7 @@ #include <netaddress.h> #include <netbase.h> #include <threadinterrupt.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> @@ -222,6 +223,7 @@ static bool ProcessUpnp() static void ThreadMapPort() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_MAP_PORT); bool ok; do { ok = false; diff --git a/src/merkleblock.h b/src/merkleblock.h index 0e4ed72130..70749b6378 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_MERKLEBLOCK_H #define BITCOIN_MERKLEBLOCK_H +#include <common/bloom.h> +#include <primitives/block.h> #include <serialize.h> #include <uint256.h> -#include <primitives/block.h> -#include <bloom.h> #include <vector> diff --git a/src/miner.cpp b/src/miner.cpp index 168ade5507..1ef246cd14 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -5,10 +5,10 @@ #include <miner.h> -#include <amount.h> #include <chain.h> #include <chainparams.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_verify.h> @@ -72,7 +72,7 @@ static BlockAssembler::Options DefaultOptions() // Block resource limits // If -blockmaxweight is not given, limit to DEFAULT_BLOCK_MAX_WEIGHT BlockAssembler::Options options; - options.nBlockMaxWeight = gArgs.GetArg("-blockmaxweight", DEFAULT_BLOCK_MAX_WEIGHT); + options.nBlockMaxWeight = gArgs.GetIntArg("-blockmaxweight", DEFAULT_BLOCK_MAX_WEIGHT); if (gArgs.IsArgSet("-blockmintxfee")) { std::optional<CAmount> parsed = ParseMoney(gArgs.GetArg("-blockmintxfee", "")); options.blockMinFeeRate = CFeeRate{parsed.value_or(DEFAULT_BLOCK_MIN_TX_FEE)}; @@ -125,7 +125,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc // -regtest only: allow overriding block.nVersion with // -blockversion=N to test forking scenarios if (chainparams.MineBlocksOnDemand()) - pblock->nVersion = gArgs.GetArg("-blockversion", pblock->nVersion); + pblock->nVersion = gArgs.GetIntArg("-blockversion", pblock->nVersion); pblock->nTime = GetAdjustedTime(); const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast(); @@ -237,7 +237,7 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) bool fPrintPriority = gArgs.GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); if (fPrintPriority) { - LogPrintf("fee %s txid %s\n", + LogPrintf("fee rate %s txid %s\n", CFeeRate(iter->GetModifiedFee(), iter->GetTxSize()).ToString(), iter->GetTx().GetHash().ToString()); } diff --git a/src/minisketch/.cirrus.yml b/src/minisketch/.cirrus.yml new file mode 100644 index 0000000000..4a5353f137 --- /dev/null +++ b/src/minisketch/.cirrus.yml @@ -0,0 +1,154 @@ +env: + BUILD: check + HOST: + MAKEFLAGS: -j4 + BENCH: yes + TESTRUNS: + EXEC_CMD: + ENABLE_FIELDS: + +cat_logs_snippet: &CAT_LOGS + on_failure: + cat_test_log_script: + - cat test-suite.log || true + cat_config_log_script: + - cat config.log || true + cat_test_env_script: + - cat test_env.log || true + cat_ci_env_script: + - env + +merge_base_script_snippet: &MERGE_BASE + merge_base_script: + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + +env_matrix_snippet: &ENV_MATRIX_VALGRIND + - env: + ENABLE_FIELDS: "7,32,58" + - env: + BUILD: distcheck + - env: + EXEC_CMD: valgrind --error-exitcode=42 + TESTRUNS: 1 + BUILD: + +env_matrix_snippet: &ENV_MATRIX_SAN + - env: + ENABLE_FIELDS: 28 + - env: + BUILD: distcheck + - env: + CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" + BENCH: no + +env_matrix_snippet: &ENV_MATRIX_SAN_VALGRIND + - env: + ENABLE_FIELDS: "11,64,37" + - env: + BUILD: distcheck + - env: + EXEC_CMD: valgrind --error-exitcode=42 + TESTRUNS: 1 + BUILD: + - env: + CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" + BENCH: no + +task: + name: "x86_64: Linux (Debian stable)" + container: + dockerfile: ci/linux-debian.Dockerfile + memory: 2G + cpu: 4 + matrix: + << : *ENV_MATRIX_SAN_VALGRIND + matrix: + - env: + CC: gcc + - env: + CC: clang + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "i686: Linux (Debian stable)" + container: + dockerfile: ci/linux-debian.Dockerfile + memory: 2G + cpu: 4 + env: + HOST: i686-linux-gnu + matrix: + << : *ENV_MATRIX_VALGRIND + matrix: + - env: + CC: i686-linux-gnu-gcc + - env: + CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "x86_64: macOS Catalina" + macos_instance: + image: catalina-base + env: + # Cirrus gives us a fixed number of 12 virtual CPUs. + MAKEFLAGS: -j13 + matrix: + << : *ENV_MATRIX_SAN + matrix: + - env: + CC: gcc-9 + - env: + CC: clang + brew_script: + - brew update + - brew install automake libtool gcc@9 + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "s390x (big-endian): Linux (Debian stable, QEMU)" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 4 + memory: 2G + env: + EXEC_CMD: qemu-s390x -L /usr/s390x-linux-gnu + HOST: s390x-linux-gnu + BUILD: + << : *MERGE_BASE + test_script: + # https://sourceware.org/bugzilla/show_bug.cgi?id=27008 + - rm /etc/ld.so.cache + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "x86_64-w64-mingw32: Linux (Debian stable, Wine)" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 4 + memory: 2G + env: + EXEC_CMD: wine + HOST: x86_64-w64-mingw32 + BUILD: + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS diff --git a/src/minisketch/.gitignore b/src/minisketch/.gitignore new file mode 100644 index 0000000000..f5be6fab88 --- /dev/null +++ b/src/minisketch/.gitignore @@ -0,0 +1,35 @@ +*.o +*.lo +*.la +*.dll +*.dylib +*.so.* +.* +*.a +*~ + +Makefile +Makefile.in +aclocal.m4 +autom4te.cache/ +build-aux/config.guess +build-aux/config.sub +build-aux/depcomp +build-aux/install-sh +build-aux/ltmain.sh +build-aux/m4/libtool.m4 +build-aux/m4/lt~obsolete.m4 +build-aux/m4/ltoptions.m4 +build-aux/m4/ltsugar.m4 +build-aux/m4/ltversion.m4 +build-aux/missing +build-aux/compile +build-aux/test-driver +config.log +config.status +configure +libtool +stamp-h1 + +test* +bench diff --git a/src/minisketch/LICENSE b/src/minisketch/LICENSE new file mode 100644 index 0000000000..b25e3caee3 --- /dev/null +++ b/src/minisketch/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/minisketch/Makefile.am b/src/minisketch/Makefile.am new file mode 100644 index 0000000000..86a5c9dc90 --- /dev/null +++ b/src/minisketch/Makefile.am @@ -0,0 +1,92 @@ +ACLOCAL_AMFLAGS = -I build-aux/m4 +AM_CXXFLAGS = $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) + +include sources.mk + +include_HEADERS = $(MINISKETCH_DIST_HEADERS_INT) +noinst_HEADERS = $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT) + +LIBMINISKETCH = libminisketch.la +LIBMINISKETCH_FIELD_GENERIC = libminisketch_field_generic.la +if ENABLE_CLMUL +LIBMINISKETCH_FIELD_CLMUL = libminisketch_field_clmul.la +endif +if USE_TESTS +LIBMINISKETCH_VERIFY=libminisketch_verify.la +LIBMINISKETCH_FIELD_GENERIC_VERIFY=libminisketch_field_generic_verify.la +if ENABLE_CLMUL +LIBMINISKETCH_FIELD_CLMUL_VERIFY=libminisketch_field_clmul_verify.la +endif +endif + +lib_LTLIBRARIES = +lib_LTLIBRARIES += $(LIBMINISKETCH) + +noinst_LTLIBRARIES = +noinst_LTLIBRARIES += $(LIBMINISKETCH_FIELD_GENERIC) +noinst_LTLIBRARIES += $(LIBMINISKETCH_FIELD_GENERIC_VERIFY) +noinst_LTLIBRARIES += $(LIBMINISKETCH_FIELD_CLMUL) +noinst_LTLIBRARIES += $(LIBMINISKETCH_FIELD_CLMUL_VERIFY) +noinst_LTLIBRARIES += $(LIBMINISKETCH_VERIFY) + +# Release libs +libminisketch_field_generic_la_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) +libminisketch_field_generic_la_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) + +libminisketch_field_clmul_la_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) +libminisketch_field_clmul_la_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) +libminisketch_field_clmul_la_CXXFLAGS = $(AM_CXXFLAGS) $(CLMUL_CXXFLAGS) + +libminisketch_la_SOURCES = $(MINISKETCH_LIB_SOURCES_INT) +libminisketch_la_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) +libminisketch_la_LIBADD = $(LIBMINISKETCH_FIELD_CLMUL) $(LIBMINISKETCH_FIELD_GENERIC) + +# Libs with extra verification checks +libminisketch_field_generic_verify_la_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) +libminisketch_field_generic_verify_la_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES) + +libminisketch_field_clmul_verify_la_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) +libminisketch_field_clmul_verify_la_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES) +libminisketch_field_clmul_verify_la_CXXFLAGS = $(AM_CXXFLAGS) $(CLMUL_CXXFLAGS) + +libminisketch_verify_la_SOURCES = $(MINISKETCH_LIB_SOURCES_INT) +libminisketch_verify_la_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES) +libminisketch_verify_la_LIBADD = $(LIBMINISKETCH_FIELD_CLMUL_VERIFY) $(LIBMINISKETCH_FIELD_GENERIC_VERIFY) + +noinst_PROGRAMS = +if USE_BENCHMARK +noinst_PROGRAMS += bench +endif +if USE_TESTS +noinst_PROGRAMS += test test-verify +TESTS = test test-verify +endif + +bench_SOURCES = $(MINISKETCH_BENCH_SOURCES_INT) +bench_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) +bench_LDADD = $(LIBMINISKETCH) +bench_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + +test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) +test_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) +test_LDADD = $(LIBMINISKETCH) +test_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + +test_verify_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) +test_verify_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES) +test_verify_LDADD = $(LIBMINISKETCH_VERIFY) +test_verify_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + +EXTRA_DIST= +EXTRA_DIST += LICENSE +EXTRA_DIST += README.md +EXTRA_DIST += doc/example.c +EXTRA_DIST += doc/gen_params.sage +EXTRA_DIST += doc/math.md +EXTRA_DIST += doc/moduli.md +EXTRA_DIST += doc/plot_bits.png +EXTRA_DIST += doc/plot_capacity.png +EXTRA_DIST += doc/plot_diff.png +EXTRA_DIST += doc/plot_size.png +EXTRA_DIST += doc/protocoltips.md +EXTRA_DIST += tests/pyminisketch.py diff --git a/src/minisketch/README.md b/src/minisketch/README.md new file mode 100644 index 0000000000..c0cfdc1623 --- /dev/null +++ b/src/minisketch/README.md @@ -0,0 +1,210 @@ +# Minisketch: a library for [BCH](https://en.wikipedia.org/wiki/BCH_code)-based set reconciliation +<img align="right" src="doc/minisketch-vs.png" /> + +`libminisketch` is an optimized standalone MIT-licensed library with C API for constructing and decoding *set sketches*, which can be used for compact set reconciliation and other applications. +It is an implementation of the PinSketch<sup>[[1]](#myfootnote1)</sup> algorithm. An explanation of the algorithm can be found [here](doc/math.md). + +## Sketches for set reconciliation + +Sketches, as produced by this library, can be seen as "set checksums" with two peculiar properties: +* Sketches have a predetermined capacity, and when the number of elements in the set is not higher than the capacity, `libminisketch` will always recover the entire set from the sketch. A sketch of *b*-bit elements with capacity *c* can be stored in *bc* bits. +* The sketches of two sets can be combined by adding them (XOR) to obtain a sketch of the [symmetric difference](https://en.wikipedia.org/wiki/Symmetric_difference) between the two sets (*i.e.*, all elements that occur in one but not both input sets). + +This makes them appropriate for a very bandwidth-efficient set reconciliation protocol. If Alice and Bob each have a set of elements, and they suspect that the sets largely but not entirely overlap, +they can use the following protocol to let both parties learn all the elements: +* Alice and Bob both compute a sketch of their set elements. +* Alice sends her sketch to Bob. +* Bob combines the two sketches, and obtains a sketch of the symmetric difference. +* Bob tries to recover the elements from the difference sketch. +* Bob sends every element in the difference that he has to Alice. + +This will always succeed when the size of the difference (elements that Alice has but Bob doesn't plus elements that Bob has but Alice doesn't) does not exceed the +capacity of the sketch that Alice sent. The interesting part is that this works regardless of the actual set sizes—only the difference matters. + +If the elements are large, it may be preferable to compute the sketches over *hashes* of the set elements. In that case an additional step is added to the protocol, where Bob also sends the hash +of every element he does not have to Alice, who responds with the requested elements. + +The doc/ directory has additional [tips for designing reconciliation protocols using libminisketch](doc/protocoltips.md). + +## Evaluation + +<img src="doc/plot_capacity.png" width="432" height="324" /> <img src="doc/plot_diff.png" width="432" height="324" /> + +<img src="doc/plot_size.png" width="432" height="324" /> <img src="doc/plot_bits.png" width="432" height="324" /> + +**The first graph** above shows a benchmark of `libminisketch` against three other set reconciliation algorithms/implementations. The benchmarks were performed using a single core on a system with an Intel Core i7-7820HQ CPU with clock speed locked at 2.4 GHz. The diagram shows the time needed for merging of two sketches and decoding the result. The creation of a sketch on the same machine takes around 5 ns per capacity and per set element. The other implementations are: +* [`pinsketch`](https://www.cs.bu.edu/~reyzin/code/fuzzy.html), the original PinSketch implementation. +* [`cpisync`](https://github.com/trachten/cpisync), a software project which implements a number of set reconciliation algorithms and protocols. The included benchmark analyzes the non-probabilistic version of the original CPISync algorithm<sup>[[5]](#myfootnote5)</sup> only. +* A high-performance custom IBLT implementation using 4 hash functions and 32-bit checksums. + +For the largest sizes currently of interest to the authors, such as a set of capacity 4096 with 1024 differences, `libminisketch` is forty-nine times faster than `pinsketch` and over eight thousand times faster than `cpisync`. `libminisketch` is fast enough on realistic set sizes for use on high-traffic network servers where computational resources are limited. + +Even where performance is latency-limited, small minisketches can be fast enough to improve performance. On the above i7-7820HQ, a set of 2500 30-bit entries with a difference of 20 elements can be communicated in less time with a minisketch than sending the raw set so long as the communications bandwidth is 1 gigabit per second or less; an eight-element difference can be communicated in better than one-fifth the time on a gigabit link. + +**The second graph** above shows the performance of the same algorithms on the same system, but this time keeping the capacity constant at 128, while varying the number of differences to reconcile between 1 and 128. It shows how `cpisync`'s reconciliation speed is mostly dependent on capacity, while `pinsketch`/`libminisketch` are more dependent on number of differences. + +**The third graph** above shows the size overhead of a typical IBLT scheme over the other algorithms (which are near-optimal bandwidth), for various levels of failure probability. IBLT takes tens of times the bandwidth of `libminisketch` sketches when the set difference size is small and the required failure rate is low. + +**The fourth graph** above shows the effect of the field size on speed in `libminisketch`. The three lines correspond to: +* CLMUL 64-bit: Intel Core i7-7820HQ system at 2.4 GHz +* Generic 64-bit: POWER9 CP9M06 system at 2.8 GHz (Talos II) +* Generic 32-bit: Cortex-A53 at 1.2 GHz (Raspberry Pi 3B) + +It shows how CLMUL implementations are faster for certain fields (specifically, field sizes for which an irreducible polynomial of the form *x<sup>b</sup> + x + 1* over *GF(2)* exists, and to a lesser extent, fields which are a multiple of 8 bits). It also shows how (for now) a significant performance drop exists for fields larger than 32 bits on 32-bit platforms. Note that the three lines are not at the same scale (the Raspberry Pi 3B is around 10x slower for 32-bit fields than the Core i7; the POWER9 is around 1.3x slower). + +Below we compare the PinSketch algorithm (which `libminisketch` is an implementation of) with other set reconciliation algorithms: + +| Algorithm | Sketch size | Decode success | Decoding complexity | Difference type | Secure sketch | +| ----------------------------------------------------- | ------------------------- | ---------------| ------------------- | --------------- | ------------- | +| CPISync<sup>[[2]](#myfootnote2)</sup> | *(b+1)c* | Always | *O(n<sup>3</sup>)* | Both | Yes | +| PinSketch<sup>[[1]](#myfootnote1)</sup> | *bc* | Always | *O(n<sup>2</sup>)* | Symmetric only | Yes | +| IBLT<sup>[[6]](#myfootnote1)[[7]](#myfootnote1)</sup> | *αbc* (see graph 3) | Probabilistic | *O(n)* | Depends | No | + +* **Sketch size:** This column shows the size in bits of a sketch designed for reconciling *c* different *b*-bit elements. PinSketch and CPISync have a near-optimal<sup>[[11]](#myfootnote11)</sup> communication overhead, which in practice means the sketch size is very close (or equal to) *bc* bits. That is the same size as would be needed to transfer the elements of the difference naively (which is remarkable, as the difference isn't even known by the sender). For IBLT there is an overhead factor *α*, which depends on various design parameters, but is often between *2* and *10*. +* **Decode success:** Whenever a sketch is designed with a capacity not lower than the actual difference size, CPISync and PinSketch guarantee that decoding of the difference will always succeed. IBLT always has a chance of failure, though that chance can be made arbitrarily small by increasing the communication overhead. +* **Decoding complexity:** The space savings achieved by near-optimal algorithms come at a cost in performance, as their asymptotic decode complexity is quadratic or cubic, while IBLT is linear. This means that using near-optimal algorithms can be too expensive for applications where the difference is sufficiently large. +* **Difference type:** PinSketch can only compute the symmetric difference from a merged sketch, while CPISync and IBLT can distinguish which side certain elements were missing on. When the decoder has access to one of the sets, this generally doesn't matter, as he can look up each of the elements in the symmetric difference with one of the sets. +* **Secure sketch:** Whether the sketch satisfies the definition of a secure sketch<sup>[[1]](#myfootnote1)</sup>, which implies a minimal amount about a set can be extracted from a sketch by anyone who does not know most of the elements already. This makes the algorithm appropriate for applications like fingerprint authentication. + +## Building + +The build system is very rudimentary for now, and [improvements](https://github.com/sipa/minisketch/pulls) are welcome. + +The following may work and produce a `libminisketch.a` file you can link against: + +```bash +git clone https://github.com/sipa/minisketch +cd minisketch +./autogen.sh && ./configure && make +``` + +## Usage + +In this section Alice and Bob are trying to find the difference between their sets. +Alice has the set *[3000 ... 3009]*, while Bob has *[3002 ... 3011]*. + +First, Alice creates a sketch: + +```c +#include <stdio.h> +#include <assert.h> +#include "../include/minisketch.h" +int main(void) { + + minisketch *sketch_a = minisketch_create(12, 0, 4); +``` + +The arguments are: +* The field size *b*, which specifies the size of the elements being reconciled. With a field size *b*, the supported range of set elements is the integers from *1* to *2<sup>b</sub>* *- 1*, inclusive. Note that elements cannot be zero. +* The implementation number. Implementation *0* is always supported, but more efficient algorithms may be available on some hardware. The serialized form of a sketch is independent of the implementation, so different implementations can interoperate. +* The capacity *c*, which specifies how many differences the resulting sketch can reconcile. + +Then Alice adds her elements to her sketch. Note that adding the same element a second time removes it again, as sketches have set semantics, not multiset semantics. + +```c + for (int i = 3000; i < 3010; ++i) { + minisketch_add_uint64(sketch_a, i); + } +``` + +The next step is serializing the sketch into a byte array: + +```c + size_t sersize = minisketch_serialized_size(sketch_a); + assert(sersize == 12 * 4 / 8); // 4 12-bit values is 6 bytes. + unsigned char *buffer_a = malloc(sersize); + minisketch_serialize(sketch_a, buffer_a); + minisketch_destroy(sketch_a); +``` + +The contents of the buffer can then be submitted to Bob, who can create his own sketch: + +```c + minisketch *sketch_b = minisketch_create(12, 0, 4); // Bob's own sketch + for (int i = 3002; i < 3012; ++i) { + minisketch_add_uint64(sketch_b, i); + } +``` + +After Bob receives Alice's serialized sketch, he can reconcile: + +```c + sketch_a = minisketch_create(12, 0, 4); // Alice's sketch + minisketch_deserialize(sketch_a, buffer_a); // Load Alice's sketch + free(buffer_a); + + // Merge the elements from sketch_a into sketch_b. The result is a sketch_b + // which contains all elements that occurred in Alice's or Bob's sets, but not + // in both. + minisketch_merge(sketch_b, sketch_a); + + uint64_t differences[4]; + ssize_t num_differences = minisketch_decode(sketch_b, 4, differences); + minisketch_destroy(sketch_a); + minisketch_destroy(sketch_b); + if (num_differences < 0) { + printf("More than 4 differences!\n"); + } else { + ssize_t i; + for (i = 0; i < num_differences; ++i) { + printf("%u is in only one of the two sets\n", (unsigned)differences[i]); + } + } +} +``` + +In this example Bob would see output such as: + +``` +$ gcc -std=c99 -Wall -Wextra -o example ./doc/example.c -Lsrc/ -lminisketch -lstdc++ && ./example +3000 is in only one of the two sets +3011 is in only one of the two sets +3001 is in only one of the two sets +3010 is in only one of the two sets +``` + +The order of the output is arbitrary and will differ on different runs of minisketch_decode(). + +## Applications + +Communications efficient set reconciliation has been proposed to optimize Bitcoin transaction distribution<sup>[[8]](#myfootnote8)</sup>, which would allow Bitcoin nodes to have many more peers while reducing bandwidth usage. It could also be used for Bitcoin block distribution<sup>[[9]](#myfootnote9)</sup>, particularly for very low bandwidth links such as satellite. A similar approach (CPISync) is used by PGP SKS keyservers to synchronize their databases efficiently. Secure sketches can also be used as helper data to reliably extract a consistent cryptographic key from fuzzy biometric data while leaking minimal information<sup>[[1]](#myfootnote1)</sup>. They can be combined with [dcnets](https://en.wikipedia.org/wiki/Dining_cryptographers_problem) to create cryptographic multiparty anonymous communication<sup>[[10]](#myfootnote10)</sup>. + +## Implementation notes + +`libminisketch` is written in C++11, but has a [C API](include/minisketch.h) for compatibility reasons. + +Specific algorithms and optimizations used: +* Finite field implementations: + * A generic implementation using C unsigned integer bit operations, and one using the [CLMUL instruction](https://en.wikipedia.org/wiki/CLMUL_instruction_set) where available. The latter has specializations for different classes of fields that permit optimizations (those with trinomial irreducible polynomials, and those whose size is a multiple of 8 bits). + * Precomputed tables for (repeated) squaring, and for solving equations of the form *x<sup>2</sup> + x = a*<sup>[[2]](#myfootnote2)</sup>. + * Inverses are computed using an [exponentiation ladder](https://en.wikipedia.org/w/index.php?title=Exponentiation_by_squaring&oldid=868883860)<sup>[[12]](#myfootnote12)</sup> on systems where multiplications are relatively fast, and using an [extended GCD algorithm](https://en.wikipedia.org/w/index.php?title=Extended_Euclidean_algorithm&oldid=865802511#Computing_multiplicative_inverses_in_modular_structures) otherwise. + * Repeated multiplications are accelerated using runtime precomputations on systems where multiplications are relatively slow. + * The serialization of field elements always represents them as bits that are coefficients of the lowest-weight (using lexicographic order as tie breaker) irreducible polynomials over *GF(2)* (see [this list](doc/moduli.md)), but for some implementations they are converted to a different representation internally. +* The sketch algorithms are specialized for each separate field implementation, permitting inlining and specific optimizations while avoiding dynamic allocations and branching costs. +* Decoding of sketches uses the [Berlekamp-Massey algorithm](https://en.wikipedia.org/w/index.php?title=Berlekamp%E2%80%93Massey_algorithm&oldid=870768940)<sup>[[3]](#myfootnote3)</sup> to compute the characteristic polynomial. +* Finding the roots of polynomials is done using the Berlekamp trace algorithm with explicit formula for quadratic polynomials<sup>[[4]](#myfootnote4)</sup>. The root finding is randomized to prevent adversarial inputs that intentionally trigger worst-case decode time. +* A (possibly) novel optimization combines a test for unique roots with the Berlekamp trace algorithm. + +Some improvements that are still TODO: +* Explicit formulas for the roots of polynomials of higher degree than 2 +* Subquadratic multiplication and modulus algorithms +* The [Half-GCD algorithm](http://mathworld.wolfram.com/Half-GCD.html) for faster GCDs +* An interface for incremental decoding: most of the computation in most failed decodes can be reused when attempting to decode a longer sketch of the same set +* Platform specific optimizations for platforms other than x86 +* Avoid using slow uint64_t for calculations on 32-bit hosts +* Optional IBLT / Hybrid and set entropy coder under the same interface + +## References + +* <a name="myfootnote1">[1]</a> Dodis, Ostrovsky, Reyzin and Smith. *Fuzzy Extractors: How to Generate Strong Keys from Biometrics and Other Noisy Data.* SIAM Journal on Computing, volume 38, number 1, pages 97-139, 2008). [[URL]](http://eprint.iacr.org/2003/235) [[PDF]](https://eprint.iacr.org/2003/235.pdf) +* <a name="myfootnote5">[5]</a> A. Trachtenberg, D. Starobinski and S. Agarwal. *Fast PDA synchronization using characteristic polynomial interpolation.* Proceedings, Twenty-First Annual Joint Conference of the IEEE Computer and Communications Societies, New York, NY, USA, 2002, pp. 1510-1519 vol.3. [[PDF]](https://pdfs.semanticscholar.org/43da/2070b6b7b2320a1fed2fd5e70e87332c9c5e.pdf) +* <a name="myfootnote2">[2]</a> Cherly, Jørgen, Luis Gallardo, Leonid Vaserstein, and Ethel Wheland. *Solving quadratic equations over polynomial rings of characteristic two.* Publicacions Matemàtiques (1998): 131-142. [[PDF]](https://www.raco.cat/index.php/PublicacionsMatematiques/article/viewFile/37927/40412) +* <a name="myfootnote3">[3]</a> J. Massey. *Shift-register synthesis and BCH decoding.* IEEE Transactions on Information Theory, vol. 15, no. 1, pp. 122-127, January 1969. [[PDF]](http://crypto.stanford.edu/~mironov/cs359/massey.pdf) +* <a name="myfootnote4">[4]</a> Bhaskar Biswas, Vincent Herbert. *Efficient Root Finding of Polynomials over Fields of Characteristic 2.* 2009. hal-00626997. [[URL]](https://hal.archives-ouvertes.fr/hal-00626997) [[PDF]](https://hal.archives-ouvertes.fr/hal-00626997/document) +* <a name="myfootnote6">[6]</a> Eppstein, David, Michael T. Goodrich, Frank Uyeda, and George Varghese. *What's the difference?: efficient set reconciliation without prior context.* ACM SIGCOMM Computer Communication Review, vol. 41, no. 4, pp. 218-229. ACM, 2011. [[PDF]](https://www.ics.uci.edu/~eppstein/pubs/EppGooUye-SIGCOMM-11.pdf) +* <a name="myfootnote7">[7]</a> Goodrich, Michael T. and Michael Mitzenmacher. *Invertible bloom lookup tables.* 2011 49th Annual Allerton Conference on Communication, Control, and Computing (Allerton) (2011): 792-799. [[PDF]](https://arxiv.org/pdf/1101.2245.pdf) +* <a name="myfootnote8">[8]</a> Maxwell, Gregory F. *[Blocksonly mode BW savings, the limits of efficient block xfer, and better relay](https://bitcointalk.org/index.php?topic=1377345.0)* Bitcointalk 2016, *[Technical notes on mempool synchronizing relay](https://people.xiph.org/~greg/mempool_sync_relay.txt)* #bitcoin-wizards 2016. +* <a name="myfootnote9">[9]</a> Maxwell, Gregory F. *[Block network coding](https://en.bitcoin.it/wiki/User:Gmaxwell/block_network_coding)* Bitcoin Wiki 2014, *[Technical notes on efficient block xfer](https://people.xiph.org/~greg/efficient.block.xfer.txt)* #bitcoin-wizards 2015. +* <a name="myfootnote10">[10]</a> Ruffing, Tim, Moreno-Sanchez, Pedro, Aniket, Kate, *P2P Mixing and Unlinkable Bitcoin Transactions* NDSS Symposium 2017 [[URL]](https://eprint.iacr.org/2016/824) [[PDF]](https://eprint.iacr.org/2016/824.pdf) +* <a name="myfootnote11">[11]</a> Y. Misky, A. Trachtenberg, R. Zippel. *Set Reconciliation with Nearly Optimal Communication Complexity.* Cornell University, 2000. [[URL]](https://ecommons.cornell.edu/handle/1813/5803) [[PDF]](https://ecommons.cornell.edu/bitstream/handle/1813/5803/2000-1813.pdf) +* <a name="myfootnote12">[12]</a> Itoh, Toshiya, and Shigeo Tsujii. "A fast algorithm for computing multiplicative inverses in GF (2m) using normal bases." Information and computation 78, no. 3 (1988): 171-177. [[URL]](https://www.sciencedirect.com/science/article/pii/0890540188900247) diff --git a/src/minisketch/autogen.sh b/src/minisketch/autogen.sh new file mode 100755 index 0000000000..27417daf76 --- /dev/null +++ b/src/minisketch/autogen.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Copyright (c) 2013-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +set -e +srcdir="$(dirname $0)" +cd "$srcdir" +if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="`which glibtoolize 2>/dev/null`"; then + LIBTOOLIZE="${GLIBTOOLIZE}" + export LIBTOOLIZE +fi +which autoreconf >/dev/null || \ + (echo "configuration failed, please install autoconf first" && exit 1) +autoreconf --install --force --warnings=all diff --git a/src/minisketch/build-aux/m4/ax_check_compile_flag.m4 b/src/minisketch/build-aux/m4/ax_check_compile_flag.m4 new file mode 100644 index 0000000000..bd753b34d7 --- /dev/null +++ b/src/minisketch/build-aux/m4/ax_check_compile_flag.m4 @@ -0,0 +1,53 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_COMPILE_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 6 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/src/minisketch/build-aux/m4/ax_check_link_flag.m4 b/src/minisketch/build-aux/m4/ax_check_link_flag.m4 new file mode 100644 index 0000000000..03a30ce4c7 --- /dev/null +++ b/src/minisketch/build-aux/m4/ax_check_link_flag.m4 @@ -0,0 +1,53 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_LINK_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the linker or gives an error. +# (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the linker's default flags +# when the check is done. The check is thus made with the flags: "LDFLAGS +# EXTRA-FLAGS FLAG". This can for example be used to force the linker to +# issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_LINK_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,COMPILE}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 6 + +AC_DEFUN([AX_CHECK_LINK_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_ldflags_$4_$1])dnl +AC_CACHE_CHECK([whether the linker accepts $1], CACHEVAR, [ + ax_check_save_flags=$LDFLAGS + LDFLAGS="$LDFLAGS $4 $1" + AC_LINK_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + LDFLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_LINK_FLAGS diff --git a/src/minisketch/build-aux/m4/ax_check_preproc_flag.m4 b/src/minisketch/build-aux/m4/ax_check_preproc_flag.m4 new file mode 100644 index 0000000000..e43560fbd3 --- /dev/null +++ b/src/minisketch/build-aux/m4/ax_check_preproc_flag.m4 @@ -0,0 +1,53 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_PREPROC_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's +# preprocessor or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the preprocessor's default +# flags when the check is done. The check is thus made with the flags: +# "CPPFLAGS EXTRA-FLAGS FLAG". This can for example be used to force the +# preprocessor to issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_PREPROC_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{COMPILE,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 6 + +AC_DEFUN([AX_CHECK_PREPROC_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]cppflags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG preprocessor accepts $1], CACHEVAR, [ + ax_check_save_flags=$CPPFLAGS + CPPFLAGS="$CPPFLAGS $4 $1" + AC_PREPROC_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + CPPFLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_PREPROC_FLAGS diff --git a/src/minisketch/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/src/minisketch/build-aux/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 0000000000..f7e5137003 --- /dev/null +++ b/src/minisketch/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,962 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for no added switch, and then for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com> +# Copyright (c) 2012 Zack Weinberg <zackw@panix.com> +# Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu> +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com> +# Copyright (c) 2015 Paul Norman <penorman@mac.com> +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> +# Copyright (c) 2020 Jason Merrill <jason@redhat.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 12 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [], [dnl + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi]) + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template <typename T> + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check<void> single_type; + typedef check<check<void>> double_type; + typedef check<check<check<void>>> triple_type; + typedef check<check<check<check<void>>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same<int, decltype(0)>::value == true, ""); + static_assert(is_same<int, decltype(c)>::value == false, ""); + static_assert(is_same<int, decltype(v)>::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same<int, decltype(ac)>::value == true, ""); + static_assert(is_same<int, decltype(av)>::value == true, ""); + static_assert(is_same<int, decltype(sumi)>::value == true, ""); + static_assert(is_same<int, decltype(sumf)>::value == false, ""); + static_assert(is_same<int, decltype(add(c, v))>::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template <int...> + struct sum; + + template <int N0, int... N1toN> + struct sum<N0, N1toN...> + { + static constexpr auto value = N0 + sum<N1toN...>::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template<typename T> + using member = typename T::member_type; + + template<typename T> + void func(...) {} + + template<typename T> + void func(member<T>*) {} + + void test(); + + void test() { func<foo>(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same<int, decltype(f(x))>::value, ""); + static_assert(is_same<int&, decltype(g(x))>::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include <initializer_list> +#include <utility> +#include <type_traits> + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template<typename... Args> + int multiply(Args... args) + { + return (args * ... * 1); + } + + template<typename... Args> + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value); + static_assert(std::is_same<int, decltype(bar)>::value); + } + + namespace test_typename_in_template_template_parameter + { + + template<template<typename> typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template <bool cond> + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template <typename T1, typename T2> + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template <auto n> + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair<int, int> pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair<int, int>& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template<typename T> + Bad + f(T*, T*); + + template<typename T1, typename T2> + Good + f(T1*, T2*); + + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>); + + } + + namespace test_inline_variables + { + + template<class T> void f(T) + {} + + template<class T> inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/src/minisketch/ci/cirrus.sh b/src/minisketch/ci/cirrus.sh new file mode 100755 index 0000000000..02f737ca7f --- /dev/null +++ b/src/minisketch/ci/cirrus.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +set -e +set -x + +export LC_ALL=C + +env >> test_env.log + +$CC -v || true +valgrind --version || true + +./autogen.sh + +FIELDS= +if [ -n "$ENABLE_FIELDS" ]; then + FIELDS="--enable-fields=$ENABLE_FIELDS" +fi +./configure --host="$HOST" --enable-benchmark="$BENCH" $FIELDS + +# We have set "-j<n>" in MAKEFLAGS. +make + +# Print information about binaries so that we can see that the architecture is correct +file test* || true +file bench* || true +file .libs/* || true + +if [ -n "$BUILD" ] +then + make "$BUILD" +fi + +if [ -n "$EXEC_CMD" ]; then + $EXEC_CMD ./test $TESTRUNS + $EXEC_CMD ./test-verify $TESTRUNS +fi + +if [ "$BENCH" = "yes" ]; then + $EXEC_CMD ./bench +fi diff --git a/src/minisketch/ci/linux-debian.Dockerfile b/src/minisketch/ci/linux-debian.Dockerfile new file mode 100644 index 0000000000..63e5412ee7 --- /dev/null +++ b/src/minisketch/ci/linux-debian.Dockerfile @@ -0,0 +1,17 @@ +FROM debian:stable + +RUN dpkg --add-architecture i386 +RUN dpkg --add-architecture s390x +RUN apt-get update + +# dkpg-dev: to make pkg-config work in cross-builds +RUN apt-get install --no-install-recommends --no-upgrade -y \ + git ca-certificates \ + make automake libtool pkg-config dpkg-dev valgrind qemu-user \ + gcc g++ clang libc6-dbg \ + gcc-i686-linux-gnu g++-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 \ + g++-s390x-linux-gnu gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \ + wine g++-mingw-w64-x86-64 + +# Run a dummy command in wine to make it set up configuration +RUN wine true || true diff --git a/src/minisketch/configure.ac b/src/minisketch/configure.ac new file mode 100644 index 0000000000..9dc66e7fd2 --- /dev/null +++ b/src/minisketch/configure.ac @@ -0,0 +1,162 @@ +AC_INIT([minisketch], [0.0.1], [http://github.com/sipa/minisketch/]) + +m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) + +AC_PREREQ(2.60) +AC_CONFIG_SRCDIR([src/minisketch.cpp]) +AC_CONFIG_AUX_DIR([build-aux]) +AC_CONFIG_MACRO_DIR([build-aux/m4]) +AM_INIT_AUTOMAKE([subdir-objects foreign]) + +LT_INIT +LT_LANG([C++]) +AC_LANG([C++]) + +AC_PATH_PROG(CCACHE,ccache) + +AC_ARG_ENABLE([ccache], + [AS_HELP_STRING([--disable-ccache], + [do not use ccache for building (default is to use if found)])], + [use_ccache=$enableval], + [use_ccache=auto]) + +AC_ARG_ENABLE(tests, + AS_HELP_STRING([--enable-tests],[compile tests (default is yes)]), + [use_tests=$enableval], + [use_tests=yes]) + +AC_ARG_ENABLE(benchmark, + AS_HELP_STRING([--enable-benchmark],[compile benchmark (default is no)]), + [use_benchmark=$enableval], + [use_benchmark=no]) + +m4_define([SUPPORTED_FIELDS], [2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]) + +AC_MSG_CHECKING([which field sizes to build]) +AC_ARG_ENABLE([fields], AS_HELP_STRING([--enable-fields=LIST], [Comma-separated list of field sizes to build. Default=all. Available sizes:] m4_translit(m4_defn([SUPPORTED_FIELDS]), [,], [ ])), [], [enable_fields=SUPPORTED_FIELDS]) +have_disabled_fields=no +have_enabled_fields=no +m4_foreach([FIELD], [SUPPORTED_FIELDS], [ + case ",$enable_fields," in + *,FIELD,*) + have_enabled_fields=yes + ;; + *) + AC_DEFINE(DISABLE_FIELD_[]FIELD, [1], + [Define to 1 to remove support for field size] FIELD [.]) + have_disabled_fields=yes + ;; + esac +]) +AC_MSG_RESULT([$enable_fields]) +if test "x$have_enabled_fields" = xno; then + AC_MSG_ERROR([No field sizes are enabled.]) +fi + +AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) + +AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory]) +enable_clmul= +AX_CHECK_COMPILE_FLAG([-mpclmul],[[enable_clmul=yes]],,[[$CXXFLAG_WERROR]],[AC_LANG_PROGRAM([ + #include <stdint.h> + #include <x86intrin.h> +], [ + __m128i a = _mm_cvtsi64_si128((uint64_t)7); + __m128i b = _mm_clmulepi64_si128(a, a, 37); + __m128i c = _mm_srli_epi64(b, 41); + __m128i d = _mm_xor_si128(b, c); + uint64_t e = _mm_cvtsi128_si64(d); + return e == 0; +])]) +if test x$enable_clmul = xyes; then + CLMUL_CXXFLAGS="-mpclmul" + AC_DEFINE(HAVE_CLMUL, 1, [Define this symbol if clmul instructions can be used]) +fi + + +AC_MSG_CHECKING(for working clz builtins) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ + unsigned a = __builtin_clz(1); + unsigned long b = __builtin_clzl(1); + unsigned long long c = __builtin_clzll(1); + ])], + [ + AC_DEFINE(HAVE_CLZ, 1, [Define this symbol if clz builtins are present and working]) + AC_MSG_RESULT(yes) + ],[ + AC_MSG_RESULT(no) + ] +) + +AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[LDFLAGS="-Wl,--exclude-libs,ALL $LDFLAGS"]) + +case $host in + *mingw*) + dnl -static is interpreted by libtool, where it has a different meaning. + dnl In libtool-speak, it's -all-static. + AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) + ;; + *) + AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="-static"]) + ;; +esac + +AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],[],[$CXXFLAG_WERROR]) + +## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all +## unknown options if any other warning is produced. Test the -Wfoo case, and +## set the -Wno-foo case if it works. +AX_CHECK_COMPILE_FLAG([-Wshift-count-overflow],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-shift-count-overflow"],,[[$CXXFLAG_WERROR]]) + +if test "x$use_ccache" != "xno"; then + AC_MSG_CHECKING(if ccache should be used) + if test x$CCACHE = x; then + if test "x$use_ccache" = "xyes"; then + AC_MSG_ERROR([ccache not found.]); + else + use_ccache=no + fi + else + use_ccache=yes + CC="$ac_cv_path_CCACHE $CC" + CXX="$ac_cv_path_CCACHE $CXX" + fi + AC_MSG_RESULT($use_ccache) +fi +if test "x$use_ccache" = "xyes"; then + AX_CHECK_COMPILE_FLAG([-Qunused-arguments],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Qunused-arguments"],,[[$CXXFLAG_WERROR]]) +fi + +VERIFY_DEFINES=-DMINISKETCH_VERIFY +RELEASE_DEFINES= + +AC_CONFIG_FILES([ + Makefile +]) + +AC_SUBST(CLMUL_CXXFLAGS) +AC_SUBST(WARN_CXXFLAGS) +AC_SUBST(NOWARN_CXXFLAGS) +AC_SUBST(VERIFY_DEFINES) +AC_SUBST(RELEASE_DEFINES) +AC_SUBST(LIBTOOL_APP_LDFLAGS) +AM_CONDITIONAL([ENABLE_CLMUL],[test x$enable_clmul = xyes]) +AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) +AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) +AC_OUTPUT + +echo +echo "Build Options:" +echo " with benchmarks = $use_benchmark" +echo " with tests = $use_tests" +echo " enable clmul fields = $enable_clmul" +echo " CXX = $CXX" +echo " CXXFLAGS = $CXXFLAGS" +echo " CPPFLAGS = $CPPFLAGS" +echo " LDFLAGS = $LDFLAGS" +if test "$have_disabled_fields" = "yes"; then +echo +echo "Only compiling in support for field sizes: $enable_fields" +echo "WARNING: this means the library will lack support for other field sizes entirely" +fi diff --git a/src/minisketch/doc/example.c b/src/minisketch/doc/example.c new file mode 100644 index 0000000000..7279165845 --- /dev/null +++ b/src/minisketch/doc/example.c @@ -0,0 +1,51 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include <stdio.h> +#include <assert.h> +#include "../include/minisketch.h" + +int main(void) { + + minisketch *sketch_a = minisketch_create(12, 0, 4); + + for (int i = 3000; i < 3010; ++i) { + minisketch_add_uint64(sketch_a, i); + } + + size_t sersize = minisketch_serialized_size(sketch_a); + assert(sersize == 12 * 4 / 8); // 4 12-bit values is 6 bytes. + unsigned char *buffer_a = malloc(sersize); + minisketch_serialize(sketch_a, buffer_a); + minisketch_destroy(sketch_a); + + minisketch *sketch_b = minisketch_create(12, 0, 4); // Bob's own sketch + for (int i = 3002; i < 3012; ++i) { + minisketch_add_uint64(sketch_b, i); + } + + sketch_a = minisketch_create(12, 0, 4); // Alice's sketch + minisketch_deserialize(sketch_a, buffer_a); // Load Alice's sketch + free(buffer_a); + + // Merge the elements from sketch_a into sketch_b. The result is a sketch_b + // which contains all elements that occurred in Alice's or Bob's sets, but not + // in both. + minisketch_merge(sketch_b, sketch_a); + + uint64_t differences[4]; + ssize_t num_differences = minisketch_decode(sketch_b, 4, differences); + minisketch_destroy(sketch_a); + minisketch_destroy(sketch_b); + if (num_differences < 0) { + printf("More than 4 differences!\n"); + } else { + ssize_t i; + for (i = 0; i < num_differences; ++i) { + printf("%u is in only one of the two sets\n", (unsigned)differences[i]); + } + } +} diff --git a/src/minisketch/doc/gen_basefpbits.sage b/src/minisketch/doc/gen_basefpbits.sage new file mode 100644 index 0000000000..d1e75a6e29 --- /dev/null +++ b/src/minisketch/doc/gen_basefpbits.sage @@ -0,0 +1,78 @@ +# Require exact values up to +FPBITS = 256 + +# Overkill accuracy +F = RealField(400) + +def BaseFPBits(bits, capacity): + return bits * capacity - int(ceil(F(log(sum(binomial(2**bits - 1, i) for i in range(capacity+1)), 2)))) + +def Log2Factorial(capacity): + return int(floor(log(factorial(capacity), 2))) + +print("uint64_t BaseFPBits(uint32_t bits, uint32_t capacity) {") +print(" // Correction table for low bits/capacities") +TBLS={} +FARS={} +SKIPS={} +for bits in range(1, 32): + TBL = [] + for capacity in range(1, min(2**bits, FPBITS)): + exact = BaseFPBits(bits, capacity) + approx = Log2Factorial(capacity) + TBL.append((exact, approx)) + MIN = 10000000000 + while len(TBL) and ((TBL[-1][0] == TBL[-1][1]) or (TBL[-1][0] >= FPBITS and TBL[-1][1] >= FPBITS)): + MIN = min(MIN, TBL[-1][0] - TBL[-1][1]) + TBL.pop() + while len(TBL) and (TBL[-1][0] - TBL[-1][1] == MIN): + TBL.pop() + SKIP = 0 + while SKIP < len(TBL) and TBL[SKIP][0] == TBL[SKIP][1]: + SKIP += 1 + DIFFS = [TBL[i][0] - TBL[i][1] for i in range(SKIP, len(TBL))] + if len(DIFFS) > 0 and len(DIFFS) * Integer(max(DIFFS)).nbits() > 64: + print(" static constexpr uint8_t ADD%i[] = {%s};" % (bits, ", ".join(("%i" % (TBL[i][0] - TBL[i][1])) for i in range(SKIP, len(TBL))))) + TBLS[bits] = DIFFS + FARS[bits] = MIN + SKIPS[bits] = SKIP +print("") +print(" if (capacity == 0) return 0;") +print(" uint64_t ret = 0;") +print(" if (bits < 32 && capacity >= (1U << bits)) {") +print(" ret = uint64_t{bits} * (capacity - (1U << bits) + 1);") +print(" capacity = (1U << bits) - 1;") +print(" }") +print(" ret += Log2Factorial(capacity);") +print(" switch (bits) {") +for bits in sorted(TBLS.keys()): + if len(TBLS[bits]) == 0: + continue + width = Integer(max(TBLS[bits])).nbits() + if len(TBLS[bits]) == 1: + add = "%i" % TBLS[bits][0] + elif len(TBLS[bits]) * width <= 64: + code = sum((2**(width*i) * TBLS[bits][i]) for i in range(len(TBLS[bits]))) + if width == 1: + add = "(0x%x >> (capacity - %i)) & 1" % (code, 1 + SKIPS[bits]) + else: + add = "(0x%x >> %i * (capacity - %i)) & %i" % (code, width, 1 + SKIPS[bits], 2**width - 1) + else: + add = "ADD%i[capacity - %i]" % (bits, 1 + SKIPS[bits]) + if len(TBLS[bits]) + SKIPS[bits] == 2**bits - 1: + print(" case %i: return ret + (capacity <= %i ? 0 : %s);" % (bits, SKIPS[bits], add)) + else: + print(" case %i: return ret + (capacity <= %i ? 0 : capacity > %i ? %i : %s);" % (bits, SKIPS[bits], len(TBLS[bits]) + SKIPS[bits], FARS[bits], add)) +print(" default: return ret;") +print(" }") +print("}") + +print("void TestBaseFPBits() {") +print(" static constexpr uint16_t TBL[20][100] = {%s};" % (", ".join("{" + ", ".join(("%i" % BaseFPBits(bits, capacity)) for capacity in range(0, 100)) + "}" for bits in range(1, 21)))) +print(" for (int bits = 1; bits <= 20; ++bits) {") +print(" for (int capacity = 0; capacity < 100; ++capacity) {") +print(" uint64_t computed = BaseFPBits(bits, capacity), exact = TBL[bits - 1][capacity];") +print(" CHECK(exact == computed || (exact >= 256 && computed >= 256));") +print(" }") +print(" }") +print("}") diff --git a/src/minisketch/doc/gen_params.sage b/src/minisketch/doc/gen_params.sage new file mode 100755 index 0000000000..1cf036adb4 --- /dev/null +++ b/src/minisketch/doc/gen_params.sage @@ -0,0 +1,333 @@ +#!/usr/bin/env sage +r""" +Generate finite field parameters for minisketch. + +This script selects the finite fields used by minisketch + for various sizes and generates the required tables for + the implementation. + +The output (after formatting) can be found in src/fields/*.cpp. + +""" +B.<b> = GF(2) +P.<p> = B[] + +def apply_map(m, v): + r = 0 + i = 0 + while v != 0: + if (v & 1): + r ^^= m[i] + i += 1 + v >>= 1 + return r + +def recurse_moduli(acc, maxweight, maxdegree): + for pos in range(maxweight, maxdegree + 1, 1): + poly = acc + p^pos + if maxweight == 1: + if poly.is_irreducible(): + return (pos, poly) + else: + (deg, ret) = recurse_moduli(poly, maxweight - 1, pos - 1) + if ret is not None: + return (pos, ret) + return (None, None) + +def compute_moduli(bits): + # Return all optimal irreducible polynomials for GF(2^bits) + # The result is a list of tuples (weight, degree of second-highest nonzero coefficient, polynomial) + maxdegree = bits - 1 + result = [] + for weight in range(1, bits, 2): + deg, res = None, None + while True: + ret = recurse_moduli(p^bits + 1, weight, maxdegree) + if ret[0] is not None: + (deg, res) = ret + maxdegree = deg - 1 + else: + break + if res is not None: + result.append((weight + 2, deg, res)) + return result + +def bits_to_int(vals): + ret = 0 + base = 1 + for val in vals: + ret += Integer(val) * base + base *= 2 + return ret + +def sqr_table(f, bits, n=1): + ret = [] + for i in range(bits): + ret.append((f^(2^n*i)).integer_representation()) + return ret + +# Compute x**(2**n) +def pow2(x, n): + for i in range(n): + x = x**2 + return x + +def qrt_table(F, f, bits): + # Table for solving x2 + x = a + # This implements the technique from https://www.raco.cat/index.php/PublicacionsMatematiques/article/viewFile/37927/40412, Lemma 1 + for i in range(bits): + if (f**i).trace() != 0: + u = f**i + ret = [] + for i in range(0, bits): + d = f^i + y = sum(pow2(d, j) * sum(pow2(u, k) for k in range(j)) for j in range(1, bits)) + ret.append(y.integer_representation() ^^ (y.integer_representation() & 1)) + return ret + +def conv_tables(F, NF, bits): + # Generate a F(2) linear projection that maps elements from one field + # to an isomorphic field with a different modulus. + f = F.gen() + fp = f.minimal_polynomial() + assert(fp == F.modulus()) + nfp = fp.change_ring(NF) + nf = sorted(nfp.roots(multiplicities=False))[0] + ret = [] + matrepr = [[B(0) for x in range(bits)] for y in range(bits)] + for i in range(bits): + val = (nf**i).integer_representation() + ret.append(val) + for j in range(bits): + matrepr[j][i] = B((val >> j) & 1) + mat = Matrix(matrepr).inverse().transpose() + ret2 = [] + for i in range(bits): + ret2.append(bits_to_int(mat[i])) + + for t in range(100): + f1a = F.random_element() + f1b = F.random_element() + f1r = f1a * f1b + f2a = NF.fetch_int(apply_map(ret, f1a.integer_representation())) + f2b = NF.fetch_int(apply_map(ret, f1b.integer_representation())) + f2r = NF.fetch_int(apply_map(ret, f1r.integer_representation())) + f2s = f2a * f2b + assert(f2r == f2s) + + for t in range(100): + f2a = NF.random_element() + f2b = NF.random_element() + f2r = f2a * f2b + f1a = F.fetch_int(apply_map(ret2, f2a.integer_representation())) + f1b = F.fetch_int(apply_map(ret2, f2b.integer_representation())) + f1r = F.fetch_int(apply_map(ret2, f2r.integer_representation())) + f1s = f1a * f1b + assert(f1r == f1s) + + return (ret, ret2) + +def fmt(i,typ): + if i == 0: + return "0" + else: + return "0x%x" % i + +def lintranstype(typ, bits, maxtbl): + gsize = min(maxtbl, bits) + array_size = (bits + gsize - 1) // gsize + bits_list = [] + total = 0 + for i in range(array_size): + rsize = (bits - total + array_size - i - 1) // (array_size - i) + total += rsize + bits_list.append(rsize) + return "RecLinTrans<%s, %s>" % (typ, ", ".join("%i" % x for x in bits_list)) + +INT=0 +CLMUL=1 +CLMUL_TRI=2 +MD=3 + +def print_modulus_md(mod): + ret = "" + pos = mod.degree() + for c in reversed(list(mod)): + if c: + if ret: + ret += " + " + if pos == 0: + ret += "1" + elif pos == 1: + ret += "x" + else: + ret += "x<sup>%i</sup>" % pos + pos -= 1 + return ret + +def pick_modulus(bits, style): + # Choose the lexicographicly-first lowest-weight modulus + # optionally subject to implementation specific constraints. + moduli = compute_moduli(bits) + if style == INT or style == MD: + multi_sqr = False + need_trans = False + elif style == CLMUL: + # Fast CLMUL reduction requires that bits + the highest + # set bit are less than 66. + moduli = list(filter((lambda x: bits+x[1] <= 66), moduli)) + moduli + multi_sqr = True + need_trans = True + if not moduli or moduli[0][2].change_ring(ZZ)(2) == 3 + 2**bits: + # For modulus 3, CLMUL_TRI is obviously better. + return None + elif style == CLMUL_TRI: + moduli = list(filter(lambda x: bits+x[1] <= 66, moduli)) + moduli + moduli = list(filter(lambda x: x[0] == 3, moduli)) + multi_sqr = True + need_trans = True + else: + assert(False) + if not moduli: + return None + return moduli[0][2] + +def print_result(bits, style): + if style == INT: + multi_sqr = False + need_trans = False + table_id = "%i" % bits + elif style == MD: + pass + elif style == CLMUL: + multi_sqr = True + need_trans = True + table_id = "%i" % bits + elif style == CLMUL_TRI: + multi_sqr = True + need_trans = True + table_id = "TRI%i" % bits + else: + assert(False) + + nmodulus = pick_modulus(bits, INT) + modulus = pick_modulus(bits, style) + if modulus is None: + return + + if style == MD: + print("* *%s*" % print_modulus_md(modulus)) + return + + if bits > 32: + typ = "uint64_t" + elif bits > 16: + typ = "uint32_t" + elif bits > 8: + typ = "uint16_t" + else: + typ = "uint8_t" + + ttyp = lintranstype(typ, bits, 4) + rtyp = lintranstype(typ, bits, 6) + + F.<f> = GF(2**bits, modulus=modulus) + + include_table = True + if style != INT and style != CLMUL: + cmodulus = pick_modulus(bits, CLMUL) + if cmodulus == modulus: + include_table = False + table_id = "%i" % bits + + if include_table: + print("typedef %s StatTable%s;" % (rtyp, table_id)) + rtyp = "StatTable%s" % table_id + if (style == INT): + print("typedef %s DynTable%s;" % (ttyp, table_id)) + ttyp = "DynTable%s" % table_id + + if need_trans: + if modulus != nmodulus: + # If the bitstream modulus is not the best modulus for + # this implementation a conversion table will be needed. + ctyp = rtyp + NF.<nf> = GF(2**bits, modulus=nmodulus) + ctables = conv_tables(NF, F, bits) + loadtbl = "&LOAD_TABLE_%s" % table_id + savetbl = "&SAVE_TABLE_%s" % table_id + if include_table: + print("constexpr %s LOAD_TABLE_%s({%s});" % (ctyp, table_id, ", ".join([fmt(x,typ) for x in ctables[0]]))) + print("constexpr %s SAVE_TABLE_%s({%s});" % (ctyp, table_id, ", ".join([fmt(x,typ) for x in ctables[1]]))) + else: + ctyp = "IdTrans" + loadtbl = "&ID_TRANS" + savetbl = "&ID_TRANS" + else: + assert(modulus == nmodulus) + + if include_table: + print("constexpr %s SQR_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 1)]))) + if multi_sqr: + # Repeated squaring is a linearised polynomial so in F(2^n) it is + # F(2) linear and can be computed by a simple bit-matrix. + # Repeated squaring is especially useful in powering ladders such as + # for inversion. + # When certain repeated squaring tables are not in use, use the QRT + # table instead to make the C++ compiler happy (it always has the + # same type). + sqr2 = "&QRT_TABLE_%s" % table_id + sqr4 = "&QRT_TABLE_%s" % table_id + sqr8 = "&QRT_TABLE_%s" % table_id + sqr16 = "&QRT_TABLE_%s" % table_id + if ((bits - 1) >= 4): + if include_table: + print("constexpr %s SQR2_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 2)]))) + sqr2 = "&SQR2_TABLE_%s" % table_id + if ((bits - 1) >= 8): + if include_table: + print("constexpr %s SQR4_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 4)]))) + sqr4 = "&SQR4_TABLE_%s" % table_id + if ((bits - 1) >= 16): + if include_table: + print("constexpr %s SQR8_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 8)]))) + sqr8 = "&SQR8_TABLE_%s" % table_id + if ((bits - 1) >= 32): + if include_table: + print("constexpr %s SQR16_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 16)]))) + sqr16 = "&SQR16_TABLE_%s" % table_id + if include_table: + print("constexpr %s QRT_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in qrt_table(F, f, bits)]))) + + modulus_weight = modulus.hamming_weight() + modulus_degree = (modulus - p**bits).degree() + modulus_int = (modulus - p**bits).change_ring(ZZ)(2) + + lfsr = "" + + if style == INT: + print("typedef Field<%s, %i, %i, %s, %s, &SQR_TABLE_%s, &QRT_TABLE_%s%s> Field%i;" % (typ, bits, modulus_int, rtyp, ttyp, table_id, table_id, lfsr, bits)) + elif style == CLMUL: + print("typedef Field<%s, %i, %i, %s, &SQR_TABLE_%s, %s, %s, %s, %s, &QRT_TABLE_%s, %s, %s, %s%s> Field%i;" % (typ, bits, modulus_int, rtyp, table_id, sqr2, sqr4, sqr8, sqr16, table_id, ctyp, loadtbl, savetbl, lfsr, bits)) + elif style == CLMUL_TRI: + print("typedef FieldTri<%s, %i, %i, %s, &SQR_TABLE_%s, %s, %s, %s, %s, &QRT_TABLE_%s, %s, %s, %s> FieldTri%i;" % (typ, bits, modulus_degree, rtyp, table_id, sqr2, sqr4, sqr8, sqr16, table_id, ctyp, loadtbl, savetbl, bits)) + else: + assert(False) + +for bits in range(2, 65): + print("#ifdef ENABLE_FIELD_INT_%i" % bits) + print("// %i bit field" % bits) + print_result(bits, INT) + print("#endif") + print("") + +for bits in range(2, 65): + print("#ifdef ENABLE_FIELD_INT_%i" % bits) + print("// %i bit field" % bits) + print_result(bits, CLMUL) + print_result(bits, CLMUL_TRI) + print("#endif") + print("") + +for bits in range(2, 65): + print_result(bits, MD) diff --git a/src/minisketch/doc/log2_factorial.sage b/src/minisketch/doc/log2_factorial.sage new file mode 100644 index 0000000000..afc6d66c57 --- /dev/null +++ b/src/minisketch/doc/log2_factorial.sage @@ -0,0 +1,85 @@ +import bisect + +INPUT_BITS = 32 +TABLE_BITS = 5 +INT_BITS = 64 +EXACT_FPBITS = 256 + +F = RealField(100) # overkill + +def BestOverApproxInvLog2(mulof, maxd): + """ + Compute denominator of an approximation of 1/log(2). + + Specifically, find the value of d (<= maxd, and a multiple of mulof) + such that ceil(d/log(2))/d is the best approximation of 1/log(2). + """ + dist=1 + best=0 + # Precomputed denominators that lead to good approximations of 1/log(2) + for d in [1, 2, 9, 70, 131, 192, 445, 1588, 4319, 11369, 18419, 25469, 287209, 836158, 3057423, 8336111, 21950910, 35565709, 49180508, 161156323, 273132138, 385107953, 882191721]: + kd = lcm(mulof, d) + if kd <= maxd: + n = ceil(kd / log(2)) + dis = F((n / kd) - 1 / log(2)) + if dis < dist: + dist = dis + best = kd + return best + + +LOG2_TABLE = [] +A = 0 +B = 0 +C = 0 +D = 0 +K = 0 + +def Setup(k): + global LOG2_TABLE, A, B, C, D, K + K = k + LOG2_TABLE = [] + for i in range(2 ** TABLE_BITS): + LOG2_TABLE.append(int(floor(F(K * log(1 + i / 2**TABLE_BITS, 2))))) + + # Maximum for (2*x+1)*LogK2(x) + max_T = (2^(INPUT_BITS + 1) - 1) * (INPUT_BITS*K - 1) + # Maximum for A + max_A = (2^INT_BITS - 1) // max_T + D = BestOverApproxInvLog2(2 * K, max_A * 2 * K) + A = D // (2 * K) + B = int(ceil(F(D/log(2)))) + C = int(floor(F(D*log(2*pi,2)/2))) + +def LogK2(n): + assert(n >= 1 and n < (1 << INPUT_BITS)) + bits = Integer(n).nbits() + return K * (bits - 1) + LOG2_TABLE[((n << (INPUT_BITS - bits)) >> (INPUT_BITS - TABLE_BITS - 1)) - 2**TABLE_BITS] + +def Log2Fact(n): + # Use formula (A*(2*x+1)*LogK2(x) - B*x + C) / D + return (A*(2*n+1)*LogK2(n) - B*n + C) // D + (n < 3) + +RES = [int(F(log(factorial(i),2))) for i in range(EXACT_FPBITS * 10)] + +best_worst_ratio = 0 + +for K in range(1, 10000): + Setup(K) + assert(LogK2(1) == 0) + assert(LogK2(2) == K) + assert(LogK2(4) == 2 * K) + good = True + worst_ratio = 1 + for i in range(1, EXACT_FPBITS * 10): + exact = RES[i] + approx = Log2Fact(i) + if not (approx <= exact and ((approx == exact) or (approx >= EXACT_FPBITS and exact >= EXACT_FPBITS))): + good = False + break + if worst_ratio * exact > approx: + worst_ratio = approx / exact + if good and worst_ratio > best_worst_ratio: + best_worst_ratio = worst_ratio + print("Formula: (%i*(2*x+1)*floor(%i*log2(x)) - %i*x + %i) / %i; log(max_ratio)=%f" % (A, K, B, C, D, RR(-log(worst_ratio)))) + print("LOG2K_TABLE: %r" % LOG2_TABLE) diff --git a/src/minisketch/doc/math.md b/src/minisketch/doc/math.md new file mode 100644 index 0000000000..cf46f193ab --- /dev/null +++ b/src/minisketch/doc/math.md @@ -0,0 +1,117 @@ +# The mathematics of Minisketch sketches + +This is an unconventional mathematical overview of the PinSketch algorithm without references to coding theory<sup>[[1]](#myfootnote1)</sup>. + +## Set sketches + +A sketch, for the purpose of this description, can be seen as a "set checksum" with two peculiar properties: + +* Sketches have a predetermined capacity, and when the number of elements in the set is not higher than the capacity, minisketch will always recover the entire set from the sketch. A sketch of *b*-bit elements with capacity *c* can be stored in *bc* bits. +* The sketches of two sets can be combined by adding them (XOR) to obtain a sketch of the [symmetric difference](https://en.wikipedia.org/wiki/Symmetric_difference) between the two sets (*i.e.*, all elements that occur in one but not both input sets). + +This overview explains how sets can be converted into a sketch and how a set can be recovered from a sketch. + +## From field elements to sketches + +**Data entries as field elements** + +Every integer in the range *[1...2<sup>b</sup>-1]* (the acceptable data elements for a Minisketch sketch with field size *b*) can be mapped to a nonzero field element of *GF(2<sup>b</sup>)*. In this [finite field](https://en.wikipedia.org/wiki/Finite_field), we can add and multiply elements together, with many of the expected properties for those operations. Addition (and subtraction!) of field elements corresponds to bitwise XOR of the integers they correspond to, though multiplication is more involved. + +**Sets as power series** + +We define a function *S* which maps field elements *m* to the following [formal power series](https://en.wikipedia.org/wiki/Formal_power_series) (similar to a polynomial, except there can be an infinite number of terms, and we don't care about concepts like convergence as we're never going to actually evaluate it for a specific value of *x*): + +* *S(m) = 1 + mx + m<sup>2</sup>x<sup>2</sup> + m<sup>3</sup>x<sup>3</sup> + ...*. + +We then extend this function to operate on sets of field elements, by adding together the images of every set element. If *M = {m<sub>1</sub>, m<sub>2</sub>, ... }*: + +* *S(M) = S({m<sub>1</sub>,m<sub>2</sub>,...}) = S(m<sub>1</sub>) + S(m<sub>2</sub>) + ... = (1 + 1 + ...) + (m<sub>1</sub> + m<sub>2</sub> + ...)x + (m<sub>1</sub><sup>2</sup> + m<sub>2</sub><sup>2</sup> + ...)x<sup>2</sup> + (m<sub>1</sub><sup>3</sup> + ...* + +Because in our field addition corresponds to XOR of integers, it holds for every *a* that *a + a = 0*. This carries over to the *S* function, meaning that *S(a) + S(a) = 0* for every *a*. This means that the coefficients of these power series have the second of the properties we +desire from a sketch, namely that an efficient operation exists to +combine two sketches such that the result is a sketch of the symmetric +difference of the sets. It holds that +*S({m<sub>1</sub>,m<sub>2</sub>}) + S({m<sub>2</sub>,m<sub>3</sub>}) = S(m<sub>1</sub>) + (S(m<sub>2</sub>) + S(m<sub>2</sub>)) + S(m<sub>3</sub>) = S(m<sub>1</sub>) + S(m<sub>3</sub>) = S({m<sub>1</sub>,m<sub>3</sub>})*. The question is whether we can also efficiently recover the elements from their power series' coefficients. + +**An infinity of coefficients is hard** + +To make reasoning about these power series easier, notice that the series for a single element is in fact a [geometric series](https://en.wikipedia.org/wiki/Geometric_series). If we were working over real numbers rather than a finite field and *|mx| < 1*, it would converge to *(1 - mx)<sup>-1</sup>*. Convergence has no meaning in formal power series, however it is still the case that: + +* *(1 - mx) S(m) = 1* + +You can verify this by seeing that every coefficient except the constant one gets cancelled out by the multiplication. This can be generalized to the series for multiple set elements. For two elements we have: + +* *(1 - m<sub>1</sub>x) (1 - m<sub>2</sub>x) S({m<sub>1</sub>,m<sub>2</sub>}) = (1 - m<sub>1</sub>x) (1 - m<sub>2</sub>x) (S(m<sub>1</sub>) + S(m<sub>2</sub>)) = (1 - m<sub>2</sub>x) + (1 - m<sub>1</sub>x)* + +And for three: + +* *(1 - m<sub>1</sub>x) (1 - m<sub>2</sub>x) (1 - m<sub>3</sub>x) S({m<sub>1</sub>,m<sub>2</sub>,m<sub>3</sub>}) = (1 - m<sub>1</sub>x) (1 - m<sub>2</sub>x) (1 - m<sub>3</sub>x) (S(m<sub>1</sub>) + S(m<sub>2</sub>) + S(m<sub>3</sub>)) = (1 - m<sub>2</sub>x)(1 - m<sub>3</sub>x) + (1 - m<sub>1</sub>x)(1 - m<sub>3</sub>x) + (1 - m<sub>1</sub>x)(1 - m<sub>2</sub>x)* + +In each case, we notice that multiplying *S(M)* with *(1 - m<sub>i</sub>x)* for each element *m<sub>i</sub> ∈ M* results in a polynomial of degree *n-1*. + +**Solving for the set elements** + +The above insight lets us build a solver that extracts the set elements from the coefficients of a power series. If we can find a polynomial *L* that is the product of *n* different *(1 - m<sub>i</sub>x)* factors for various values of *m<sub>i</sub>*, such that *P = S(M)L* is an *n-1* degree polynomial, then those values *m<sub>i</sub>* are the elements of *M*. + +The coefficients of *P* are nontrivial expressions of the set elements themselves. However, we can just focus on the coefficients of degree *n* and higher in *P*, as those are all 0. Let *s<sub>i</sub>* be the coefficients of *S(M)*, and *l<sub>i</sub>* the coefficients of L. In other words, *S(M) = s<sub>0</sub> + s<sub>1</sub>x + s<sub>2</sub>x<sup>2</sup> + s<sub>3</sub>x<sup>3</sup> + ...* and *L = l<sub>0</sub> + l<sub>1</sub>x + l<sub>2</sub>x<sup>2</sup> + l<sub>3</sub>x<sup>3</sup> + ... + l<sub>n</sub>x<sup>n</sup>*. Note that *l<sub>0</sub> = 1*, as it is the product of all the *1* terms in the *(1 - m<sub>i</sub>x)* factors. + +Here are the equations for the coefficients of *S(M)L* of degree *n+1* through *2n*: +* *s<sub>n+1</sub> + s<sub>n+0</sub>l<sub>1</sub> + s<sub>n-1</sub>l<sub>2</sub> + s<sub>n-2</sub>l<sub>3</sub> + ... + s<sub>1</sub>l<sub>n</sub> = 0* +* *s<sub>n+2</sub> + s<sub>n+1</sub>l<sub>1</sub> + s<sub>n+0</sub>l<sub>2</sub> + s<sub>n-1</sub>l<sub>3</sub> + ... + s<sub>2</sub>l<sub>n</sub> = 0* +* *s<sub>n+3</sub> + s<sub>n+2</sub>l<sub>1</sub> + s<sub>n+1</sub>l<sub>2</sub> + s<sub>n+0</sub>l<sub>3</sub> + ... + s<sub>3</sub>l<sub>n</sub> = 0* +* ... +* *s<sub>2n</sub> + s<sub>2n-1</sub>l<sub>1</sub> + s<sub>2n-2</sub>l<sub>2</sub> + s<sub>2n-3</sub>l<sub>3</sub> + ... + s<sub>n</sub>l<sub>n</sub> = 0* + +These are *n* linear equations with *n* unknowns (the *l<sub>i<sub>* +values, for *i=1..n*), which can be solved using [Gaussian elimination](https://en.wikipedia.org/wiki/Gaussian_elimination). After doing so, +we have the coefficients of *L*, which can then be [factored](https://en.wikipedia.org/wiki/Factorization_of_polynomials_over_finite_fields) +into first degree factors of the form *(1 - m<sub>i</sub>x)*. The resulting *m* values are our set elements. + +**Putting it all together** + +Interestingly, only *2n* coefficients of *S(M)* were needed for solving +the set of equations above. This means we have our answer: the +coefficients *1* through *2n* of *S(M)*, or the list +*[m<sub>1</sub> + m<sub>2</sub> + ..., m<sub>1</sub><sup>2</sup> + m<sub>2</sub><sup>2</sup> + ..., ..., m<sub>1</sub><sup>2n</sup> + m<sub>2</sub><sup>2n</sup> + ...]* +functions as a sketch, satisfying the two properties we want: + +* Sketches can be combined to form the sketch of their symmetric difference, by simply pairwise adding the list elements together. +* With *2n* list elements we can efficiently recover *n* elements from a sketch. + +**Capacity and difference** + +The approach above only works when the number of elements *n* in the sketch is known. Of course we want to support cases where only an upper bound on the number of elements in the sketch is known, the capacity *c*. Given that we can reconstruct a set of size *c* from a sketch with *2c* terms, we should be able to reconstruct a set of size *n* too as long as *n ≤ c*. This is simply a matter of trying to solve the above set of equations assuming values of *n* that count down from *c* until a solution is found for one. This is known as the [Peterson-Gorenstein-Zierler algorithm](https://en.wikipedia.org/wiki/BCH_code#Peterson%E2%80%93Gorenstein%E2%80%93Zierler_algorithm). + +## Optimizations + +**Halving the sketch size** + +We can in fact only include the odd terms in the sketch, and reconstruct the even ones before solving the equation to find *L*. This means the size of a sketch becomes just *c* field elements, the same size as would be needed to send its contents naively. + +To see how this is possible, we need the [Frobenius endomorphism](https://en.wikipedia.org/wiki/Frobenius_endomorphism), which in short states that in fields where *x + x = 0* it holds that *(x + y)<sup>2</sup> = x<sup>2</sup> + y<sup>2</sup>* for every *x* and *y* (the dream of every high school math student!). This means that: + +* *s<sub>2</sub> = m<sub>1</sub><sup>2</sup> + m<sub>2</sub><sup>2</sup> + ... = (m<sub>1</sub> + m<sub>2</sub> + ...)<sup>2</sup> = s<sub>1</sub><sup>2</sup>*. +* *s<sub>4</sub> = m<sub>1</sub><sup>4</sup> + m<sub>2</sub><sup>4</sup> + ... = (m<sub>1</sub><sup>2</sup> + m<sub>2</sub><sup>2</sup> + ...)<sup>2</sup> = s<sub>2</sub><sup>2</sup>*. +* *s<sub>6</sub> = m<sub>1</sub><sup>6</sup> + m<sub>2</sub><sup>6</sup> + ... = (m<sub>1</sub><sup>3</sup> + m<sub>2</sub><sup>3</sup> + ...)<sup>2</sup> = s<sub>3</sub><sup>2</sup>*. +* ... + +In other words, we only need to send *s<sub>1</sub>, s<sub>3</sub>, s<sub>5</sub>, ..., s<sub>2n-1</sub>* to recover all *2n* *s<sub>i</sub>* values, and proceed with reconstruction. + +**Quadratic performance rather than cubic** + +Using Gaussian elimination to solve the set of equations above for the *l<sub>i</sub>* values requires *O(n<sup>3</sup>)* field operations. However, due to the special structure in the equations (look at the repeated *s<sub>i</sub>* values), it can be solved in *O(n<sup>2</sup>)* time using a number of techniques, including the [Berlekamp-Massey algorithm](https://en.wikipedia.org/wiki/Berlekamp%E2%80%93Massey_algorithm) (BM). + +**Roots instead of factorization** + +As explained above, the polynomial *L* can be factored into *(1 - m<sub>i</sub>x)* factors, where the values *m<sub>i</sub>* are the set elements. However, since we know that a decodable sketch must result in a polynomial that is fully factorizable into degree-*1* factors, we can instead use a more efficient root-finding algorithm rather than a factorization algorithm. As the root of each *(1 - m<sub>i</sub>x)* factor is *m<sub>i</sub><sup>-1</sup>*, we conclude that the set elements are in fact the inverses of the roots of *L*. + +**Avoiding inversions** + +As inversions are a relatively expensive operation, it would be useful to avoid them. + +Say that we're trying to find the inverses of the roots of *L = 1 + l<sub>1</sub>x + l<sub>2</sub>x<sup>2</sup> + ... + l<sub>n</sub>x<sup>n</sup>*, then we're really interested in the solutions *y* for *1 + l<sub>1</sub>y<sup>-1</sup> + l<sub>2</sub>y<sup>-2</sup> + ... + l<sub>n</sub>y<sup>-n</sup> = 0*. By multiplying both sides in the equations with *y<sup>n</sup>*, we find *l<sub>n</sub> + l<sub>n-1</sub>y + l<sub>n-2</sub>y<sup>2</sup> + ... + y<sup>n</sup> = 0*. + +In other words, we can find the inverses of the roots of *L* by instead factoring the polynomial with the coefficients of *L* in reverse order. + +* <a name="myfootnote1">[1]</a> For those familiar with coding theory: PinSketch communicates a set difference by encoding the set members as errors in a binary [BCH](https://en.wikipedia.org/wiki/BCH_code) codeword 2<sup>bits</sup> in size and sends the syndromes. + The linearity of the syndromes provides all the properties needed for a sketch. Sketch decoding is simply finding the error locations. Decode is much faster than an ordinary BCH decoder for such a large codeword because the need to take a discrete log is avoided by storing the set in the roots directly instead of in an exponent (logically permuting the bits of the codeword). diff --git a/src/minisketch/doc/minisketch-vs.png b/src/minisketch/doc/minisketch-vs.png Binary files differnew file mode 100644 index 0000000000..aed810de8a --- /dev/null +++ b/src/minisketch/doc/minisketch-vs.png diff --git a/src/minisketch/doc/moduli.md b/src/minisketch/doc/moduli.md new file mode 100644 index 0000000000..379ac481b3 --- /dev/null +++ b/src/minisketch/doc/moduli.md @@ -0,0 +1,65 @@ +These are the irreducible polynomials over *GF(2)* used to represent field elements: + +* *x<sup>2</sup> + x + 1* +* *x<sup>3</sup> + x + 1* +* *x<sup>4</sup> + x + 1* +* *x<sup>5</sup> + x<sup>2</sup> + 1* +* *x<sup>6</sup> + x + 1* +* *x<sup>7</sup> + x + 1* +* *x<sup>8</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>9</sup> + x + 1* +* *x<sup>10</sup> + x<sup>3</sup> + 1* +* *x<sup>11</sup> + x<sup>2</sup> + 1* +* *x<sup>12</sup> + x<sup>3</sup> + 1* +* *x<sup>13</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>14</sup> + x<sup>5</sup> + 1* +* *x<sup>15</sup> + x + 1* +* *x<sup>16</sup> + x<sup>5</sup> + x<sup>3</sup> + x + 1* +* *x<sup>17</sup> + x<sup>3</sup> + 1* +* *x<sup>18</sup> + x<sup>3</sup> + 1* +* *x<sup>19</sup> + x<sup>5</sup> + x<sup>2</sup> + x + 1* +* *x<sup>20</sup> + x<sup>3</sup> + 1* +* *x<sup>21</sup> + x<sup>2</sup> + 1* +* *x<sup>22</sup> + x + 1* +* *x<sup>23</sup> + x<sup>5</sup> + 1* +* *x<sup>24</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>25</sup> + x<sup>3</sup> + 1* +* *x<sup>26</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>27</sup> + x<sup>5</sup> + x<sup>2</sup> + x + 1* +* *x<sup>28</sup> + x + 1* +* *x<sup>29</sup> + x<sup>2</sup> + 1* +* *x<sup>30</sup> + x + 1* +* *x<sup>31</sup> + x<sup>3</sup> + 1* +* *x<sup>32</sup> + x<sup>7</sup> + x<sup>3</sup> + x<sup>2</sup> + 1* +* *x<sup>33</sup> + x<sup>10</sup> + 1* +* *x<sup>34</sup> + x<sup>7</sup> + 1* +* *x<sup>35</sup> + x<sup>2</sup> + 1* +* *x<sup>36</sup> + x<sup>9</sup> + 1* +* *x<sup>37</sup> + x<sup>6</sup> + x<sup>4</sup> + x + 1* +* *x<sup>38</sup> + x<sup>6</sup> + x<sup>5</sup> + x + 1* +* *x<sup>39</sup> + x<sup>4</sup> + 1* +* *x<sup>40</sup> + x<sup>5</sup> + x<sup>4</sup> + x<sup>3</sup> + 1* +* *x<sup>41</sup> + x<sup>3</sup> + 1* +* *x<sup>42</sup> + x<sup>7</sup> + 1* +* *x<sup>43</sup> + x<sup>6</sup> + x<sup>4</sup> + x<sup>3</sup> + 1* +* *x<sup>44</sup> + x<sup>5</sup> + 1* +* *x<sup>45</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>46</sup> + x + 1* +* *x<sup>47</sup> + x<sup>5</sup> + 1* +* *x<sup>48</sup> + x<sup>5</sup> + x<sup>3</sup> + x<sup>2</sup> + 1* +* *x<sup>49</sup> + x<sup>9</sup> + 1* +* *x<sup>50</sup> + x<sup>4</sup> + x<sup>3</sup> + x<sup>2</sup> + 1* +* *x<sup>51</sup> + x<sup>6</sup> + x<sup>3</sup> + x + 1* +* *x<sup>52</sup> + x<sup>3</sup> + 1* +* *x<sup>53</sup> + x<sup>6</sup> + x<sup>2</sup> + x + 1* +* *x<sup>54</sup> + x<sup>9</sup> + 1* +* *x<sup>55</sup> + x<sup>7</sup> + 1* +* *x<sup>56</sup> + x<sup>7</sup> + x<sup>4</sup> + x<sup>2</sup> + 1* +* *x<sup>57</sup> + x<sup>4</sup> + 1* +* *x<sup>58</sup> + x<sup>19</sup> + 1* +* *x<sup>59</sup> + x<sup>7</sup> + x<sup>4</sup> + x<sup>2</sup> + 1* +* *x<sup>60</sup> + x + 1* +* *x<sup>61</sup> + x<sup>5</sup> + x<sup>2</sup> + x + 1* +* *x<sup>62</sup> + x<sup>29</sup> + 1* +* *x<sup>63</sup> + x + 1* +* *x<sup>64</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* diff --git a/src/minisketch/doc/plot_bits.png b/src/minisketch/doc/plot_bits.png Binary files differnew file mode 100644 index 0000000000..6e907d6b20 --- /dev/null +++ b/src/minisketch/doc/plot_bits.png diff --git a/src/minisketch/doc/plot_capacity.png b/src/minisketch/doc/plot_capacity.png Binary files differnew file mode 100644 index 0000000000..b4f760da36 --- /dev/null +++ b/src/minisketch/doc/plot_capacity.png diff --git a/src/minisketch/doc/plot_diff.png b/src/minisketch/doc/plot_diff.png Binary files differnew file mode 100644 index 0000000000..08ab6a86b9 --- /dev/null +++ b/src/minisketch/doc/plot_diff.png diff --git a/src/minisketch/doc/plot_size.png b/src/minisketch/doc/plot_size.png Binary files differnew file mode 100644 index 0000000000..b21921776a --- /dev/null +++ b/src/minisketch/doc/plot_size.png diff --git a/src/minisketch/doc/protocoltips.md b/src/minisketch/doc/protocoltips.md new file mode 100644 index 0000000000..610407ebc2 --- /dev/null +++ b/src/minisketch/doc/protocoltips.md @@ -0,0 +1,30 @@ +# Tips for designing protocols using `libminisketch` + +Sending a sketch is less efficient than just sending your whole set with efficient entropy coding if the number of differences is larger than *log<sub>2</sub>( 2<sup>b</sup> choose set_size ) / b*. + +In most applications your set can be hashed to entries just large enough to make the probability of collision negligible. This can be a considerable speedup and bandwidth savings. Short hashes (<128 bits) should be salted with an unpredictable value to prevent malicious inputs from intentionally causing collisions. Salting also allows an entry missed due to a collision to be reconciled on a later run with a different salt. Pre-hashing may not be possible in some applications, such as where there is only one-way communication, where the confidentiality of entry origin matters, or where security depends on the total absence of collisions. + +Some element sizes are faster to decode than others; see the benchmarks in the readme. + +Almost all the computational burden of reconciliation is in minisketch_decode(). Denial-of-service attacks can be mitigated by arranging protocol flow so that a party requests a sketch and decodes it rather than a construction where the participants will decode unsolicited sketches. Decode times can be constrained by limiting sketch capacity or via the max_count argument to minisketch_decode(). + +In most cases you don't actually know the size of the set difference in advance, but often you know a lower bound on it (the difference in set sizes). + +* There are difference size estimation techniques such as min-wise hashing<sup>[[1]](#myfootnote1)</sup> or random projections<sup>[[2]](#myfootnote2)</sup>, but complex estimators can end up using more bandwidth than they save. + +* It may be useful to always overestimate the sketch size needed to amortize communications overheads (*e.g.* packet headers, round trip delays). + +* If the total data sent would end up leaving you better off having just sent the whole set, per above, then you can send the set in response to a failure but leave out as many elements as the size of the previously sent sketch. The receiver can decode the partial set and use the data they already have to complete it, reducing bandwidth waste. + +* Additional elements can be sent for a sketch as few as one at a time with little decode cost until enough data is received to decode. This is most easily implemented by always computing the largest sketch size and sending it incrementally as needed. + +* Because sketches are linear you can adaptively subdivide to decode an overfull set. The sender uses a hash function to select approximately half their set members and sends a sketch of those members. The receiver can do the same and combine the result with the initially sent sketch to get two sketches with roughly half the number of members and attempt to decode them. Repeat recursively on failure. This adaptive subdivision procedure makes decode time essentially linear at the cost of communications inefficiency. Minisketches can also be used as the cells in an IBLT for similar reasons. + +Less efficient reconciliation techniques like IBLT or adaptive subdivision, or overheads like complex estimators effectively lower the threshold where sending the whole set efficiently would use less bandwidth. + +When the number of differences is more than 2<sup>b/2-1</sup> an alternative sketch encoding is possible that is somewhat smaller, but requires a table of size 2<sup>b</sup>; contact the authors if you have an application where that might be useful. + +## References + +* <a name="myfootnote1">[1]</a> Broder, A. *On the Resemblance and Containment of Documents* Proceedings of the Compression and Complexity of Sequences 1997 [[PDF]](https://www.cs.princeton.edu/courses/archive/spring13/cos598C/broder97resemblance.pdf) +* <a name="myfootnote2">[2]</a> Feigenbaum, Joan and Kannan, Sampath and Strauss, Martin J. and Viswanathan, Mahesh. *An Approximate L1-Difference Algorithm for Massive Data Streams* SIAM J. Comput. 2003 [[PDF]](http://www.cs.yale.edu/homes/jf/FKSV1.pdf) diff --git a/src/minisketch/include/minisketch.h b/src/minisketch/include/minisketch.h new file mode 100644 index 0000000000..0b5d8372e8 --- /dev/null +++ b/src/minisketch/include/minisketch.h @@ -0,0 +1,367 @@ +#ifndef _MINISKETCH_H_ +#define _MINISKETCH_H_ 1 + +#include <stdint.h> +#include <stdlib.h> + +#ifdef _MSC_VER +# include <compat.h> +#else +# include <unistd.h> +#endif + +#ifndef MINISKETCH_API +# if defined(_WIN32) +# ifdef MINISKETCH_BUILD +# define MINISKETCH_API __declspec(dllexport) +# else +# define MINISKETCH_API +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(MINISKETCH_BUILD) +# define MINISKETCH_API __attribute__ ((visibility ("default"))) +# else +# define MINISKETCH_API +# endif +#endif + +#ifdef __cplusplus +# if __cplusplus >= 201103L +# include <memory> +# include <vector> +# include <cassert> +# if __cplusplus >= 201703L +# include <optional> +# endif // __cplusplus >= 201703L +# endif // __cplusplus >= 201103L +extern "C" { +#endif // __cplusplus + +/** Opaque type for decoded sketches. */ +typedef struct minisketch minisketch; + +/** Determine whether support for elements of `bits` bits was compiled in. */ +MINISKETCH_API int minisketch_bits_supported(uint32_t bits); + +/** Determine the maximum number of implementations available. + * + * Multiple implementations may be available for a given element size, with + * different performance characteristics on different hardware. + * + * Each implementation is identified by a number from 0 to the output of this + * function call, inclusive. Note that not every combination of implementation + * and element size may exist (see further). +*/ +MINISKETCH_API uint32_t minisketch_implementation_max(void); + +/** Determine if the a combination of bits and implementation number is available. + * + * Returns 1 if it is, 0 otherwise. + */ +MINISKETCH_API int minisketch_implementation_supported(uint32_t bits, uint32_t implementation); + +/** Construct a sketch for a given element size, implementation and capacity. + * + * If the combination of `bits` and `implementation` is unavailable, or when + * OOM occurs, NULL is returned. If minisketch_implementation_supported + * returns 1 for the specified bits and implementation, this will always succeed + * (except when allocation fails). + * + * If the result is not NULL, it must be destroyed using minisketch_destroy. + */ +MINISKETCH_API minisketch* minisketch_create(uint32_t bits, uint32_t implementation, size_t capacity); + +/** Get the element size of a sketch in bits. */ +MINISKETCH_API uint32_t minisketch_bits(const minisketch* sketch); + +/** Get the capacity of a sketch. */ +MINISKETCH_API size_t minisketch_capacity(const minisketch* sketch); + +/** Get the implementation of a sketch. */ +MINISKETCH_API uint32_t minisketch_implementation(const minisketch* sketch); + +/** Set the seed for randomizing algorithm choices to a fixed value. + * + * By default, sketches are initialized with a random seed. This is important + * to avoid scenarios where an attacker could force worst-case behavior. + * + * This function initializes the seed to a user-provided value (any 64-bit + * integer is acceptable, regardless of field size). + * + * When seed is -1, a fixed internal value with predictable behavior is + * used. It is only intended for testing. + */ +MINISKETCH_API void minisketch_set_seed(minisketch* sketch, uint64_t seed); + +/** Clone a sketch. + * + * The result must be destroyed using minisketch_destroy. + */ +MINISKETCH_API minisketch* minisketch_clone(const minisketch* sketch); + +/** Destroy a sketch. + * + * The pointer that was passed in may not be used anymore afterwards. + */ +MINISKETCH_API void minisketch_destroy(minisketch* sketch); + +/** Compute the size in bytes for serializing a given sketch. */ +MINISKETCH_API size_t minisketch_serialized_size(const minisketch* sketch); + +/** Serialize a sketch to bytes. */ +MINISKETCH_API void minisketch_serialize(const minisketch* sketch, unsigned char* output); + +/** Deserialize a sketch from bytes. */ +MINISKETCH_API void minisketch_deserialize(minisketch* sketch, const unsigned char* input); + +/** Add an element to a sketch. + * + * If the element to be added is too large for the sketch, the most significant + * bits of the element are dropped. More precisely, if the element size of + * `sketch` is b bits, then this function adds the unsigned integer represented + * by the b least significant bits of `element` to `sketch`. + * + * If the element to be added is 0 (after potentially dropping the most significant + * bits), then this function is a no-op. Sketches cannot contain an element with + * the value 0. + * + * Note that adding the same element a second time removes it again. + */ +MINISKETCH_API void minisketch_add_uint64(minisketch* sketch, uint64_t element); + +/** Merge the elements of another sketch into this sketch. + * + * After merging, `sketch` will contain every element that existed in one but not + * both of the input sketches. It can be seen as an exclusive or operation on + * the set elements. If the capacity of `other_sketch` is lower than `sketch`'s, + * merging reduces the capacity of `sketch` to that of `other_sketch`. + * + * This function returns the capacity of `sketch` after merging has been performed + * (where this capacity is at least 1), or 0 to indicate that merging has failed because + * the two input sketches differ in their element size or implementation. If 0 is + * returned, `sketch` (and its capacity) have not been modified. + * + * It is also possible to perform this operation directly on the serializations + * of two sketches with the same element size and capacity by performing a bitwise XOR + * of the serializations. + */ +MINISKETCH_API size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch); + +/** Decode a sketch. + * + * `output` is a pointer to an array of `max_element` uint64_t's, which will be + * filled with the elements in this sketch. + * + * The return value is the number of decoded elements, or -1 if decoding failed. + */ +MINISKETCH_API ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_t* output); + +/** Compute the capacity needed to achieve a certain rate of false positives. + * + * A sketch with capacity c and no more than c elements can always be decoded + * correctly. However, if it has more than c elements, or contains just random + * bytes, it is possible that it will still decode, but the result will be + * nonsense. This can be counteracted by increasing the capacity slightly. + * + * Given a field size bits, an intended number of elements that can be decoded + * max_elements, and a false positive probability of 1 in 2**fpbits, this + * function computes the necessary capacity. It is only guaranteed to be + * accurate up to fpbits=256. + */ +MINISKETCH_API size_t minisketch_compute_capacity(uint32_t bits, size_t max_elements, uint32_t fpbits); + +/** Compute what max_elements can be decoded for a certain rate of false positives. + * + * This is the inverse operation of minisketch_compute_capacity. It determines, + * given a field size bits, a capacity of a sketch, and an acceptable false + * positive probability of 1 in 2**fpbits, what the maximum allowed + * max_elements value is. If no value of max_elements would give the desired + * false positive probability, 0 is returned. + * + * Note that this is not an exact inverse of minisketch_compute_capacity. For + * example, with bits=32, fpbits=16, and max_elements=8, + * minisketch_compute_capacity will return 9, as capacity 8 would only have a + * false positive chance of 1 in 2^15.3. Increasing the capacity to 9 however + * decreases the fp chance to 1 in 2^47.3, enough for max_elements=9 (with fp + * chance of 1 in 2^18.5). Therefore, minisketch_compute_max_elements with + * capacity=9 will return 9. + */ +MINISKETCH_API size_t minisketch_compute_max_elements(uint32_t bits, size_t capacity, uint32_t fpbits); + +#ifdef __cplusplus +} + +#if __cplusplus >= 201103L +/** Simple RAII C++11 wrapper around the minisketch API. */ +class Minisketch +{ + struct Deleter + { + void operator()(minisketch* ptr) const + { + minisketch_destroy(ptr); + } + }; + + std::unique_ptr<minisketch, Deleter> m_minisketch; + +public: + /** Check whether the library supports fields of the given size. */ + static bool BitsSupported(uint32_t bits) noexcept { return minisketch_bits_supported(bits); } + + /** Get the highest supported implementation number. */ + static uint32_t MaxImplementation() noexcept { return minisketch_implementation_max(); } + + /** Check whether the library supports fields with a given size and implementation number. + * If a particular field size `bits` is supported, implementation 0 is always supported for it. + * Higher implementation numbers may or may not be available as well, up to MaxImplementation(). + */ + static bool ImplementationSupported(uint32_t bits, uint32_t implementation) noexcept { return minisketch_implementation_supported(bits, implementation); } + + /** Given field size and a maximum number of decodable elements n, compute what capacity c to + * use so that sketches with more elements than n have a chance no higher than 2^-fpbits of + * being decoded incorrectly (and will instead fail when decoding for up to n elements). + * + * See minisketch_compute_capacity for more details. */ + static size_t ComputeCapacity(uint32_t bits, size_t max_elements, uint32_t fpbits) noexcept { return minisketch_compute_capacity(bits, max_elements, fpbits); } + + /** Reverse operation of ComputeCapacity. See minisketch_compute_max_elements. */ + static size_t ComputeMaxElements(uint32_t bits, size_t capacity, uint32_t fpbits) noexcept { return minisketch_compute_max_elements(bits, capacity, fpbits); } + + /** Construct a clone of the specified sketch. */ + Minisketch(const Minisketch& sketch) noexcept + { + if (sketch.m_minisketch) { + m_minisketch = std::unique_ptr<minisketch, Deleter>(minisketch_clone(sketch.m_minisketch.get())); + } + } + + /** Make this Minisketch a clone of the specified one. */ + Minisketch& operator=(const Minisketch& sketch) noexcept + { + if (sketch.m_minisketch) { + m_minisketch = std::unique_ptr<minisketch, Deleter>(minisketch_clone(sketch.m_minisketch.get())); + } + return *this; + } + + /** Check whether this Minisketch object is valid. */ + explicit operator bool() const noexcept { return bool{m_minisketch}; } + + /** Construct an (invalid) Minisketch object. */ + Minisketch() noexcept = default; + + /** Move constructor. */ + Minisketch(Minisketch&&) noexcept = default; + + /** Move assignment. */ + Minisketch& operator=(Minisketch&&) noexcept = default; + + /** Construct a Minisketch object with the specified parameters. + * + * If bits is not BitsSupported(), or the combination of bits and capacity is not + * ImplementationSupported(), or OOM occurs internally, an invalid Minisketch + * object will be constructed. Use operator bool() to check that this isn't the + * case before performing any other operations. */ + Minisketch(uint32_t bits, uint32_t implementation, size_t capacity) noexcept + { + m_minisketch = std::unique_ptr<minisketch, Deleter>(minisketch_create(bits, implementation, capacity)); + } + + /** Create a Minisketch object sufficiently large for the specified number of elements at given fpbits. + * It may construct an invalid object, which you may need to check for. */ + static Minisketch CreateFP(uint32_t bits, uint32_t implementation, size_t max_elements, uint32_t fpbits) noexcept + { + return Minisketch(bits, implementation, ComputeCapacity(bits, max_elements, fpbits)); + } + + /** Return the field size for a (valid) Minisketch object. */ + uint32_t GetBits() const noexcept { return minisketch_bits(m_minisketch.get()); } + + /** Return the capacity for a (valid) Minisketch object. */ + size_t GetCapacity() const noexcept { return minisketch_capacity(m_minisketch.get()); } + + /** Return the implementation number for a (valid) Minisketch object. */ + uint32_t GetImplementation() const noexcept { return minisketch_implementation(m_minisketch.get()); } + + /** Set the seed for a (valid) Minisketch object. See minisketch_set_seed(). */ + Minisketch& SetSeed(uint64_t seed) noexcept + { + minisketch_set_seed(m_minisketch.get(), seed); + return *this; + } + + /** Add (or remove, if already present) an element to a (valid) Minisketch object. + * See minisketch_add_uint64(). */ + Minisketch& Add(uint64_t element) noexcept + { + minisketch_add_uint64(m_minisketch.get(), element); + return *this; + } + + /** Merge sketch into *this; both have to be valid Minisketch objects. + * See minisketch_merge for details. */ + Minisketch& Merge(const Minisketch& sketch) noexcept + { + minisketch_merge(m_minisketch.get(), sketch.m_minisketch.get()); + return *this; + } + + /** Decode this (valid) Minisketch object into the result vector, up to as many elements as the + * vector's size permits. */ + bool Decode(std::vector<uint64_t>& result) const + { + ssize_t ret = minisketch_decode(m_minisketch.get(), result.size(), result.data()); + if (ret == -1) return false; + result.resize(ret); + return true; + } + + /** Get the serialized size in bytes for this (valid) Minisketch object.. */ + size_t GetSerializedSize() const noexcept { return minisketch_serialized_size(m_minisketch.get()); } + + /** Serialize this (valid) Minisketch object as a byte vector. */ + std::vector<unsigned char> Serialize() const + { + std::vector<unsigned char> result(GetSerializedSize()); + minisketch_serialize(m_minisketch.get(), result.data()); + return result; + } + + /** Deserialize into this (valid) Minisketch from an object containing its bytes (which has data() + * and size() members). */ + template<typename T> + Minisketch& Deserialize( + const T& obj, + typename std::enable_if< + std::is_convertible<typename std::remove_pointer<decltype(obj.data())>::type (*)[], const unsigned char (*)[]>::value && + std::is_convertible<decltype(obj.size()), std::size_t>::value, + std::nullptr_t + >::type = nullptr) noexcept + { + assert(GetSerializedSize() == obj.size()); + minisketch_deserialize(m_minisketch.get(), obj.data()); + return *this; + } + +#if __cplusplus >= 201703L + /** C++17 only: like Decode(), but up to a specified number of elements into an optional vector. */ + std::optional<std::vector<uint64_t>> Decode(size_t max_elements) const + { + std::vector<uint64_t> result(max_elements); + ssize_t ret = minisketch_decode(m_minisketch.get(), max_elements, result.data()); + if (ret == -1) return {}; + result.resize(ret); + return result; + } + + /** C++17 only: similar to Decode(), but with specified false positive probability. */ + std::optional<std::vector<uint64_t>> DecodeFP(uint32_t fpbits) const + { + return Decode(ComputeMaxElements(GetBits(), GetCapacity(), fpbits)); + } +#endif // __cplusplus >= 201703L +}; +#endif // __cplusplus >= 201103L +#endif // __cplusplus + +#endif // _MINISKETCH_H_ diff --git a/src/minisketch/sources.mk b/src/minisketch/sources.mk new file mode 100644 index 0000000000..386a4fcc23 --- /dev/null +++ b/src/minisketch/sources.mk @@ -0,0 +1,58 @@ +# - All variables are namespaced with MINISKETCH_ to avoid colliding with +# downstream makefiles. +# - All Variables ending in _HEADERS or _SOURCES confuse automake, so the +# _INT postfix is applied. +# - Convenience variables, for example a MINISKETCH_FIELDS_DIR should not be used +# as they interfere with automatic dependency generation +# - The %reldir% is the relative path from the Makefile.am. This allows +# downstreams to use these variables without having to manually account for +# the path change. + +MINISKETCH_INCLUDE_DIR_INT = %reldir%/include + +MINISKETCH_DIST_HEADERS_INT = +MINISKETCH_DIST_HEADERS_INT += %reldir%/include/minisketch.h + +MINISKETCH_LIB_HEADERS_INT = +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/false_positives.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/fielddefines.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/int_utils.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/lintrans.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/sketch.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/sketch_impl.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/util.h + +MINISKETCH_LIB_SOURCES_INT = +MINISKETCH_LIB_SOURCES_INT += %reldir%/src/minisketch.cpp + +MINISKETCH_FIELD_GENERIC_HEADERS_INT = +MINISKETCH_FIELD_GENERIC_HEADERS_INT += %reldir%/src/fields/generic_common_impl.h + +MINISKETCH_FIELD_GENERIC_SOURCES_INT = +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_1byte.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_2bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_3bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_4bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_5bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_6bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_7bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_8bytes.cpp + +MINISKETCH_FIELD_CLMUL_HEADERS_INT = +MINISKETCH_FIELD_CLMUL_HEADERS_INT += %reldir%/src/fields/clmul_common_impl.h + +MINISKETCH_FIELD_CLMUL_SOURCES_INT = +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_1byte.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_2bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_3bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_4bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_5bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_6bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_7bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_8bytes.cpp + +MINISKETCH_BENCH_SOURCES_INT = +MINISKETCH_BENCH_SOURCES_INT += %reldir%/src/bench.cpp + +MINISKETCH_TEST_SOURCES_INT = +MINISKETCH_TEST_SOURCES_INT += %reldir%/src/test.cpp diff --git a/src/minisketch/src/bench.cpp b/src/minisketch/src/bench.cpp new file mode 100644 index 0000000000..f55944a448 --- /dev/null +++ b/src/minisketch/src/bench.cpp @@ -0,0 +1,122 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include "../include/minisketch.h" +#include <string.h> +#include <memory> +#include <vector> +#include <chrono> +#include <random> +#include <set> +#include <algorithm> + +int main(int argc, char** argv) { + if (argc < 1 || argc > 4) { + printf("Usage: %s [syndromes=150] [errors=syndromes] [iters=10]\n", argv[0]); + return 1; + } + int syndromes = argc > 1 ? strtoul(argv[1], NULL, 10) : 150; + int errors = argc > 2 ? strtoul(argv[2], NULL, 10) : syndromes; + int iters = argc > 3 ? strtoul(argv[3], NULL, 10) : 10; + if (syndromes < 0 || syndromes > 1000000) { + printf("Number of syndromes (%i) out of range 0..1000000\n", syndromes); + return 1; + } + if (errors < 0) { + printf("Number of errors (%i) is negative(%i)\n", errors, syndromes); + return 1; + } + if (iters < 0 || iters > 1000000000) { + printf("Number of iterations (%i) out of range 0..1000000000\n", iters); + return 1; + } + uint32_t max_impl = minisketch_implementation_max(); + for (int bits = 2; bits <= 64; ++bits) { + if (errors > pow(2.0, bits - 1)) continue; + if (!minisketch_bits_supported(bits)) continue; + printf("recover[ms]\t% 3i\t", bits); + for (uint32_t impl = 0; impl <= max_impl; ++impl) { + std::vector<minisketch*> states; + std::vector<uint64_t> roots(2 * syndromes); + std::random_device rng; + std::uniform_int_distribution<uint64_t> dist(1, (uint64_t(1) << bits) - 1); + states.resize(iters); + std::vector<double> benches; + benches.reserve(iters); + for (int i = 0; i < iters; ++i) { + states[i] = minisketch_create(bits, impl, syndromes); + if (!states[i]) break; + std::set<uint64_t> done; + for (int j = 0; j < errors; ++j) { + uint64_t r; + do { + r = dist(rng); + } while (done.count(r)); + done.insert(r); + minisketch_add_uint64(states[i], r); + } + } + if (!states[0]) { + printf(" -\t"); + } else { + double total = 0.0; + for (auto& state : states) { + auto start = std::chrono::steady_clock::now(); + minisketch_decode(state, 2 * syndromes, roots.data()); + auto stop = std::chrono::steady_clock::now(); + std::chrono::duration<double> dur(stop - start); + total += dur.count(); + benches.push_back(dur.count()); + } + std::sort(benches.begin(), benches.end()); + printf("% 10.5f\t", benches[0] * 1000.0); + } + for (auto& state : states) { + minisketch_destroy(state); + } + } + printf("\n"); + printf("create[ns]\t% 3i\t", bits); + for (uint32_t impl = 0; impl <= max_impl; ++impl) { + std::vector<minisketch*> states; + std::random_device rng; + std::uniform_int_distribution<uint64_t> dist; + std::vector<uint64_t> data; + data.resize(errors * 10); + states.resize(iters); + std::vector<double> benches; + benches.reserve(iters); + for (int i = 0; i < iters; ++i) { + states[i] = minisketch_create(bits, impl, syndromes); + } + for (size_t i = 0; i < data.size(); ++i) { + data[i] = dist(rng); + } + if (!states[0]) { + printf(" -\t"); + } else { + double total = 0.0; + for (auto& state : states) { + auto start = std::chrono::steady_clock::now(); + for (auto val : data) { + minisketch_add_uint64(state, val); + } + auto stop = std::chrono::steady_clock::now(); + std::chrono::duration<double> dur(stop - start); + total += dur.count(); + benches.push_back(dur.count()); + } + std::sort(benches.begin(), benches.end()); + printf("% 10.5f\t", benches[0] * 1000000000.0 / data.size() / syndromes); + } + for (auto& state : states) { + minisketch_destroy(state); + } + } + printf("\n"); + } + return 0; +} diff --git a/src/minisketch/src/false_positives.h b/src/minisketch/src/false_positives.h new file mode 100644 index 0000000000..44ebb3e94c --- /dev/null +++ b/src/minisketch/src/false_positives.h @@ -0,0 +1,110 @@ +/********************************************************************** + * Copyright (c) 2020 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_FALSE_POSITIVES_H_ +#define _MINISKETCH_FALSE_POSITIVES_H_ + +#include "util.h" + +#include "int_utils.h" + +#include <stdint.h> + +namespace { + +/** Compute floor(log2(x!)), exactly up to x=57; an underestimate up to x=2^32-1. */ +uint64_t Log2Factorial(uint32_t x) { + //! Values of floor(106*log2(1 + i/32)) for i=0..31 + static constexpr uint8_t T[32] = { + 0, 4, 9, 13, 18, 22, 26, 30, 34, 37, 41, 45, 48, 52, 55, 58, 62, 65, 68, + 71, 74, 77, 80, 82, 85, 88, 90, 93, 96, 98, 101, 103 + }; + int bits = CountBits(x, 32); + // Compute an (under)estimate of floor(106*log2(x)). + // This works by relying on floor(log2(x)) = countbits(x)-1, and adding + // precision using the top 6 bits of x (the highest one of which is always + // one). + unsigned l2_106 = 106 * (bits - 1) + T[((x << (32 - bits)) >> 26) & 31]; + // Based on Stirling approximation for log2(x!): + // log2(x!) = log(x!) / log(2) + // = ((x + 1/2) * log(x) - x + log(2*pi)/2 + ...) / log(2) + // = (x + 1/2) * log2(x) - x/log(2) + log2(2*pi)/2 + ... + // = 1/2*(2*x+1)*log2(x) - (1/log(2))*x + log2(2*pi)/2 + ... + // = 1/212*(2*x+1)*(106*log2(x)) + (-1/log(2))*x + log2(2*pi)/2 + ... + // where 418079/88632748 is exactly 1/212 + // -127870026/88632748 is slightly less than -1/log(2) + // 117504694/88632748 is less than log2(2*pi)/2 + // A correction term is only needed for x < 3. + // + // See doc/log2_factorial.sage for how these constants were obtained. + return (418079 * (2 * uint64_t{x} + 1) * l2_106 - 127870026 * uint64_t{x} + 117504694 + 88632748 * (x < 3)) / 88632748; +} + +/** Compute floor(log2(2^(bits * capacity) / sum((2^bits - 1) choose k, k=0..capacity))), for bits>1 + * + * See doc/gen_basefpbits.sage for how the tables were obtained. */ +uint64_t BaseFPBits(uint32_t bits, uint32_t capacity) { + // Correction table for low bits/capacities + static constexpr uint8_t ADD5[] = {1, 1, 1, 1, 2, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12}; + static constexpr uint8_t ADD6[] = {1, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 5, 6, 6, 6, 7, 8, 8, 10, 10, 11, 12, 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 20, 20, 21, 21, 22, 22, 23, 23, 23, 24, 24, 24, 24}; + static constexpr uint8_t ADD7[] = {1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 8, 7, 8, 9, 9, 9, 10, 11, 11, 12, 12, 13, 13, 15, 15, 15, 16, 17, 17, 18, 19, 20, 20}; + static constexpr uint8_t ADD8[] = {1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 3, 4, 4, 5, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9}; + static constexpr uint8_t ADD9[] = {1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 3, 2, 3, 3, 3, 3, 4, 3, 3, 4, 4, 4, 4}; + + if (capacity == 0) return 0; + uint64_t ret = 0; + if (bits < 32 && capacity >= (1U << bits)) { + ret = uint64_t{bits} * (capacity - (1U << bits) + 1); + capacity = (1U << bits) - 1; + } + ret += Log2Factorial(capacity); + switch (bits) { + case 2: return ret + (capacity <= 2 ? 0 : 1); + case 3: return ret + (capacity <= 2 ? 0 : (0x2a5 >> 2 * (capacity - 3)) & 3); + case 4: return ret + (capacity <= 3 ? 0 : (0xb6d91a449 >> 3 * (capacity - 4)) & 7); + case 5: return ret + (capacity <= 4 ? 0 : ADD5[capacity - 5]); + case 6: return ret + (capacity <= 4 ? 0 : capacity > 54 ? 25 : ADD6[capacity - 5]); + case 7: return ret + (capacity <= 4 ? 0 : capacity > 57 ? 21 : ADD7[capacity - 5]); + case 8: return ret + (capacity <= 9 ? 0 : capacity > 56 ? 10 : ADD8[capacity - 10]); + case 9: return ret + (capacity <= 11 ? 0 : capacity > 54 ? 5 : ADD9[capacity - 12]); + case 10: return ret + (capacity <= 21 ? 0 : capacity > 50 ? 2 : (0x1a6665545555041 >> 2 * (capacity - 22)) & 3); + case 11: return ret + (capacity <= 21 ? 0 : capacity > 45 ? 1 : (0x5b3dc1 >> (capacity - 22)) & 1); + case 12: return ret + (capacity <= 21 ? 0 : capacity > 57 ? 0 : (0xe65522041 >> (capacity - 22)) & 1); + case 13: return ret + (capacity <= 27 ? 0 : capacity > 55 ? 0 : (0x8904081 >> (capacity - 28)) & 1); + case 14: return ret + (capacity <= 47 ? 0 : capacity > 48 ? 0 : 1); + default: return ret; + } +} + +size_t ComputeCapacity(uint32_t bits, size_t max_elements, uint32_t fpbits) { + if (bits == 0) return 0; + uint64_t base_fpbits = BaseFPBits(bits, max_elements); + // The fpbits provided by the base max_elements==capacity case are sufficient. + if (base_fpbits >= fpbits) return max_elements; + // Otherwise, increment capacity by ceil(fpbits / bits) beyond that. + return max_elements + (fpbits - base_fpbits + bits - 1) / bits; +} + +size_t ComputeMaxElements(uint32_t bits, size_t capacity, uint32_t fpbits) { + if (bits == 0) return 0; + // Start with max_elements=capacity, and decrease max_elements until the corresponding capacity is capacity. + size_t max_elements = capacity; + while (true) { + size_t capacity_for_max_elements = ComputeCapacity(bits, max_elements, fpbits); + CHECK_SAFE(capacity_for_max_elements >= capacity); + if (capacity_for_max_elements <= capacity) return max_elements; + size_t adjust = capacity_for_max_elements - capacity; + // Decrementing max_elements by N will at most decrement the corresponding capacity by N. + // As the observed capacity is adjust too high, we can safely decrease max_elements by adjust. + // If that brings us into negative max_elements territory, no solution exists and we return 0. + if (max_elements < adjust) return 0; + max_elements -= adjust; + } +} + +} // namespace + +#endif diff --git a/src/minisketch/src/fielddefines.h b/src/minisketch/src/fielddefines.h new file mode 100644 index 0000000000..510cb81f42 --- /dev/null +++ b/src/minisketch/src/fielddefines.h @@ -0,0 +1,560 @@ +/********************************************************************** + * Copyright (c) 2021 Cory Fields * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_FIELDDEFINES_H_ +#define _MINISKETCH_FIELDDEFINES_H_ + +/* + +This file translates external defines ENABLE_FIELD_FOO, DISABLE_FIELD_FOO, and +DISABLE_DEFAULT_FIELDS to internal ones: ENABLE_FIELD_INT_FOO. Only the +resulting internal includes should be used. + +Default: All fields enabled +-DDISABLE_FIELD_3: All fields except 3 are enabled +-DENABLE_FIELD_3: All fields enabled +-DDISABLE_DEFAULT_FIELDS: Error, no fields enabled +-DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_3: Only field 3 enabled +-DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_2 -DENABLE_FIELD_3: Only fields 2 and 3 are enabled +-DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_2 -DENABLE_FIELD_3 -DDISABLE_FIELD_3: Only field 2 enabled + +*/ + +#ifdef DISABLE_DEFAULT_FIELDS +#if defined(ENABLE_FIELD_2) && !defined(DISABLE_FIELD_2) +#define ENABLE_FIELD_INT_2 +#endif +#if defined(ENABLE_FIELD_3) && !defined(DISABLE_FIELD_3) +#define ENABLE_FIELD_INT_3 +#endif +#if defined(ENABLE_FIELD_4) && !defined(DISABLE_FIELD_4) +#define ENABLE_FIELD_INT_4 +#endif +#if defined(ENABLE_FIELD_5) && !defined(DISABLE_FIELD_5) +#define ENABLE_FIELD_INT_5 +#endif +#if defined(ENABLE_FIELD_6) && !defined(DISABLE_FIELD_6) +#define ENABLE_FIELD_INT_6 +#endif +#if defined(ENABLE_FIELD_7) && !defined(DISABLE_FIELD_7) +#define ENABLE_FIELD_INT_7 +#endif +#if defined(ENABLE_FIELD_8) && !defined(DISABLE_FIELD_8) +#define ENABLE_FIELD_INT_8 +#endif +#if defined(ENABLE_FIELD_9) && !defined(DISABLE_FIELD_9) +#define ENABLE_FIELD_INT_9 +#endif +#if defined(ENABLE_FIELD_10) && !defined(DISABLE_FIELD_10) +#define ENABLE_FIELD_INT_10 +#endif +#if defined(ENABLE_FIELD_11) && !defined(DISABLE_FIELD_11) +#define ENABLE_FIELD_INT_11 +#endif +#if defined(ENABLE_FIELD_12) && !defined(DISABLE_FIELD_12) +#define ENABLE_FIELD_INT_12 +#endif +#if defined(ENABLE_FIELD_13) && !defined(DISABLE_FIELD_13) +#define ENABLE_FIELD_INT_13 +#endif +#if defined(ENABLE_FIELD_14) && !defined(DISABLE_FIELD_14) +#define ENABLE_FIELD_INT_14 +#endif +#if defined(ENABLE_FIELD_15) && !defined(DISABLE_FIELD_15) +#define ENABLE_FIELD_INT_15 +#endif +#if defined(ENABLE_FIELD_16) && !defined(DISABLE_FIELD_16) +#define ENABLE_FIELD_INT_16 +#endif +#if defined(ENABLE_FIELD_17) && !defined(DISABLE_FIELD_17) +#define ENABLE_FIELD_INT_17 +#endif +#if defined(ENABLE_FIELD_18) && !defined(DISABLE_FIELD_18) +#define ENABLE_FIELD_INT_18 +#endif +#if defined(ENABLE_FIELD_19) && !defined(DISABLE_FIELD_19) +#define ENABLE_FIELD_INT_19 +#endif +#if defined(ENABLE_FIELD_20) && !defined(DISABLE_FIELD_20) +#define ENABLE_FIELD_INT_20 +#endif +#if defined(ENABLE_FIELD_21) && !defined(DISABLE_FIELD_21) +#define ENABLE_FIELD_INT_21 +#endif +#if defined(ENABLE_FIELD_22) && !defined(DISABLE_FIELD_22) +#define ENABLE_FIELD_INT_22 +#endif +#if defined(ENABLE_FIELD_23) && !defined(DISABLE_FIELD_23) +#define ENABLE_FIELD_INT_23 +#endif +#if defined(ENABLE_FIELD_24) && !defined(DISABLE_FIELD_24) +#define ENABLE_FIELD_INT_24 +#endif +#if defined(ENABLE_FIELD_25) && !defined(DISABLE_FIELD_25) +#define ENABLE_FIELD_INT_25 +#endif +#if defined(ENABLE_FIELD_26) && !defined(DISABLE_FIELD_26) +#define ENABLE_FIELD_INT_26 +#endif +#if defined(ENABLE_FIELD_27) && !defined(DISABLE_FIELD_27) +#define ENABLE_FIELD_INT_27 +#endif +#if defined(ENABLE_FIELD_28) && !defined(DISABLE_FIELD_28) +#define ENABLE_FIELD_INT_28 +#endif +#if defined(ENABLE_FIELD_29) && !defined(DISABLE_FIELD_29) +#define ENABLE_FIELD_INT_29 +#endif +#if defined(ENABLE_FIELD_30) && !defined(DISABLE_FIELD_30) +#define ENABLE_FIELD_INT_30 +#endif +#if defined(ENABLE_FIELD_31) && !defined(DISABLE_FIELD_31) +#define ENABLE_FIELD_INT_31 +#endif +#if defined(ENABLE_FIELD_32) && !defined(DISABLE_FIELD_32) +#define ENABLE_FIELD_INT_32 +#endif +#if defined(ENABLE_FIELD_33) && !defined(DISABLE_FIELD_33) +#define ENABLE_FIELD_INT_33 +#endif +#if defined(ENABLE_FIELD_34) && !defined(DISABLE_FIELD_34) +#define ENABLE_FIELD_INT_34 +#endif +#if defined(ENABLE_FIELD_35) && !defined(DISABLE_FIELD_35) +#define ENABLE_FIELD_INT_35 +#endif +#if defined(ENABLE_FIELD_36) && !defined(DISABLE_FIELD_36) +#define ENABLE_FIELD_INT_36 +#endif +#if defined(ENABLE_FIELD_37) && !defined(DISABLE_FIELD_37) +#define ENABLE_FIELD_INT_37 +#endif +#if defined(ENABLE_FIELD_38) && !defined(DISABLE_FIELD_38) +#define ENABLE_FIELD_INT_38 +#endif +#if defined(ENABLE_FIELD_39) && !defined(DISABLE_FIELD_39) +#define ENABLE_FIELD_INT_39 +#endif +#if defined(ENABLE_FIELD_40) && !defined(DISABLE_FIELD_40) +#define ENABLE_FIELD_INT_40 +#endif +#if defined(ENABLE_FIELD_41) && !defined(DISABLE_FIELD_41) +#define ENABLE_FIELD_INT_41 +#endif +#if defined(ENABLE_FIELD_42) && !defined(DISABLE_FIELD_42) +#define ENABLE_FIELD_INT_42 +#endif +#if defined(ENABLE_FIELD_43) && !defined(DISABLE_FIELD_43) +#define ENABLE_FIELD_INT_43 +#endif +#if defined(ENABLE_FIELD_44) && !defined(DISABLE_FIELD_44) +#define ENABLE_FIELD_INT_44 +#endif +#if defined(ENABLE_FIELD_45) && !defined(DISABLE_FIELD_45) +#define ENABLE_FIELD_INT_45 +#endif +#if defined(ENABLE_FIELD_46) && !defined(DISABLE_FIELD_46) +#define ENABLE_FIELD_INT_46 +#endif +#if defined(ENABLE_FIELD_47) && !defined(DISABLE_FIELD_47) +#define ENABLE_FIELD_INT_47 +#endif +#if defined(ENABLE_FIELD_48) && !defined(DISABLE_FIELD_48) +#define ENABLE_FIELD_INT_48 +#endif +#if defined(ENABLE_FIELD_49) && !defined(DISABLE_FIELD_49) +#define ENABLE_FIELD_INT_49 +#endif +#if defined(ENABLE_FIELD_50) && !defined(DISABLE_FIELD_50) +#define ENABLE_FIELD_INT_50 +#endif +#if defined(ENABLE_FIELD_51) && !defined(DISABLE_FIELD_51) +#define ENABLE_FIELD_INT_51 +#endif +#if defined(ENABLE_FIELD_52) && !defined(DISABLE_FIELD_52) +#define ENABLE_FIELD_INT_52 +#endif +#if defined(ENABLE_FIELD_53) && !defined(DISABLE_FIELD_53) +#define ENABLE_FIELD_INT_53 +#endif +#if defined(ENABLE_FIELD_54) && !defined(DISABLE_FIELD_54) +#define ENABLE_FIELD_INT_54 +#endif +#if defined(ENABLE_FIELD_55) && !defined(DISABLE_FIELD_55) +#define ENABLE_FIELD_INT_55 +#endif +#if defined(ENABLE_FIELD_56) && !defined(DISABLE_FIELD_56) +#define ENABLE_FIELD_INT_56 +#endif +#if defined(ENABLE_FIELD_57) && !defined(DISABLE_FIELD_57) +#define ENABLE_FIELD_INT_57 +#endif +#if defined(ENABLE_FIELD_58) && !defined(DISABLE_FIELD_58) +#define ENABLE_FIELD_INT_58 +#endif +#if defined(ENABLE_FIELD_59) && !defined(DISABLE_FIELD_59) +#define ENABLE_FIELD_INT_59 +#endif +#if defined(ENABLE_FIELD_60) && !defined(DISABLE_FIELD_60) +#define ENABLE_FIELD_INT_60 +#endif +#if defined(ENABLE_FIELD_61) && !defined(DISABLE_FIELD_61) +#define ENABLE_FIELD_INT_61 +#endif +#if defined(ENABLE_FIELD_62) && !defined(DISABLE_FIELD_62) +#define ENABLE_FIELD_INT_62 +#endif +#if defined(ENABLE_FIELD_63) && !defined(DISABLE_FIELD_63) +#define ENABLE_FIELD_INT_63 +#endif +#if defined(ENABLE_FIELD_64) && !defined(DISABLE_FIELD_64) +#define ENABLE_FIELD_INT_64 +#endif +#else +#if !defined(DISABLE_FIELD_2) +#define ENABLE_FIELD_INT_2 +#endif +#if !defined(DISABLE_FIELD_3) +#define ENABLE_FIELD_INT_3 +#endif +#if !defined(DISABLE_FIELD_4) +#define ENABLE_FIELD_INT_4 +#endif +#if !defined(DISABLE_FIELD_5) +#define ENABLE_FIELD_INT_5 +#endif +#if !defined(DISABLE_FIELD_6) +#define ENABLE_FIELD_INT_6 +#endif +#if !defined(DISABLE_FIELD_7) +#define ENABLE_FIELD_INT_7 +#endif +#if !defined(DISABLE_FIELD_8) +#define ENABLE_FIELD_INT_8 +#endif +#if !defined(DISABLE_FIELD_9) +#define ENABLE_FIELD_INT_9 +#endif +#if !defined(DISABLE_FIELD_10) +#define ENABLE_FIELD_INT_10 +#endif +#if !defined(DISABLE_FIELD_11) +#define ENABLE_FIELD_INT_11 +#endif +#if !defined(DISABLE_FIELD_12) +#define ENABLE_FIELD_INT_12 +#endif +#if !defined(DISABLE_FIELD_13) +#define ENABLE_FIELD_INT_13 +#endif +#if !defined(DISABLE_FIELD_14) +#define ENABLE_FIELD_INT_14 +#endif +#if !defined(DISABLE_FIELD_15) +#define ENABLE_FIELD_INT_15 +#endif +#if !defined(DISABLE_FIELD_16) +#define ENABLE_FIELD_INT_16 +#endif +#if !defined(DISABLE_FIELD_17) +#define ENABLE_FIELD_INT_17 +#endif +#if !defined(DISABLE_FIELD_18) +#define ENABLE_FIELD_INT_18 +#endif +#if !defined(DISABLE_FIELD_19) +#define ENABLE_FIELD_INT_19 +#endif +#if !defined(DISABLE_FIELD_20) +#define ENABLE_FIELD_INT_20 +#endif +#if !defined(DISABLE_FIELD_21) +#define ENABLE_FIELD_INT_21 +#endif +#if !defined(DISABLE_FIELD_22) +#define ENABLE_FIELD_INT_22 +#endif +#if !defined(DISABLE_FIELD_23) +#define ENABLE_FIELD_INT_23 +#endif +#if !defined(DISABLE_FIELD_24) +#define ENABLE_FIELD_INT_24 +#endif +#if !defined(DISABLE_FIELD_25) +#define ENABLE_FIELD_INT_25 +#endif +#if !defined(DISABLE_FIELD_26) +#define ENABLE_FIELD_INT_26 +#endif +#if !defined(DISABLE_FIELD_27) +#define ENABLE_FIELD_INT_27 +#endif +#if !defined(DISABLE_FIELD_28) +#define ENABLE_FIELD_INT_28 +#endif +#if !defined(DISABLE_FIELD_29) +#define ENABLE_FIELD_INT_29 +#endif +#if !defined(DISABLE_FIELD_30) +#define ENABLE_FIELD_INT_30 +#endif +#if !defined(DISABLE_FIELD_31) +#define ENABLE_FIELD_INT_31 +#endif +#if !defined(DISABLE_FIELD_32) +#define ENABLE_FIELD_INT_32 +#endif +#if !defined(DISABLE_FIELD_33) +#define ENABLE_FIELD_INT_33 +#endif +#if !defined(DISABLE_FIELD_34) +#define ENABLE_FIELD_INT_34 +#endif +#if !defined(DISABLE_FIELD_35) +#define ENABLE_FIELD_INT_35 +#endif +#if !defined(DISABLE_FIELD_36) +#define ENABLE_FIELD_INT_36 +#endif +#if !defined(DISABLE_FIELD_37) +#define ENABLE_FIELD_INT_37 +#endif +#if !defined(DISABLE_FIELD_38) +#define ENABLE_FIELD_INT_38 +#endif +#if !defined(DISABLE_FIELD_39) +#define ENABLE_FIELD_INT_39 +#endif +#if !defined(DISABLE_FIELD_40) +#define ENABLE_FIELD_INT_40 +#endif +#if !defined(DISABLE_FIELD_41) +#define ENABLE_FIELD_INT_41 +#endif +#if !defined(DISABLE_FIELD_42) +#define ENABLE_FIELD_INT_42 +#endif +#if !defined(DISABLE_FIELD_43) +#define ENABLE_FIELD_INT_43 +#endif +#if !defined(DISABLE_FIELD_44) +#define ENABLE_FIELD_INT_44 +#endif +#if !defined(DISABLE_FIELD_45) +#define ENABLE_FIELD_INT_45 +#endif +#if !defined(DISABLE_FIELD_46) +#define ENABLE_FIELD_INT_46 +#endif +#if !defined(DISABLE_FIELD_47) +#define ENABLE_FIELD_INT_47 +#endif +#if !defined(DISABLE_FIELD_48) +#define ENABLE_FIELD_INT_48 +#endif +#if !defined(DISABLE_FIELD_49) +#define ENABLE_FIELD_INT_49 +#endif +#if !defined(DISABLE_FIELD_50) +#define ENABLE_FIELD_INT_50 +#endif +#if !defined(DISABLE_FIELD_51) +#define ENABLE_FIELD_INT_51 +#endif +#if !defined(DISABLE_FIELD_52) +#define ENABLE_FIELD_INT_52 +#endif +#if !defined(DISABLE_FIELD_53) +#define ENABLE_FIELD_INT_53 +#endif +#if !defined(DISABLE_FIELD_54) +#define ENABLE_FIELD_INT_54 +#endif +#if !defined(DISABLE_FIELD_55) +#define ENABLE_FIELD_INT_55 +#endif +#if !defined(DISABLE_FIELD_56) +#define ENABLE_FIELD_INT_56 +#endif +#if !defined(DISABLE_FIELD_57) +#define ENABLE_FIELD_INT_57 +#endif +#if !defined(DISABLE_FIELD_58) +#define ENABLE_FIELD_INT_58 +#endif +#if !defined(DISABLE_FIELD_59) +#define ENABLE_FIELD_INT_59 +#endif +#if !defined(DISABLE_FIELD_60) +#define ENABLE_FIELD_INT_60 +#endif +#if !defined(DISABLE_FIELD_61) +#define ENABLE_FIELD_INT_61 +#endif +#if !defined(DISABLE_FIELD_62) +#define ENABLE_FIELD_INT_62 +#endif +#if !defined(DISABLE_FIELD_63) +#define ENABLE_FIELD_INT_63 +#endif +#if !defined(DISABLE_FIELD_64) +#define ENABLE_FIELD_INT_64 +#endif +#endif + +#if !defined(ENABLE_FIELD_INT_2) && \ + !defined(ENABLE_FIELD_INT_3) && \ + !defined(ENABLE_FIELD_INT_4) && \ + !defined(ENABLE_FIELD_INT_5) && \ + !defined(ENABLE_FIELD_INT_6) && \ + !defined(ENABLE_FIELD_INT_7) && \ + !defined(ENABLE_FIELD_INT_8) && \ + !defined(ENABLE_FIELD_INT_9) && \ + !defined(ENABLE_FIELD_INT_10) && \ + !defined(ENABLE_FIELD_INT_11) && \ + !defined(ENABLE_FIELD_INT_12) && \ + !defined(ENABLE_FIELD_INT_13) && \ + !defined(ENABLE_FIELD_INT_14) && \ + !defined(ENABLE_FIELD_INT_15) && \ + !defined(ENABLE_FIELD_INT_16) && \ + !defined(ENABLE_FIELD_INT_17) && \ + !defined(ENABLE_FIELD_INT_18) && \ + !defined(ENABLE_FIELD_INT_19) && \ + !defined(ENABLE_FIELD_INT_20) && \ + !defined(ENABLE_FIELD_INT_21) && \ + !defined(ENABLE_FIELD_INT_22) && \ + !defined(ENABLE_FIELD_INT_23) && \ + !defined(ENABLE_FIELD_INT_24) && \ + !defined(ENABLE_FIELD_INT_25) && \ + !defined(ENABLE_FIELD_INT_26) && \ + !defined(ENABLE_FIELD_INT_27) && \ + !defined(ENABLE_FIELD_INT_28) && \ + !defined(ENABLE_FIELD_INT_29) && \ + !defined(ENABLE_FIELD_INT_30) && \ + !defined(ENABLE_FIELD_INT_31) && \ + !defined(ENABLE_FIELD_INT_32) && \ + !defined(ENABLE_FIELD_INT_33) && \ + !defined(ENABLE_FIELD_INT_34) && \ + !defined(ENABLE_FIELD_INT_35) && \ + !defined(ENABLE_FIELD_INT_36) && \ + !defined(ENABLE_FIELD_INT_37) && \ + !defined(ENABLE_FIELD_INT_38) && \ + !defined(ENABLE_FIELD_INT_39) && \ + !defined(ENABLE_FIELD_INT_40) && \ + !defined(ENABLE_FIELD_INT_41) && \ + !defined(ENABLE_FIELD_INT_42) && \ + !defined(ENABLE_FIELD_INT_43) && \ + !defined(ENABLE_FIELD_INT_44) && \ + !defined(ENABLE_FIELD_INT_45) && \ + !defined(ENABLE_FIELD_INT_46) && \ + !defined(ENABLE_FIELD_INT_47) && \ + !defined(ENABLE_FIELD_INT_48) && \ + !defined(ENABLE_FIELD_INT_49) && \ + !defined(ENABLE_FIELD_INT_50) && \ + !defined(ENABLE_FIELD_INT_51) && \ + !defined(ENABLE_FIELD_INT_52) && \ + !defined(ENABLE_FIELD_INT_53) && \ + !defined(ENABLE_FIELD_INT_54) && \ + !defined(ENABLE_FIELD_INT_55) && \ + !defined(ENABLE_FIELD_INT_56) && \ + !defined(ENABLE_FIELD_INT_57) && \ + !defined(ENABLE_FIELD_INT_58) && \ + !defined(ENABLE_FIELD_INT_59) && \ + !defined(ENABLE_FIELD_INT_60) && \ + !defined(ENABLE_FIELD_INT_61) && \ + !defined(ENABLE_FIELD_INT_62) && \ + !defined(ENABLE_FIELD_INT_63) && \ + !defined(ENABLE_FIELD_INT_64) +#error No fields enabled +#endif + +#if defined(ENABLE_FIELD_INT_2) || \ + defined(ENABLE_FIELD_INT_3) || \ + defined(ENABLE_FIELD_INT_4) || \ + defined(ENABLE_FIELD_INT_5) || \ + defined(ENABLE_FIELD_INT_6) || \ + defined(ENABLE_FIELD_INT_7) || \ + defined(ENABLE_FIELD_INT_8) +#define ENABLE_FIELD_BYTES_INT_1 +#endif + +#if defined(ENABLE_FIELD_INT_9) || \ + defined(ENABLE_FIELD_INT_10) || \ + defined(ENABLE_FIELD_INT_11) || \ + defined(ENABLE_FIELD_INT_12) || \ + defined(ENABLE_FIELD_INT_13) || \ + defined(ENABLE_FIELD_INT_14) || \ + defined(ENABLE_FIELD_INT_15) || \ + defined(ENABLE_FIELD_INT_16) +#define ENABLE_FIELD_BYTES_INT_2 +#endif + +#if defined(ENABLE_FIELD_INT_17) || \ + defined(ENABLE_FIELD_INT_18) || \ + defined(ENABLE_FIELD_INT_19) || \ + defined(ENABLE_FIELD_INT_20) || \ + defined(ENABLE_FIELD_INT_21) || \ + defined(ENABLE_FIELD_INT_22) || \ + defined(ENABLE_FIELD_INT_23) || \ + defined(ENABLE_FIELD_INT_24) +#define ENABLE_FIELD_BYTES_INT_3 +#endif + +#if defined(ENABLE_FIELD_INT_25) || \ + defined(ENABLE_FIELD_INT_26) || \ + defined(ENABLE_FIELD_INT_27) || \ + defined(ENABLE_FIELD_INT_28) || \ + defined(ENABLE_FIELD_INT_29) || \ + defined(ENABLE_FIELD_INT_30) || \ + defined(ENABLE_FIELD_INT_31) || \ + defined(ENABLE_FIELD_INT_32) +#define ENABLE_FIELD_BYTES_INT_4 +#endif + +#if defined(ENABLE_FIELD_INT_33) || \ + defined(ENABLE_FIELD_INT_34) || \ + defined(ENABLE_FIELD_INT_35) || \ + defined(ENABLE_FIELD_INT_36) || \ + defined(ENABLE_FIELD_INT_37) || \ + defined(ENABLE_FIELD_INT_38) || \ + defined(ENABLE_FIELD_INT_39) || \ + defined(ENABLE_FIELD_INT_40) +#define ENABLE_FIELD_BYTES_INT_5 +#endif + +#if defined(ENABLE_FIELD_INT_41) || \ + defined(ENABLE_FIELD_INT_42) || \ + defined(ENABLE_FIELD_INT_43) || \ + defined(ENABLE_FIELD_INT_44) || \ + defined(ENABLE_FIELD_INT_45) || \ + defined(ENABLE_FIELD_INT_46) || \ + defined(ENABLE_FIELD_INT_47) || \ + defined(ENABLE_FIELD_INT_48) +#define ENABLE_FIELD_BYTES_INT_6 +#endif + +#if defined(ENABLE_FIELD_INT_49) || \ + defined(ENABLE_FIELD_INT_50) || \ + defined(ENABLE_FIELD_INT_51) || \ + defined(ENABLE_FIELD_INT_52) || \ + defined(ENABLE_FIELD_INT_53) || \ + defined(ENABLE_FIELD_INT_54) || \ + defined(ENABLE_FIELD_INT_55) || \ + defined(ENABLE_FIELD_INT_56) +#define ENABLE_FIELD_BYTES_INT_7 +#endif + +#if defined(ENABLE_FIELD_INT_57) || \ + defined(ENABLE_FIELD_INT_58) || \ + defined(ENABLE_FIELD_INT_59) || \ + defined(ENABLE_FIELD_INT_60) || \ + defined(ENABLE_FIELD_INT_61) || \ + defined(ENABLE_FIELD_INT_62) || \ + defined(ENABLE_FIELD_INT_63) || \ + defined(ENABLE_FIELD_INT_64) +#define ENABLE_FIELD_BYTES_INT_8 +#endif +#endif // _MINISKETCH_FIELDDEFINES_H_ diff --git a/src/minisketch/src/fields/clmul_1byte.cpp b/src/minisketch/src/fields/clmul_1byte.cpp new file mode 100644 index 0000000000..8826af9605 --- /dev/null +++ b/src/minisketch/src/fields/clmul_1byte.cpp @@ -0,0 +1,119 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_1) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_2 +// 2 bit field +typedef RecLinTrans<uint8_t, 2> StatTableTRI2; +constexpr StatTableTRI2 SQR_TABLE_TRI2({0x1, 0x3}); +constexpr StatTableTRI2 QRT_TABLE_TRI2({0x2, 0}); +typedef FieldTri<uint8_t, 2, 1, StatTableTRI2, &SQR_TABLE_TRI2, &QRT_TABLE_TRI2, &QRT_TABLE_TRI2, &QRT_TABLE_TRI2, &QRT_TABLE_TRI2, &QRT_TABLE_TRI2, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri2; +#endif + +#ifdef ENABLE_FIELD_INT_3 +// 3 bit field +typedef RecLinTrans<uint8_t, 3> StatTableTRI3; +constexpr StatTableTRI3 SQR_TABLE_TRI3({0x1, 0x4, 0x6}); +constexpr StatTableTRI3 QRT_TABLE_TRI3({0, 0x4, 0x6}); +typedef FieldTri<uint8_t, 3, 1, StatTableTRI3, &SQR_TABLE_TRI3, &QRT_TABLE_TRI3, &QRT_TABLE_TRI3, &QRT_TABLE_TRI3, &QRT_TABLE_TRI3, &QRT_TABLE_TRI3, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri3; +#endif + +#ifdef ENABLE_FIELD_INT_4 +// 4 bit field +typedef RecLinTrans<uint8_t, 4> StatTableTRI4; +constexpr StatTableTRI4 SQR_TABLE_TRI4({0x1, 0x4, 0x3, 0xc}); +constexpr StatTableTRI4 QRT_TABLE_TRI4({0x6, 0xa, 0x8, 0}); +typedef FieldTri<uint8_t, 4, 1, StatTableTRI4, &SQR_TABLE_TRI4, &QRT_TABLE_TRI4, &QRT_TABLE_TRI4, &QRT_TABLE_TRI4, &QRT_TABLE_TRI4, &QRT_TABLE_TRI4, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri4; +#endif + +#ifdef ENABLE_FIELD_INT_5 +// 5 bit field +typedef RecLinTrans<uint8_t, 5> StatTable5; +constexpr StatTable5 SQR_TABLE_5({0x1, 0x4, 0x10, 0xa, 0xd}); +constexpr StatTable5 SQR2_TABLE_5({0x1, 0x10, 0xd, 0xe, 0x1b}); +constexpr StatTable5 QRT_TABLE_5({0x14, 0x8, 0xa, 0, 0xe}); +typedef Field<uint8_t, 5, 5, StatTable5, &SQR_TABLE_5, &SQR2_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, IdTrans, &ID_TRANS, &ID_TRANS> Field5; +typedef FieldTri<uint8_t, 5, 2, RecLinTrans<uint8_t, 5>, &SQR_TABLE_5, &SQR2_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri5; +#endif + +#ifdef ENABLE_FIELD_INT_6 +// 6 bit field +typedef RecLinTrans<uint8_t, 6> StatTableTRI6; +constexpr StatTableTRI6 SQR_TABLE_TRI6({0x1, 0x4, 0x10, 0x3, 0xc, 0x30}); +constexpr StatTableTRI6 SQR2_TABLE_TRI6({0x1, 0x10, 0xc, 0x5, 0x13, 0x3c}); +constexpr StatTableTRI6 QRT_TABLE_TRI6({0x3a, 0x26, 0x24, 0x14, 0x20, 0}); +typedef FieldTri<uint8_t, 6, 1, StatTableTRI6, &SQR_TABLE_TRI6, &SQR2_TABLE_TRI6, &QRT_TABLE_TRI6, &QRT_TABLE_TRI6, &QRT_TABLE_TRI6, &QRT_TABLE_TRI6, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri6; +#endif + +#ifdef ENABLE_FIELD_INT_7 +// 7 bit field +typedef RecLinTrans<uint8_t, 4, 3> StatTableTRI7; +constexpr StatTableTRI7 SQR_TABLE_TRI7({0x1, 0x4, 0x10, 0x40, 0x6, 0x18, 0x60}); +constexpr StatTableTRI7 SQR2_TABLE_TRI7({0x1, 0x10, 0x6, 0x60, 0x14, 0x46, 0x78}); +constexpr StatTableTRI7 QRT_TABLE_TRI7({0, 0x14, 0x16, 0x72, 0x12, 0x40, 0x7a}); +typedef FieldTri<uint8_t, 7, 1, StatTableTRI7, &SQR_TABLE_TRI7, &SQR2_TABLE_TRI7, &QRT_TABLE_TRI7, &QRT_TABLE_TRI7, &QRT_TABLE_TRI7, &QRT_TABLE_TRI7, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri7; +#endif + +#ifdef ENABLE_FIELD_INT_8 +// 8 bit field +typedef RecLinTrans<uint8_t, 4, 4> StatTable8; +constexpr StatTable8 SQR_TABLE_8({0x1, 0x4, 0x10, 0x40, 0x1b, 0x6c, 0xab, 0x9a}); +constexpr StatTable8 SQR2_TABLE_8({0x1, 0x10, 0x1b, 0xab, 0x5e, 0x97, 0xb3, 0xc5}); +constexpr StatTable8 QRT_TABLE_8({0xbc, 0x2a, 0x28, 0x86, 0x2c, 0xde, 0x8e, 0}); +typedef Field<uint8_t, 8, 27, StatTable8, &SQR_TABLE_8, &SQR2_TABLE_8, &QRT_TABLE_8, &QRT_TABLE_8, &QRT_TABLE_8, &QRT_TABLE_8, IdTrans, &ID_TRANS, &ID_TRANS> Field8; +#endif +} + +Sketch* ConstructClMul1Byte(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_5 + case 5: return new SketchImpl<Field5>(implementation, 5); +#endif +#ifdef ENABLE_FIELD_INT_8 + case 8: return new SketchImpl<Field8>(implementation, 8); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri1Byte(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_2 + case 2: return new SketchImpl<FieldTri2>(implementation, 2); +#endif +#ifdef ENABLE_FIELD_INT_3 + case 3: return new SketchImpl<FieldTri3>(implementation, 3); +#endif +#ifdef ENABLE_FIELD_INT_4 + case 4: return new SketchImpl<FieldTri4>(implementation, 4); +#endif +#ifdef ENABLE_FIELD_INT_5 + case 5: return new SketchImpl<FieldTri5>(implementation, 5); +#endif +#ifdef ENABLE_FIELD_INT_6 + case 6: return new SketchImpl<FieldTri6>(implementation, 6); +#endif +#ifdef ENABLE_FIELD_INT_7 + case 7: return new SketchImpl<FieldTri7>(implementation, 7); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_2bytes.cpp b/src/minisketch/src/fields/clmul_2bytes.cpp new file mode 100644 index 0000000000..43930254dd --- /dev/null +++ b/src/minisketch/src/fields/clmul_2bytes.cpp @@ -0,0 +1,154 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_2) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_9 +// 9 bit field +typedef RecLinTrans<uint16_t, 5, 4> StatTableTRI9; +constexpr StatTableTRI9 SQR_TABLE_TRI9({0x1, 0x4, 0x10, 0x40, 0x100, 0x6, 0x18, 0x60, 0x180}); +constexpr StatTableTRI9 SQR2_TABLE_TRI9({0x1, 0x10, 0x100, 0x18, 0x180, 0x14, 0x140, 0x1e, 0x1e0}); +constexpr StatTableTRI9 SQR4_TABLE_TRI9({0x1, 0x180, 0x1e0, 0x198, 0x1fe, 0x80, 0xa0, 0x88, 0xaa}); +constexpr StatTableTRI9 QRT_TABLE_TRI9({0, 0x4e, 0x4c, 0x1aa, 0x48, 0x22, 0x1a2, 0x100, 0x58}); +typedef FieldTri<uint16_t, 9, 1, StatTableTRI9, &SQR_TABLE_TRI9, &SQR2_TABLE_TRI9, &SQR4_TABLE_TRI9, &QRT_TABLE_TRI9, &QRT_TABLE_TRI9, &QRT_TABLE_TRI9, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri9; +#endif + +#ifdef ENABLE_FIELD_INT_10 +// 10 bit field +typedef RecLinTrans<uint16_t, 5, 5> StatTable10; +constexpr StatTable10 SQR_TABLE_10({0x1, 0x4, 0x10, 0x40, 0x100, 0x9, 0x24, 0x90, 0x240, 0x112}); +constexpr StatTable10 SQR2_TABLE_10({0x1, 0x10, 0x100, 0x24, 0x240, 0x41, 0x19, 0x190, 0x136, 0x344}); +constexpr StatTable10 SQR4_TABLE_10({0x1, 0x240, 0x136, 0x141, 0x35d, 0x18, 0x265, 0x2e6, 0x227, 0x36b}); +constexpr StatTable10 QRT_TABLE_10({0xec, 0x86, 0x84, 0x30e, 0x80, 0x3c2, 0x306, 0, 0x90, 0x296}); +typedef Field<uint16_t, 10, 9, StatTable10, &SQR_TABLE_10, &SQR2_TABLE_10, &SQR4_TABLE_10, &QRT_TABLE_10, &QRT_TABLE_10, &QRT_TABLE_10, IdTrans, &ID_TRANS, &ID_TRANS> Field10; +typedef FieldTri<uint16_t, 10, 3, RecLinTrans<uint16_t, 5, 5>, &SQR_TABLE_10, &SQR2_TABLE_10, &SQR4_TABLE_10, &QRT_TABLE_10, &QRT_TABLE_10, &QRT_TABLE_10, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri10; +#endif + +#ifdef ENABLE_FIELD_INT_11 +// 11 bit field +typedef RecLinTrans<uint16_t, 6, 5> StatTable11; +constexpr StatTable11 SQR_TABLE_11({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0xa, 0x28, 0xa0, 0x280, 0x205}); +constexpr StatTable11 SQR2_TABLE_11({0x1, 0x10, 0x100, 0xa, 0xa0, 0x205, 0x44, 0x440, 0x428, 0x2a8, 0x291}); +constexpr StatTable11 SQR4_TABLE_11({0x1, 0xa0, 0x428, 0x1a, 0x645, 0x3a9, 0x144, 0x2d5, 0x9e, 0x4e7, 0x649}); +constexpr StatTable11 QRT_TABLE_11({0x734, 0x48, 0x4a, 0x1de, 0x4e, 0x35e, 0x1d6, 0x200, 0x5e, 0, 0x37e}); +typedef Field<uint16_t, 11, 5, StatTable11, &SQR_TABLE_11, &SQR2_TABLE_11, &SQR4_TABLE_11, nullptr, nullptr, &QRT_TABLE_11, IdTrans, &ID_TRANS, &ID_TRANS> Field11; +typedef FieldTri<uint16_t, 11, 2, RecLinTrans<uint16_t, 6, 5>, &SQR_TABLE_11, &SQR2_TABLE_11, &SQR4_TABLE_11, &QRT_TABLE_11, &QRT_TABLE_11, &QRT_TABLE_11, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri11; +#endif + +#ifdef ENABLE_FIELD_INT_12 +// 12 bit field +typedef RecLinTrans<uint16_t, 6, 6> StatTable12; +constexpr StatTable12 SQR_TABLE_12({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x9, 0x24, 0x90, 0x240, 0x900, 0x412}); +constexpr StatTable12 SQR2_TABLE_12({0x1, 0x10, 0x100, 0x9, 0x90, 0x900, 0x41, 0x410, 0x124, 0x249, 0x482, 0x804}); +constexpr StatTable12 SQR4_TABLE_12({0x1, 0x90, 0x124, 0x8, 0x480, 0x920, 0x40, 0x412, 0x924, 0x200, 0x82, 0x904}); +constexpr StatTable12 QRT_TABLE_12({0x48, 0xc10, 0xc12, 0x208, 0xc16, 0xd82, 0x200, 0x110, 0xc06, 0, 0xda2, 0x5a4}); +typedef Field<uint16_t, 12, 9, StatTable12, &SQR_TABLE_12, &SQR2_TABLE_12, &SQR4_TABLE_12, &QRT_TABLE_12, &QRT_TABLE_12, &QRT_TABLE_12, IdTrans, &ID_TRANS, &ID_TRANS> Field12; +typedef FieldTri<uint16_t, 12, 3, RecLinTrans<uint16_t, 6, 6>, &SQR_TABLE_12, &SQR2_TABLE_12, &SQR4_TABLE_12, &QRT_TABLE_12, &QRT_TABLE_12, &QRT_TABLE_12, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri12; +#endif + +#ifdef ENABLE_FIELD_INT_13 +// 13 bit field +typedef RecLinTrans<uint16_t, 5, 4, 4> StatTable13; +constexpr StatTable13 SQR_TABLE_13({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x36, 0xd8, 0x360, 0xd80, 0x161b, 0x185a}); +constexpr StatTable13 SQR2_TABLE_13({0x1, 0x10, 0x100, 0x1000, 0xd8, 0xd80, 0x185a, 0x514, 0x1176, 0x17b8, 0x1b75, 0x17ff, 0x1f05}); +constexpr StatTable13 SQR4_TABLE_13({0x1, 0xd8, 0x1176, 0x1f05, 0xd96, 0x18e8, 0x68, 0xbdb, 0x1a61, 0x1af2, 0x1a37, 0x3b9, 0x1440}); +constexpr StatTable13 QRT_TABLE_13({0xcfc, 0x1500, 0x1502, 0x382, 0x1506, 0x149c, 0x38a, 0x118, 0x1516, 0, 0x14bc, 0x100e, 0x3ca}); +typedef Field<uint16_t, 13, 27, StatTable13, &SQR_TABLE_13, &SQR2_TABLE_13, &SQR4_TABLE_13, &QRT_TABLE_13, &QRT_TABLE_13, &QRT_TABLE_13, IdTrans, &ID_TRANS, &ID_TRANS> Field13; +#endif + +#ifdef ENABLE_FIELD_INT_14 +// 14 bit field +typedef RecLinTrans<uint16_t, 5, 5, 4> StatTable14; +constexpr StatTable14 SQR_TABLE_14({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x21, 0x84, 0x210, 0x840, 0x2100, 0x442, 0x1108}); +constexpr StatTable14 SQR2_TABLE_14({0x1, 0x10, 0x100, 0x1000, 0x84, 0x840, 0x442, 0x401, 0x31, 0x310, 0x3100, 0x118c, 0x1844, 0x486}); +constexpr StatTable14 SQR4_TABLE_14({0x1, 0x84, 0x31, 0x1844, 0x501, 0x15ce, 0x3552, 0x3101, 0x8c5, 0x3a5, 0x1cf3, 0xd74, 0xc8a, 0x3411}); +constexpr StatTable14 QRT_TABLE_14({0x13f2, 0x206, 0x204, 0x3e06, 0x200, 0x1266, 0x3e0e, 0x114, 0x210, 0, 0x1246, 0x2848, 0x3e4e, 0x2258}); +typedef Field<uint16_t, 14, 33, StatTable14, &SQR_TABLE_14, &SQR2_TABLE_14, &SQR4_TABLE_14, &QRT_TABLE_14, &QRT_TABLE_14, &QRT_TABLE_14, IdTrans, &ID_TRANS, &ID_TRANS> Field14; +typedef FieldTri<uint16_t, 14, 5, RecLinTrans<uint16_t, 5, 5, 4>, &SQR_TABLE_14, &SQR2_TABLE_14, &SQR4_TABLE_14, &QRT_TABLE_14, &QRT_TABLE_14, &QRT_TABLE_14, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri14; +#endif + +#ifdef ENABLE_FIELD_INT_15 +// 15 bit field +typedef RecLinTrans<uint16_t, 5, 5, 5> StatTableTRI15; +constexpr StatTableTRI15 SQR_TABLE_TRI15({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x6, 0x18, 0x60, 0x180, 0x600, 0x1800, 0x6000}); +constexpr StatTableTRI15 SQR2_TABLE_TRI15({0x1, 0x10, 0x100, 0x1000, 0x6, 0x60, 0x600, 0x6000, 0x14, 0x140, 0x1400, 0x4006, 0x78, 0x780, 0x7800}); +constexpr StatTableTRI15 SQR4_TABLE_TRI15({0x1, 0x6, 0x14, 0x78, 0x110, 0x660, 0x1540, 0x7f80, 0x106, 0x614, 0x1478, 0x7910, 0x1666, 0x7554, 0x3ffe}); +constexpr StatTableTRI15 QRT_TABLE_TRI15({0, 0x114, 0x116, 0x428, 0x112, 0x137a, 0x420, 0x6d62, 0x102, 0x73a, 0x135a, 0x6460, 0x460, 0x4000, 0x6de2}); +typedef FieldTri<uint16_t, 15, 1, StatTableTRI15, &SQR_TABLE_TRI15, &SQR2_TABLE_TRI15, &SQR4_TABLE_TRI15, &QRT_TABLE_TRI15, &QRT_TABLE_TRI15, &QRT_TABLE_TRI15, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri15; +#endif + +#ifdef ENABLE_FIELD_INT_16 +// 16 bit field +typedef RecLinTrans<uint16_t, 6, 5, 5> StatTable16; +constexpr StatTable16 SQR_TABLE_16({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x2b, 0xac, 0x2b0, 0xac0, 0x2b00, 0xac00, 0xb056, 0xc10e}); +constexpr StatTable16 SQR2_TABLE_16({0x1, 0x10, 0x100, 0x1000, 0x2b, 0x2b0, 0x2b00, 0xb056, 0x445, 0x4450, 0x45ac, 0x5a6c, 0xa647, 0x657e, 0x571a, 0x7127}); +constexpr StatTable16 SQR4_TABLE_16({0x1, 0x2b, 0x445, 0xa647, 0x12a1, 0xf69d, 0x7f07, 0x9825, 0x6fad, 0x399d, 0xb515, 0xd7d1, 0x3fb4, 0x4b06, 0xe4df, 0x93c7}); +constexpr StatTable16 QRT_TABLE_16({0x732, 0x72b8, 0x72ba, 0x7e96, 0x72be, 0x78b2, 0x7e9e, 0x8cba, 0x72ae, 0xfa24, 0x7892, 0x5892, 0x7ede, 0xbec6, 0x8c3a, 0}); +typedef Field<uint16_t, 16, 43, StatTable16, &SQR_TABLE_16, &SQR2_TABLE_16, &SQR4_TABLE_16, &QRT_TABLE_16, &QRT_TABLE_16, &QRT_TABLE_16, IdTrans, &ID_TRANS, &ID_TRANS> Field16; +#endif +} + +Sketch* ConstructClMul2Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_10 + case 10: return new SketchImpl<Field10>(implementation, 10); +#endif +#ifdef ENABLE_FIELD_INT_11 + case 11: return new SketchImpl<Field11>(implementation, 11); +#endif +#ifdef ENABLE_FIELD_INT_12 + case 12: return new SketchImpl<Field12>(implementation, 12); +#endif +#ifdef ENABLE_FIELD_INT_13 + case 13: return new SketchImpl<Field13>(implementation, 13); +#endif +#ifdef ENABLE_FIELD_INT_14 + case 14: return new SketchImpl<Field14>(implementation, 14); +#endif +#ifdef ENABLE_FIELD_INT_16 + case 16: return new SketchImpl<Field16>(implementation, 16); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri2Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_9 + case 9: return new SketchImpl<FieldTri9>(implementation, 9); +#endif +#ifdef ENABLE_FIELD_INT_10 + case 10: return new SketchImpl<FieldTri10>(implementation, 10); +#endif +#ifdef ENABLE_FIELD_INT_11 + case 11: return new SketchImpl<FieldTri11>(implementation, 11); +#endif +#ifdef ENABLE_FIELD_INT_12 + case 12: return new SketchImpl<FieldTri12>(implementation, 12); +#endif +#ifdef ENABLE_FIELD_INT_14 + case 14: return new SketchImpl<FieldTri14>(implementation, 14); +#endif +#ifdef ENABLE_FIELD_INT_15 + case 15: return new SketchImpl<FieldTri15>(implementation, 15); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_3bytes.cpp b/src/minisketch/src/fields/clmul_3bytes.cpp new file mode 100644 index 0000000000..b473f66ba2 --- /dev/null +++ b/src/minisketch/src/fields/clmul_3bytes.cpp @@ -0,0 +1,166 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_3) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_17 +// 17 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5> StatTable17; +constexpr StatTable17 SQR_TABLE_17({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x8012}); +constexpr StatTable17 SQR2_TABLE_17({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x48, 0x480, 0x4800, 0x8012, 0x104, 0x1040, 0x10400, 0x4048, 0x492, 0x4920, 0x9212, 0x12104}); +constexpr StatTable17 SQR4_TABLE_17({0x1, 0x10000, 0x8012, 0x4048, 0x12104, 0x1480, 0x5840, 0x14d20, 0x19202, 0x8112, 0x44c8, 0x13144, 0x5da0, 0x15850, 0x1cd7a, 0x1d34e, 0x1a484}); +constexpr StatTable17 SQR8_TABLE_17({0x1, 0x1a484, 0x1f24a, 0x1d572, 0x1eec4, 0x15448, 0xf9de, 0x9af0, 0x1ab78, 0x6048, 0xdc9a, 0x1eb24, 0x2ef4, 0x7c5e, 0x170b2, 0x16c1a, 0xa660}); +constexpr StatTable17 QRT_TABLE_17({0, 0x4c3e, 0x4c3c, 0x1a248, 0x4c38, 0x428, 0x1a240, 0x1b608, 0x4c28, 0x206, 0x408, 0x4000, 0x1a200, 0x18006, 0x1b688, 0x14d2e, 0x4d28}); +typedef Field<uint32_t, 17, 9, StatTable17, &SQR_TABLE_17, &SQR2_TABLE_17, &SQR4_TABLE_17, &SQR8_TABLE_17, &QRT_TABLE_17, &QRT_TABLE_17, IdTrans, &ID_TRANS, &ID_TRANS> Field17; +typedef FieldTri<uint32_t, 17, 3, RecLinTrans<uint32_t, 6, 6, 5>, &SQR_TABLE_17, &SQR2_TABLE_17, &SQR4_TABLE_17, &SQR8_TABLE_17, &QRT_TABLE_17, &QRT_TABLE_17, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri17; +#endif + +#ifdef ENABLE_FIELD_INT_18 +// 18 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6> StatTable18; +constexpr StatTable18 SQR_TABLE_18({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x10012}); +constexpr StatTable18 SQR2_TABLE_18({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x24, 0x240, 0x2400, 0x24000, 0x41, 0x410, 0x4100, 0x1009, 0x10090, 0x924, 0x9240, 0x12412, 0x24104}); +constexpr StatTable18 SQR4_TABLE_18({0x1, 0x10000, 0x24000, 0x1009, 0x12412, 0x124, 0x201, 0x10480, 0x24820, 0x241, 0x10410, 0x24924, 0x8, 0x12, 0x20024, 0x8048, 0x12082, 0x920}); +constexpr StatTable18 SQR8_TABLE_18({0x1, 0x12082, 0x20904, 0x1000, 0x92, 0x904, 0x240, 0x12012, 0x4104, 0x41, 0x10080, 0x4924, 0x1009, 0x2412, 0x24804, 0x9240, 0x12410, 0x20}); +constexpr StatTable18 QRT_TABLE_18({0x9208, 0x422, 0x420, 0x8048, 0x424, 0x68b0, 0x8040, 0x30086, 0x434, 0x1040, 0x6890, 0x30ca2, 0x8000, 0x32896, 0x30006, 0, 0x534, 0x20532}); +typedef Field<uint32_t, 18, 9, StatTable18, &SQR_TABLE_18, &SQR2_TABLE_18, &SQR4_TABLE_18, &SQR8_TABLE_18, &QRT_TABLE_18, &QRT_TABLE_18, IdTrans, &ID_TRANS, &ID_TRANS> Field18; +typedef FieldTri<uint32_t, 18, 3, RecLinTrans<uint32_t, 6, 6, 6>, &SQR_TABLE_18, &SQR2_TABLE_18, &SQR4_TABLE_18, &SQR8_TABLE_18, &QRT_TABLE_18, &QRT_TABLE_18, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri18; +#endif + +#ifdef ENABLE_FIELD_INT_19 +// 19 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 4> StatTable19; +constexpr StatTable19 SQR_TABLE_19({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x3804e, 0x6011f}); +constexpr StatTable19 SQR2_TABLE_19({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x4e, 0x4e0, 0x4e00, 0x4e000, 0x6011f, 0x1054, 0x10540, 0x544e, 0x544e0, 0x44f76, 0x4f658, 0x7649f, 0x6481a, 0x48004}); +constexpr StatTable19 SQR4_TABLE_19({0x1, 0x10000, 0x4e000, 0x544e, 0x7649f, 0x15f0, 0x5afa, 0x35b7d, 0x17dca, 0x7390f, 0x151ae, 0x3902b, 0x41e9c, 0x7f117, 0x23ec7, 0x62c2f, 0x5e852, 0x69238, 0x775c}); +constexpr StatTable19 SQR8_TABLE_19({0x1, 0x5e852, 0x394a3, 0x29f41, 0x618e5, 0x4210, 0x7add9, 0x31105, 0x5d098, 0x7bb13, 0x44f00, 0x966, 0x11ae6, 0x70901, 0x664bf, 0x67449, 0x3d2bf, 0x4cbf9, 0x54e0c}); +constexpr StatTable19 QRT_TABLE_19({0x5d6b0, 0x2f476, 0x2f474, 0x1d6a2, 0x2f470, 0x42a, 0x1d6aa, 0x1060, 0x2f460, 0x19e92, 0x40a, 0x1da98, 0x1d6ea, 0x28c78, 0x10e0, 0xf56a, 0x2f560, 0, 0x19c92}); +typedef Field<uint32_t, 19, 39, StatTable19, &SQR_TABLE_19, &SQR2_TABLE_19, &SQR4_TABLE_19, &SQR8_TABLE_19, &QRT_TABLE_19, &QRT_TABLE_19, IdTrans, &ID_TRANS, &ID_TRANS> Field19; +#endif + +#ifdef ENABLE_FIELD_INT_20 +// 20 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 5> StatTable20; +constexpr StatTable20 SQR_TABLE_20({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x90000, 0x40012}); +constexpr StatTable20 SQR2_TABLE_20({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x9, 0x90, 0x900, 0x9000, 0x90000, 0x41, 0x410, 0x4100, 0x41000, 0x10024, 0x249, 0x2490, 0x24900, 0x49012, 0x90104}); +constexpr StatTable20 SQR4_TABLE_20({0x1, 0x10000, 0x9000, 0x4100, 0x2490, 0x1001, 0x10900, 0x9410, 0x4349, 0x92594, 0x91, 0x10041, 0x19024, 0x4d112, 0x2599, 0x91091, 0x51941, 0x3dd34, 0x5d34b, 0x9b494}); +constexpr StatTable20 SQR8_TABLE_20({0x1, 0x51941, 0x880b5, 0x66d0, 0x46103, 0x19025, 0x45a49, 0x8a4b4, 0x80b45, 0x81f9f, 0xb081, 0x41040, 0xd19f5, 0xc11be, 0x4634b, 0xd8d70, 0x11027, 0xf8651, 0x141fa, 0xdc63}); +constexpr StatTable20 QRT_TABLE_20({0xc5dea, 0xc0110, 0xc0112, 0xe11de, 0xc0116, 0x24814, 0xe11d6, 0x20080, 0xc0106, 0xfe872, 0x24834, 0xe4106, 0xe1196, 0x1d9a4, 0x20000, 0x31190, 0xc0006, 0, 0xfea72, 0x7ea74}); +typedef Field<uint32_t, 20, 9, StatTable20, &SQR_TABLE_20, &SQR2_TABLE_20, &SQR4_TABLE_20, &SQR8_TABLE_20, &QRT_TABLE_20, &QRT_TABLE_20, IdTrans, &ID_TRANS, &ID_TRANS> Field20; +typedef FieldTri<uint32_t, 20, 3, RecLinTrans<uint32_t, 5, 5, 5, 5>, &SQR_TABLE_20, &SQR2_TABLE_20, &SQR4_TABLE_20, &SQR8_TABLE_20, &QRT_TABLE_20, &QRT_TABLE_20, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri20; +#endif + +#ifdef ENABLE_FIELD_INT_21 +// 21 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5> StatTable21; +constexpr StatTable21 SQR_TABLE_21({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x80005}); +constexpr StatTable21 SQR2_TABLE_21({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x28, 0x280, 0x2800, 0x28000, 0x80005, 0x44, 0x440, 0x4400, 0x44000, 0x4000a, 0xaa, 0xaa0, 0xaa00, 0xaa000, 0xa0011}); +constexpr StatTable21 SQR4_TABLE_21({0x1, 0x10000, 0x2800, 0x440, 0xaa, 0xa0011, 0x101000, 0x28280, 0x4444, 0x40aaa, 0xaa101, 0x128, 0x8002d, 0xc4005, 0x4ea00, 0xba10, 0x101290, 0x1282c4, 0x6c44e, 0xeeeaa, 0xbaaa1}); +constexpr StatTable21 SQR8_TABLE_21({0x1, 0x101290, 0xc412d, 0x1ab101, 0x986d1, 0x1c6cc5, 0x3aa8c, 0x14b0fe, 0x1e7301, 0xb491d, 0x10d23e, 0xa4015, 0x4c2fa, 0xce8e5, 0xadfd9, 0xf110, 0x5220c, 0xf225f, 0xb8bdb, 0x159467, 0xc0df9}); +constexpr StatTable21 QRT_TABLE_21({0x1bd5fc, 0xbc196, 0xbc194, 0x74b96, 0xbc190, 0x1048, 0x74b9e, 0x672c8, 0xbc180, 0x4080, 0x1068, 0xc8200, 0x74bde, 0x64280, 0x67248, 0xc4280, 0xbc080, 0x80000, 0x4280, 0, 0x1468}); +typedef Field<uint32_t, 21, 5, StatTable21, &SQR_TABLE_21, &SQR2_TABLE_21, &SQR4_TABLE_21, &SQR8_TABLE_21, &QRT_TABLE_21, &QRT_TABLE_21, IdTrans, &ID_TRANS, &ID_TRANS> Field21; +typedef FieldTri<uint32_t, 21, 2, RecLinTrans<uint32_t, 6, 5, 5, 5>, &SQR_TABLE_21, &SQR2_TABLE_21, &SQR4_TABLE_21, &SQR8_TABLE_21, &QRT_TABLE_21, &QRT_TABLE_21, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri21; +#endif + +#ifdef ENABLE_FIELD_INT_22 +// 22 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5> StatTableTRI22; +constexpr StatTableTRI22 SQR_TABLE_TRI22({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000}); +constexpr StatTableTRI22 SQR2_TABLE_TRI22({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0xc, 0xc0, 0xc00, 0xc000, 0xc0000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x100003, 0x3c, 0x3c0, 0x3c00, 0x3c000, 0x3c0000}); +constexpr StatTableTRI22 SQR4_TABLE_TRI22({0x1, 0x10000, 0xc00, 0x50, 0x100003, 0x3c000, 0x1100, 0xcc, 0xc0005, 0x55000, 0x3fc0, 0x101, 0x1000c, 0xc0c00, 0x5050, 0x1003c3, 0x3c011, 0x111100, 0xcccc, 0xc0555, 0x15503f, 0x3fffc0}); +constexpr StatTableTRI22 SQR8_TABLE_TRI22({0x1, 0x3c011, 0x3ec1, 0x101103, 0x14503e, 0x28282, 0xd0009, 0x1d9c, 0xcc598, 0x25c81, 0x47304, 0xc0004, 0x3cc41, 0xcf758, 0x11415f, 0x1d11f7, 0x128280, 0x1b9027, 0x1070ce, 0x10eb5e, 0x5c0ec, 0x2097e0}); +constexpr StatTableTRI22 QRT_TABLE_TRI22({0x210d16, 0x104a, 0x1048, 0x4088, 0x104c, 0x200420, 0x4080, 0x492dc, 0x105c, 0x1a67f0, 0x200400, 0x21155c, 0x40c0, 0x20346c, 0x4925c, 0x1af7ac, 0x115c, 0x2274ac, 0x1a65f0, 0x2a65f0, 0x200000, 0}); +typedef FieldTri<uint32_t, 22, 1, StatTableTRI22, &SQR_TABLE_TRI22, &SQR2_TABLE_TRI22, &SQR4_TABLE_TRI22, &SQR8_TABLE_TRI22, &QRT_TABLE_TRI22, &QRT_TABLE_TRI22, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri22; +#endif + +#ifdef ENABLE_FIELD_INT_23 +// 23 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 5> StatTable23; +constexpr StatTable23 SQR_TABLE_23({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x42, 0x108, 0x420, 0x1080, 0x4200, 0x10800, 0x42000, 0x108000, 0x420000, 0x80042, 0x200108}); +constexpr StatTable23 SQR2_TABLE_23({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x42, 0x420, 0x4200, 0x42000, 0x420000, 0x200108, 0x1004, 0x10040, 0x100400, 0x4042, 0x40420, 0x404200, 0x42108, 0x421080, 0x210908, 0x109004, 0x90002}); +constexpr StatTable23 SQR4_TABLE_23({0x1, 0x10000, 0x4200, 0x1004, 0x40420, 0x210908, 0x52, 0x520000, 0x142400, 0x52148, 0x494202, 0x10c204, 0x1104, 0x40462, 0x630908, 0x100452, 0x562108, 0x1d2402, 0x57348, 0x495626, 0x34c72c, 0x21584e, 0x4614b0}); +constexpr StatTable23 SQR8_TABLE_23({0x1, 0x562108, 0x662840, 0x5304, 0x6d3842, 0x738f46, 0x50472, 0x6ff79e, 0x7cf204, 0x436274, 0x3e4bde, 0x42a93e, 0x147704, 0x6c3810, 0x28bff4, 0x78815c, 0x7ab4b0, 0x62852a, 0x255b30, 0x5653d0, 0x1afd36, 0x5f118, 0x601dd4}); +constexpr StatTable23 QRT_TABLE_23({0, 0x1040, 0x1042, 0x43056, 0x1046, 0x121d76, 0x4305e, 0x40a0, 0x1056, 0x15176, 0x121d56, 0x7ee1f6, 0x4301e, 0x40000, 0x4020, 0x4f0be, 0x1156, 0x7cf0a0, 0x15376, 0x1ee9e8, 0x121956, 0x3ac9f6, 0x7ee9f6}); +typedef Field<uint32_t, 23, 33, StatTable23, &SQR_TABLE_23, &SQR2_TABLE_23, &SQR4_TABLE_23, &SQR8_TABLE_23, &QRT_TABLE_23, &QRT_TABLE_23, IdTrans, &ID_TRANS, &ID_TRANS> Field23; +typedef FieldTri<uint32_t, 23, 5, RecLinTrans<uint32_t, 6, 6, 6, 5>, &SQR_TABLE_23, &SQR2_TABLE_23, &SQR4_TABLE_23, &SQR8_TABLE_23, nullptr, &QRT_TABLE_23, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri23; +#endif + +#ifdef ENABLE_FIELD_INT_24 +// 24 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6> StatTable24; +constexpr StatTable24 SQR_TABLE_24({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0xb0001b, 0xc0005a}); +constexpr StatTable24 SQR2_TABLE_24({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1b, 0x1b0, 0x1b00, 0x1b000, 0x1b0000, 0xb0001b, 0x145, 0x1450, 0x14500, 0x145000, 0x45001b, 0x5001dc, 0x1db7, 0x1db70, 0x1db700, 0xdb701b, 0xb7011f, 0x701105}); +constexpr StatTable24 SQR4_TABLE_24({0x1, 0x10000, 0x1b00, 0x145, 0x45001b, 0x1db700, 0x11011, 0x111ab0, 0xb1aa5e, 0x51450e, 0x96db7, 0xb7c60f, 0x1a1a, 0x1a015e, 0x5f5e1b, 0x1ceef2, 0xf30ca2, 0xabbdb4, 0xba1aff, 0xf0bf5e, 0x579fc9, 0xce3da9, 0xa2c07f, 0x71dd40}); +constexpr StatTable24 SQR8_TABLE_24({0x1, 0xf30ca2, 0x573345, 0xb0a14e, 0xafd77d, 0x1419b, 0xb616a2, 0xba7db, 0xbe1560, 0xe0d0a3, 0x15bf5, 0x1056dd, 0xa29845, 0xf83d32, 0x13e0e9, 0xe2d8d3, 0xa10841, 0x57ac5a, 0x1c432f, 0x57044e, 0x454fba, 0x2bb37c, 0xf50fa, 0x85d5b9}); +constexpr StatTable24 QRT_TABLE_24({0x104e, 0xaf42a8, 0xaf42aa, 0xb78186, 0xaf42ae, 0x4090, 0xb7818e, 0x4a37c, 0xaf42be, 0x3688c0, 0x40b0, 0x80080e, 0xb781ce, 0xaf2232, 0x4a3fc, 0x856a82, 0xaf43be, 0x29c970, 0x368ac0, 0x968ace, 0x44b0, 0x77d570, 0x80000e, 0}); +typedef Field<uint32_t, 24, 27, StatTable24, &SQR_TABLE_24, &SQR2_TABLE_24, &SQR4_TABLE_24, &SQR8_TABLE_24, &QRT_TABLE_24, &QRT_TABLE_24, IdTrans, &ID_TRANS, &ID_TRANS> Field24; +#endif +} + +Sketch* ConstructClMul3Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_17 + case 17: return new SketchImpl<Field17>(implementation, 17); +#endif +#ifdef ENABLE_FIELD_INT_18 + case 18: return new SketchImpl<Field18>(implementation, 18); +#endif +#ifdef ENABLE_FIELD_INT_19 + case 19: return new SketchImpl<Field19>(implementation, 19); +#endif +#ifdef ENABLE_FIELD_INT_20 + case 20: return new SketchImpl<Field20>(implementation, 20); +#endif +#ifdef ENABLE_FIELD_INT_21 + case 21: return new SketchImpl<Field21>(implementation, 21); +#endif +#ifdef ENABLE_FIELD_INT_23 + case 23: return new SketchImpl<Field23>(implementation, 23); +#endif +#ifdef ENABLE_FIELD_INT_24 + case 24: return new SketchImpl<Field24>(implementation, 24); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri3Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_17 + case 17: return new SketchImpl<FieldTri17>(implementation, 17); +#endif +#ifdef ENABLE_FIELD_INT_18 + case 18: return new SketchImpl<FieldTri18>(implementation, 18); +#endif +#ifdef ENABLE_FIELD_INT_20 + case 20: return new SketchImpl<FieldTri20>(implementation, 20); +#endif +#ifdef ENABLE_FIELD_INT_21 + case 21: return new SketchImpl<FieldTri21>(implementation, 21); +#endif +#ifdef ENABLE_FIELD_INT_22 + case 22: return new SketchImpl<FieldTri22>(implementation, 22); +#endif +#ifdef ENABLE_FIELD_INT_23 + case 23: return new SketchImpl<FieldTri23>(implementation, 23); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_4bytes.cpp b/src/minisketch/src/fields/clmul_4bytes.cpp new file mode 100644 index 0000000000..c65974394c --- /dev/null +++ b/src/minisketch/src/fields/clmul_4bytes.cpp @@ -0,0 +1,158 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_4) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_25 +// 25 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 5, 5> StatTable25; +constexpr StatTable25 SQR_TABLE_25({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x800012}); +constexpr StatTable25 SQR2_TABLE_25({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x48, 0x480, 0x4800, 0x48000, 0x480000, 0x800012, 0x104, 0x1040, 0x10400, 0x104000, 0x1040000, 0x400048, 0x492, 0x4920, 0x49200, 0x492000, 0x920012, 0x1200104}); +constexpr StatTable25 SQR4_TABLE_25({0x1, 0x10000, 0x480, 0x800012, 0x104000, 0x4920, 0x1200104, 0x1001000, 0x48048, 0x481040, 0x410448, 0x492492, 0x930002, 0x580, 0x1800012, 0x14c000, 0x5960, 0x160014c, 0x1493000, 0x58058, 0x5814c0, 0xc14c5a, 0x596596, 0x1974922, 0x1249684}); +constexpr StatTable25 SQR8_TABLE_25({0x1, 0x5960, 0x1411448, 0x1860922, 0x1d814d2, 0x1cdede8, 0x1e15e16, 0x1b79686, 0xfdf116, 0x1efe4c8, 0x1b839a8, 0x10ced66, 0xae05ce, 0x1459400, 0xa29fa6, 0x85e4d2, 0x7eecee, 0x183a96, 0x1eb2fa8, 0xede876, 0xf6e440, 0x1f7140a, 0xd07d7c, 0x10e4ea2, 0x1222a54}); +constexpr StatTable25 QRT_TABLE_25({0, 0x482110, 0x482112, 0x1b3c3e6, 0x482116, 0x4960ae, 0x1b3c3ee, 0x4088, 0x482106, 0x58a726, 0x49608e, 0x5ce52e, 0x1b3c3ae, 0x2006, 0x4008, 0x1c1a8, 0x482006, 0x1e96488, 0x58a526, 0x400000, 0x49648e, 0x1800006, 0x5ced2e, 0xb3d3a8, 0x1b3d3ae}); +typedef Field<uint32_t, 25, 9, StatTable25, &SQR_TABLE_25, &SQR2_TABLE_25, &SQR4_TABLE_25, &SQR8_TABLE_25, &QRT_TABLE_25, &QRT_TABLE_25, IdTrans, &ID_TRANS, &ID_TRANS> Field25; +typedef FieldTri<uint32_t, 25, 3, RecLinTrans<uint32_t, 5, 5, 5, 5, 5>, &SQR_TABLE_25, &SQR2_TABLE_25, &SQR4_TABLE_25, &SQR8_TABLE_25, &QRT_TABLE_25, &QRT_TABLE_25, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri25; +#endif + +#ifdef ENABLE_FIELD_INT_26 +// 26 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5, 5> StatTable26; +constexpr StatTable26 SQR_TABLE_26({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0x1b00000, 0x2c0001b, 0x300005a}); +constexpr StatTable26 SQR2_TABLE_26({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x6c, 0x6c0, 0x6c00, 0x6c000, 0x6c0000, 0x2c0001b, 0x145, 0x1450, 0x14500, 0x145000, 0x1450000, 0x500077, 0x100076b, 0x76dc, 0x76dc0, 0x76dc00, 0x36dc01b, 0x2dc011f, 0x1c01105}); +constexpr StatTable26 SQR4_TABLE_26({0x1, 0x10000, 0x6c0, 0x2c0001b, 0x145000, 0x76dc, 0x2dc011f, 0x1101100, 0x106ac6c, 0x6ad515, 0x1145127, 0x121b6dc, 0x2da1d0f, 0x10007c1, 0x3c7c01b, 0x128290, 0x29062e0, 0x2ee8d68, 0x167abcd, 0x3cabbce, 0x3c7a862, 0x6b83ce, 0x3cf5620, 0x229b787, 0x38a6b0f, 0x3071ade}); +constexpr StatTable26 SQR8_TABLE_26({0x1, 0x29062e0, 0x2b2942d, 0x34ab63, 0x3bddebb, 0x7b1823, 0x58b9ae, 0x391720e, 0x1385e18, 0x3891746, 0x13069c5, 0x2dfd089, 0x12a35ff, 0x3e534f, 0x172c6a2, 0x55338f, 0x3887137, 0x3f45b03, 0x164a695, 0x2c7e7ef, 0x29c907d, 0x636c85, 0x3db4007, 0x97e7ff, 0x3cbfe55, 0x31c0d96}); +constexpr StatTable26 QRT_TABLE_26({0x217b530, 0x2ae82a8, 0x2ae82aa, 0x2001046, 0x2ae82ae, 0x2de032e, 0x200104e, 0x70c10c, 0x2ae82be, 0x20151f2, 0x2de030e, 0xbc1400, 0x200100e, 0x178570, 0x70c18c, 0x2ae4232, 0x2ae83be, 0x211d742, 0x20153f2, 0x21f54f2, 0x2de070e, 0x5e0700, 0xbc1c00, 0x3abb97e, 0x200000e, 0}); +typedef Field<uint32_t, 26, 27, StatTable26, &SQR_TABLE_26, &SQR2_TABLE_26, &SQR4_TABLE_26, &SQR8_TABLE_26, &QRT_TABLE_26, &QRT_TABLE_26, IdTrans, &ID_TRANS, &ID_TRANS> Field26; +#endif + +#ifdef ENABLE_FIELD_INT_27 +// 27 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5, 5> StatTable27; +constexpr StatTable27 SQR_TABLE_27({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x138000, 0x4e0000, 0x1380000, 0x4e00000, 0x380004e, 0x600011f}); +constexpr StatTable27 SQR2_TABLE_27({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x4e, 0x4e0, 0x4e00, 0x4e000, 0x4e0000, 0x4e00000, 0x600011f, 0x1054, 0x10540, 0x105400, 0x1054000, 0x54004e, 0x54004e0, 0x4004f76, 0x4f658, 0x4f6580, 0x4f65800, 0x765811f, 0x658101a, 0x5810004}); +constexpr StatTable27 SQR4_TABLE_27({0x1, 0x10000, 0x4e0, 0x4e00000, 0x105400, 0x4004f76, 0x765811f, 0x1001110, 0x114e04e, 0x4abe54, 0x6551445, 0x45e212e, 0x13ccbdc, 0x3d805ef, 0x5e10100, 0x114b0e0, 0xe4bf22, 0x721c505, 0x51b3ba8, 0x3bf04d5, 0x4dabba0, 0x3b0aa45, 0x24a80cb, 0xc3d4b0, 0x4b34626, 0x6372e18, 0x6028c1b}); +constexpr StatTable27 SQR8_TABLE_27({0x1, 0xe4bf22, 0x430cb3c, 0x73b7225, 0x6526539, 0x3c278e3, 0x4724a6e, 0x48b39b4, 0x1dbf7de, 0x106508, 0x3564785, 0x33ae33f, 0x61d6685, 0x6adaca3, 0x2786b6f, 0x4e76784, 0x869f42, 0x466b048, 0x415e00e, 0x46c3c9a, 0x73ffd91, 0x49002e0, 0x3734fed, 0x3c04a43, 0x191d3ee, 0xe828b9, 0xfab68c}); +constexpr StatTable27 QRT_TABLE_27({0x6bf0530, 0x2be4496, 0x2be4494, 0x2bf0522, 0x2be4490, 0x1896cca, 0x2bf052a, 0x408a, 0x2be4480, 0x368ae72, 0x1896cea, 0x18d2ee0, 0x2bf056a, 0x1c76d6a, 0x400a, 0x336e9f8, 0x2be4580, 0x36baf12, 0x368ac72, 0x430360, 0x18968ea, 0x34a6b80, 0x18d26e0, 0xbf1560, 0x2bf156a, 0, 0x1c74d6a}); +typedef Field<uint32_t, 27, 39, StatTable27, &SQR_TABLE_27, &SQR2_TABLE_27, &SQR4_TABLE_27, &SQR8_TABLE_27, &QRT_TABLE_27, &QRT_TABLE_27, IdTrans, &ID_TRANS, &ID_TRANS> Field27; +#endif + +#ifdef ENABLE_FIELD_INT_28 +// 28 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 5, 5> StatTableTRI28; +constexpr StatTableTRI28 SQR_TABLE_TRI28({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000}); +constexpr StatTableTRI28 SQR2_TABLE_TRI28({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x3, 0x30, 0x300, 0x3000, 0x30000, 0x300000, 0x3000000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x500000, 0x5000000, 0xf, 0xf0, 0xf00, 0xf000, 0xf0000, 0xf00000, 0xf000000}); +constexpr StatTableTRI28 SQR4_TABLE_TRI28({0x1, 0x10000, 0x30, 0x300000, 0x500, 0x5000000, 0xf000, 0x11, 0x110000, 0x330, 0x3300000, 0x5500, 0x500000f, 0xff000, 0x101, 0x1010000, 0x3030, 0x300005, 0x50500, 0x50000f0, 0xf0f000, 0x1111, 0x1110003, 0x33330, 0x3300055, 0x555500, 0x5000fff, 0xffff000}); +constexpr StatTableTRI28 SQR8_TABLE_TRI28({0x1, 0x3030, 0x5000500, 0xf0e111, 0x3210000, 0x6300faa, 0x40ef10e, 0x501, 0xf0c030, 0x5110630, 0x395b444, 0x621010e, 0x6010f9b, 0x13bc4cb, 0x110001, 0x3303065, 0xff50f, 0xf0e120, 0x3243530, 0x330fabb, 0x5ec232c, 0x511050e, 0x3c1c064, 0x2ec60a, 0x3954175, 0x7c5c43d, 0x20acba, 0x943bc43}); +constexpr StatTableTRI28 QRT_TABLE_TRI28({0x121d57a, 0x40216, 0x40214, 0x8112578, 0x40210, 0x10110, 0x8112570, 0x12597ec, 0x40200, 0x6983e00, 0x10130, 0x972b99c, 0x8112530, 0x8002000, 0x125976c, 0x815a76c, 0x40300, 0x936b29c, 0x6983c00, 0x97bb8ac, 0x10530, 0x9103000, 0x972b19c, 0xf6384ac, 0x8113530, 0x4113530, 0x8000000, 0}); +typedef FieldTri<uint32_t, 28, 1, StatTableTRI28, &SQR_TABLE_TRI28, &SQR2_TABLE_TRI28, &SQR4_TABLE_TRI28, &SQR8_TABLE_TRI28, &QRT_TABLE_TRI28, &QRT_TABLE_TRI28, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri28; +#endif + +#ifdef ENABLE_FIELD_INT_29 +// 29 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6, 5> StatTable29; +constexpr StatTable29 SQR_TABLE_29({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x280000, 0xa00000, 0x2800000, 0xa000000, 0x8000005}); +constexpr StatTable29 SQR2_TABLE_29({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x28, 0x280, 0x2800, 0x28000, 0x280000, 0x2800000, 0x8000005, 0x44, 0x440, 0x4400, 0x44000, 0x440000, 0x4400000, 0x400000a, 0xaa, 0xaa0, 0xaa00, 0xaa000, 0xaa0000, 0xaa00000, 0xa000011}); +constexpr StatTable29 SQR4_TABLE_29({0x1, 0x10000, 0x28, 0x280000, 0x440, 0x4400000, 0xaa00, 0xa000011, 0x101000, 0x10000280, 0x2828000, 0x4444, 0x444000a, 0xaaaa0, 0xaa00101, 0x1000100, 0x1002800, 0x8002805, 0x8044005, 0x440aa, 0xaa00aa, 0xaa1010, 0x10101010, 0x10128280, 0x28282c4, 0x2c44444, 0x4444eaa, 0xeaaaaaa, 0xaaba001}); +constexpr StatTable29 SQR8_TABLE_29({0x1, 0x1002800, 0x4680000, 0xae50ba, 0x2822a00, 0x14545eba, 0x110aed64, 0xc6eeaaf, 0x4ee00a0, 0x10aba290, 0x1bd6efc1, 0x8222b29, 0x1c791ebf, 0x174e85da, 0x1cc66c7f, 0x29292c4, 0x2886c20, 0xea04467, 0xc0eeb87, 0xccd4115, 0x16d5fa2e, 0x1cf8fe75, 0xe45a4e1, 0x19018b3f, 0x1d64778, 0x2e0bdf8, 0xa1bd96b, 0xff5b70e, 0x14d89770}); +constexpr StatTable29 QRT_TABLE_29({0x1b8351dc, 0xb87135e, 0xb87135c, 0xda7b35e, 0xb871358, 0x621a116, 0xda7b356, 0x40200, 0xb871348, 0xc9e2620, 0x621a136, 0x478b16, 0xda7b316, 0x6762e20, 0x40280, 0x6202000, 0xb871248, 0x627a316, 0xc9e2420, 0xcd1ad36, 0x621a536, 0x760e20, 0x478316, 0xa760e20, 0xda7a316, 0x8000000, 0x6760e20, 0, 0x44280}); +typedef Field<uint32_t, 29, 5, StatTable29, &SQR_TABLE_29, &SQR2_TABLE_29, &SQR4_TABLE_29, &SQR8_TABLE_29, &QRT_TABLE_29, &QRT_TABLE_29, IdTrans, &ID_TRANS, &ID_TRANS> Field29; +typedef FieldTri<uint32_t, 29, 2, RecLinTrans<uint32_t, 6, 6, 6, 6, 5>, &SQR_TABLE_29, &SQR2_TABLE_29, &SQR4_TABLE_29, &SQR8_TABLE_29, &QRT_TABLE_29, &QRT_TABLE_29, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri29; +#endif + +#ifdef ENABLE_FIELD_INT_30 +// 30 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6, 6> StatTableTRI30; +constexpr StatTableTRI30 SQR_TABLE_TRI30({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000}); +constexpr StatTableTRI30 SQR2_TABLE_TRI30({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0xc, 0xc0, 0xc00, 0xc000, 0xc0000, 0xc00000, 0xc000000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x500000, 0x5000000, 0x10000003, 0x3c, 0x3c0, 0x3c00, 0x3c000, 0x3c0000, 0x3c00000, 0x3c000000}); +constexpr StatTableTRI30 SQR4_TABLE_TRI30({0x1, 0x10000, 0xc, 0xc0000, 0x50, 0x500000, 0x3c0, 0x3c00000, 0x1100, 0x11000000, 0xcc00, 0xc000005, 0x55000, 0x1000003f, 0x3fc000, 0x101, 0x1010000, 0xc0c, 0xc0c0000, 0x5050, 0x10500003, 0x3c3c0, 0x3c00011, 0x111100, 0x110000cc, 0xcccc00, 0xc000555, 0x5555000, 0x10003fff, 0x3fffc000}); +constexpr StatTableTRI30 SQR8_TABLE_TRI30({0x1, 0x1010000, 0xc000c, 0xc0c5050, 0x390, 0x13900012, 0x12c012c0, 0x121ddddd, 0x54100, 0x1003f33, 0xc3f0d04, 0x9555558, 0xd379000, 0x105d3fa2, 0x1d615e9e, 0x1101, 0x100100cc, 0xc0ccc09, 0x5590505, 0x3a9390, 0x3913fec, 0x13fedfcd, 0x121ddd8c, 0x11544103, 0x2cc3cff, 0x3e24c45, 0x9558bc8, 0x3a7958b, 0x1e98b158, 0x29d629e9}); +constexpr StatTableTRI30 QRT_TABLE_TRI30({0x2159df4a, 0x109134a, 0x1091348, 0x10114, 0x109134c, 0x3a203420, 0x1011c, 0x20004080, 0x109135c, 0x2005439c, 0x3a203400, 0x100400, 0x1015c, 0x3eb21930, 0x20004000, 0x20504c00, 0x109125c, 0x3b2b276c, 0x2005419c, 0x210450c0, 0x3a203000, 0x3e93186c, 0x100c00, 0x3aa23530, 0x1115c, 0x6b3286c, 0x3eb23930, 0xeb23930, 0x20000000, 0}); +typedef FieldTri<uint32_t, 30, 1, StatTableTRI30, &SQR_TABLE_TRI30, &SQR2_TABLE_TRI30, &SQR4_TABLE_TRI30, &SQR8_TABLE_TRI30, &QRT_TABLE_TRI30, &QRT_TABLE_TRI30, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri30; +#endif + +#ifdef ENABLE_FIELD_INT_31 +// 31 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5, 5, 5> StatTable31; +constexpr StatTable31 SQR_TABLE_31({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x4800000, 0x12000000, 0x48000000, 0x20000012}); +constexpr StatTable31 SQR2_TABLE_31({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x12, 0x120, 0x1200, 0x12000, 0x120000, 0x1200000, 0x12000000, 0x20000012, 0x104, 0x1040, 0x10400, 0x104000, 0x1040000, 0x10400000, 0x4000012, 0x40000120, 0x1248, 0x12480, 0x124800, 0x1248000, 0x12480000, 0x24800012, 0x48000104}); +constexpr StatTable31 SQR4_TABLE_31({0x1, 0x10000, 0x12, 0x120000, 0x104, 0x1040000, 0x1248, 0x12480000, 0x10010, 0x100012, 0x120120, 0x1200104, 0x1041040, 0x10401248, 0x12492480, 0x24810002, 0x112, 0x1120000, 0x1304, 0x13040000, 0x11648, 0x16480012, 0x134810, 0x48100116, 0x1121120, 0x11201304, 0x13053040, 0x3041165a, 0x16596492, 0x64934922, 0x49248016}); +constexpr StatTable31 SQR8_TABLE_31({0x1, 0x112, 0x10104, 0x1131648, 0x10002, 0x1120224, 0x106021a, 0x146e3f86, 0x16, 0x174c, 0x161658, 0x175b1130, 0x16002c, 0x174c2e98, 0x16742dfc, 0x3f877966, 0x114, 0x10768, 0x1151050, 0x66b75b2, 0x1140228, 0x76a0ec2, 0x127a33da, 0x79648102, 0x1738, 0x1665f0, 0x172f64e0, 0x73cc668c, 0x17382e70, 0x65dccaac, 0x4abf956e}); +constexpr StatTable31 QRT_TABLE_31({0, 0x10110, 0x10112, 0x15076e, 0x10116, 0x117130e, 0x150766, 0x4743fa0, 0x10106, 0x1121008, 0x117132e, 0x176b248e, 0x150726, 0x172a2c88, 0x4743f20, 0x7eb81e86, 0x10006, 0x20008, 0x1121208, 0x56b2c8e, 0x117172e, 0x133f1bae, 0x176b2c8e, 0x7f2a0c8e, 0x151726, 0x10000000, 0x172a0c88, 0x60000006, 0x4747f20, 0x3eb89e80, 0x7eb89e86}); +typedef Field<uint32_t, 31, 9, StatTable31, &SQR_TABLE_31, &SQR2_TABLE_31, &SQR4_TABLE_31, &SQR8_TABLE_31, &QRT_TABLE_31, &QRT_TABLE_31, IdTrans, &ID_TRANS, &ID_TRANS> Field31; +typedef FieldTri<uint32_t, 31, 3, RecLinTrans<uint32_t, 6, 5, 5, 5, 5, 5>, &SQR_TABLE_31, &SQR2_TABLE_31, &SQR4_TABLE_31, &SQR8_TABLE_31, &QRT_TABLE_31, &QRT_TABLE_31, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri31; +#endif + +#ifdef ENABLE_FIELD_INT_32 +// 32 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5, 5, 5> StatTable32; +constexpr StatTable32 SQR_TABLE_32({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x8d, 0x234, 0x8d0, 0x2340, 0x8d00, 0x23400, 0x8d000, 0x234000, 0x8d0000, 0x2340000, 0x8d00000, 0x23400000, 0x8d000000, 0x3400011a, 0xd0000468, 0x40001037}); +constexpr StatTable32 SQR2_TABLE_32({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x8d, 0x8d0, 0x8d00, 0x8d000, 0x8d0000, 0x8d00000, 0x8d000000, 0xd0000468, 0x4051, 0x40510, 0x405100, 0x4051000, 0x40510000, 0x5100234, 0x51002340, 0x100236b9, 0x236b1d, 0x236b1d0, 0x236b1d00, 0x36b1d11a, 0x6b1d1037, 0xb1d1005e, 0x1d10001f, 0xd100017d}); +constexpr StatTable32 SQR4_TABLE_32({0x1, 0x10000, 0x8d, 0x8d0000, 0x4051, 0x40510000, 0x236b1d, 0x6b1d1037, 0x10001101, 0x1109d000, 0xd00859e5, 0x59881468, 0x144737e8, 0x37e2c4e3, 0xc4f9a67a, 0xa61d8c55, 0x8c010001, 0x41dc8d, 0xdc8d23cd, 0x23a60c51, 0xc41630e, 0x63087fcd, 0x7ffe7368, 0x735580f6, 0x80cd8e29, 0x8e6fe311, 0xe350f32b, 0xf35edc90, 0xdced0bd6, 0xbbd3eb1, 0x3eb4a621, 0xa63f6bc4}); +constexpr StatTable32 SQR8_TABLE_32({0x1, 0x8c010001, 0x6b9010bb, 0x7faf6b, 0xc4da8d37, 0xc10ab646, 0x445f546c, 0xe389129e, 0xd8aa2d3e, 0x85249468, 0xd599253f, 0x458976f9, 0xc9c86411, 0xccc2f34b, 0xa79e37dc, 0x9068e3c4, 0x3a30447f, 0x674c3398, 0x94f38a7, 0x402d3532, 0x116fffc7, 0x1c6b5ba2, 0xcd6a32e4, 0x49067a77, 0xa7f6a61e, 0x3cc3746, 0xeebe962e, 0x599276e1, 0x7b5fa4d9, 0x2aa3ce1, 0x990f8767, 0x1c3b66cb}); +constexpr StatTable32 QRT_TABLE_32({0x54fd1264, 0xc26fcd64, 0xc26fcd66, 0x238a7462, 0xc26fcd62, 0x973bccaa, 0x238a746a, 0x77766712, 0xc26fcd72, 0xc1bdd556, 0x973bcc8a, 0x572a094c, 0x238a742a, 0xb693be84, 0x77766792, 0x9555c03e, 0xc26fcc72, 0x568419f8, 0xc1bdd756, 0x96c3d2ca, 0x973bc88a, 0x54861fdc, 0x572a014c, 0xb79badc4, 0x238a642a, 0xb9b99fe0, 0xb6939e84, 0xc519fa86, 0x77762792, 0, 0x9555403e, 0x377627ba}); +typedef Field<uint32_t, 32, 141, StatTable32, &SQR_TABLE_32, &SQR2_TABLE_32, &SQR4_TABLE_32, &SQR8_TABLE_32, &QRT_TABLE_32, &QRT_TABLE_32, IdTrans, &ID_TRANS, &ID_TRANS> Field32; +#endif +} + +Sketch* ConstructClMul4Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_25 + case 25: return new SketchImpl<Field25>(implementation, 25); +#endif +#ifdef ENABLE_FIELD_INT_26 + case 26: return new SketchImpl<Field26>(implementation, 26); +#endif +#ifdef ENABLE_FIELD_INT_27 + case 27: return new SketchImpl<Field27>(implementation, 27); +#endif +#ifdef ENABLE_FIELD_INT_29 + case 29: return new SketchImpl<Field29>(implementation, 29); +#endif +#ifdef ENABLE_FIELD_INT_31 + case 31: return new SketchImpl<Field31>(implementation, 31); +#endif +#ifdef ENABLE_FIELD_INT_32 + case 32: return new SketchImpl<Field32>(implementation, 32); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri4Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_25 + case 25: return new SketchImpl<FieldTri25>(implementation, 25); +#endif +#ifdef ENABLE_FIELD_INT_28 + case 28: return new SketchImpl<FieldTri28>(implementation, 28); +#endif +#ifdef ENABLE_FIELD_INT_29 + case 29: return new SketchImpl<FieldTri29>(implementation, 29); +#endif +#ifdef ENABLE_FIELD_INT_30 + case 30: return new SketchImpl<FieldTri30>(implementation, 30); +#endif +#ifdef ENABLE_FIELD_INT_31 + case 31: return new SketchImpl<FieldTri31>(implementation, 31); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_5bytes.cpp b/src/minisketch/src/fields/clmul_5bytes.cpp new file mode 100644 index 0000000000..29c3fb10e7 --- /dev/null +++ b/src/minisketch/src/fields/clmul_5bytes.cpp @@ -0,0 +1,174 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_5) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_33 +// 33 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5> StatTable33; +constexpr StatTable33 SQR_TABLE_33({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x802, 0x2008, 0x8020, 0x20080, 0x80200, 0x200800, 0x802000, 0x2008000, 0x8020000, 0x20080000, 0x80200000, 0x800401, 0x2001004, 0x8004010, 0x20010040, 0x80040100}); +constexpr StatTable33 SQR2_TABLE_33({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x2008, 0x20080, 0x200800, 0x2008000, 0x20080000, 0x800401, 0x8004010, 0x80040100, 0x400004, 0x4000040, 0x40000400, 0x4802, 0x48020, 0x480200, 0x4802000, 0x48020000, 0x80200802, 0x2009024, 0x20090240, 0x902001, 0x9020010, 0x90200100, 0x102000004, 0x20002048}); +constexpr StatTable33 SQR4_TABLE_33({0x1, 0x10000, 0x100000000, 0x2008000, 0x80040100, 0x4802, 0x48020000, 0x902001, 0x20002048, 0x20081000, 0x10400004, 0x248820, 0x88204812, 0x49020410, 0x4822081, 0x20880641, 0x6000044, 0x480300, 0x3009024, 0x90220180, 0xa00c11, 0xc104050, 0x40482608, 0x2688b024, 0xb0690344, 0x102248834, 0x8a30c912, 0xc8062518, 0x24886803, 0x684a0244, 0x294a025, 0xa020294a, 0x280a1010}); +constexpr StatTable33 SQR8_TABLE_33({0x1, 0x6000044, 0x280a1010, 0x122ac8e75, 0x83209926, 0x4a7a8a1, 0xcada863d, 0x6f2ab824, 0x6b4a8654, 0x70484bd6, 0x164c04e0b, 0x2fbc1617, 0xe095e5a3, 0xeaf7847d, 0xe5625e26, 0xa6aaa3e5, 0xc0164126, 0xd06217c0, 0x1ae58d21, 0xa8600250, 0xbaf87951, 0x8e12c19a, 0xa9b413b9, 0xb75ef087, 0x17e9214d9, 0x85968f33, 0x1e299478f, 0x92bc9a0f, 0x1975d642, 0x11af0b3f1, 0x4e86ee77, 0xe75f4726, 0x38026cce}); +constexpr StatTable33 SQR16_TABLE_33({0x1, 0x185df5e91, 0x193fb40eb, 0xd464f9e4, 0x1ba2d73a6, 0x1d9288c5e, 0x5de03a49, 0x1869ea37b, 0x13faaf379, 0x195d1a8f5, 0x6afd5625, 0xf9d75bab, 0xaf44fe50, 0x101034b9e, 0xcc889caf, 0x5ec7455, 0x7d232a66, 0x17dcfe2c3, 0x1c66ff8d0, 0x17107e836, 0x1939cdead, 0x9852afa0, 0x1b946909a, 0x1846638c5, 0xdd5fa94c, 0x1cb2600fe, 0x19241c856, 0x15fe05ccd, 0xc9f9a425, 0x89e0f463, 0x37b01b39, 0xab0410e0, 0x1ace4ca03}); +constexpr StatTable33 QRT_TABLE_33({0xba504dd4, 0x1e2798ef2, 0x1e2798ef0, 0x6698a4ec, 0x1e2798ef4, 0x1c7f1bef0, 0x6698a4e4, 0x16da1b384, 0x1e2798ee4, 0x661ca6ec, 0x1c7f1bed0, 0x1483b87a6, 0x6698a4a4, 0x800000, 0x16da1b304, 0x1a185101c, 0x1e2798fe4, 0xaa400954, 0x661ca4ec, 0x667caeec, 0x1c7f1bad0, 0x400800, 0x1483b8fa6, 0, 0x6698b4a4, 0x1c61da4b8, 0x802000, 0x16e5dadec, 0x16da1f304, 0x62fc8eec, 0x1a185901c, 0x1661da5ec, 0x1e2788fe4}); +typedef Field<uint64_t, 33, 1025, StatTable33, &SQR_TABLE_33, &SQR2_TABLE_33, &SQR4_TABLE_33, &SQR8_TABLE_33, &SQR16_TABLE_33, &QRT_TABLE_33, IdTrans, &ID_TRANS, &ID_TRANS> Field33; +typedef FieldTri<uint64_t, 33, 10, RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5>, &SQR_TABLE_33, &SQR2_TABLE_33, &SQR4_TABLE_33, &SQR8_TABLE_33, &SQR16_TABLE_33, &QRT_TABLE_33, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri33; +#endif + +#ifdef ENABLE_FIELD_INT_34 +// 34 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5> StatTable34; +constexpr StatTable34 SQR_TABLE_34({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x81, 0x204, 0x810, 0x2040, 0x8100, 0x20400, 0x81000, 0x204000, 0x810000, 0x2040000, 0x8100000, 0x20400000, 0x81000000, 0x204000000, 0x10000102, 0x40000408, 0x100001020}); +constexpr StatTable34 SQR2_TABLE_34({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x204, 0x2040, 0x20400, 0x204000, 0x2040000, 0x20400000, 0x204000000, 0x40000408, 0x4001, 0x40010, 0x400100, 0x4001000, 0x40010000, 0x100081, 0x1000810, 0x10008100, 0x100081000, 0x810204, 0x8102040, 0x81020400, 0x10204102, 0x102041020, 0x20410004, 0x204100040, 0x41000008}); +constexpr StatTable34 SQR4_TABLE_34({0x1, 0x10000, 0x100000000, 0x204000, 0x40000408, 0x4001000, 0x10008100, 0x81020400, 0x204100040, 0x304, 0x3040000, 0x6041, 0x60410000, 0x1000c1010, 0x10304183, 0x4181020c, 0x102042060, 0x20400001, 0x50010, 0x100100081, 0xa14204, 0x142041428, 0x14001001, 0x10038500, 0x385020400, 0x204704140, 0x41000f1c, 0xf143040, 0x3041e145, 0x1e1430410, 0x3042c5050, 0x5030448b, 0x4481120c, 0x112048120}); +constexpr StatTable34 SQR8_TABLE_34({0x1, 0x102042060, 0x4481120c, 0x1523455ab, 0x307081050, 0x21410f1c, 0x275d0e309, 0x3f676408a, 0x143a54d38, 0x304100344, 0x181774550, 0x1003cd092, 0x3f36b6421, 0x164d51695, 0x3e7c7f2ab, 0x9309b234, 0x354f8d24c, 0x1f5431410, 0x142012478, 0xc5225409, 0x14033f3cf, 0x123bd530c, 0x1100ee58, 0x35490c368, 0x2e1f3dcba, 0x2018108d2, 0x3c61a735d, 0xbf8fa918, 0x282ab07ea, 0x19c32af, 0x175e54c02, 0x2e4dfe2bb, 0x3374ab928, 0x3124a055}); +constexpr StatTable34 SQR16_TABLE_34({0x1, 0x3448e6f02, 0x352590eb9, 0xb173da17, 0x264977d39, 0x172d45e48, 0x1e026e5d6, 0x357b54017, 0x2925d27a4, 0x1f6a32696, 0x2f49f220c, 0x3a7383d9e, 0x28111d79b, 0x5580fcf1, 0x276ede679, 0x175b379f8, 0x34d67b66, 0xc7019416, 0x3f3d9d59f, 0x2a7c2c032, 0x2b3482ba7, 0x177cd0128, 0x1d6f4bd2e, 0x31647a632, 0x41353027, 0x56292eea, 0x2733c0501, 0x6d7ed066, 0x2f3db9a75, 0x3225bc5cc, 0x3f22da089, 0xd0a7588e, 0xb60b22d1, 0xc2fddb7e}); +constexpr StatTable34 QRT_TABLE_34({0x2f973a1f6, 0x40202, 0x40200, 0x348102060, 0x40204, 0x8000420, 0x348102068, 0x1092195c8, 0x40214, 0x3f6881b6e, 0x8000400, 0x3f810383e, 0x348102028, 0x340002068, 0x109219548, 0x24015a774, 0x40314, 0x3f050343e, 0x3f688196e, 0x3f81c3a3a, 0x8000000, 0x24031a560, 0x3f810303e, 0xb08c1a12, 0x348103028, 0xb2881906, 0x340000068, 0, 0x10921d548, 0x2e131e576, 0x240152774, 0x18921d55e, 0x50314, 0x14015271c}); +typedef Field<uint64_t, 34, 129, StatTable34, &SQR_TABLE_34, &SQR2_TABLE_34, &SQR4_TABLE_34, &SQR8_TABLE_34, &SQR16_TABLE_34, &QRT_TABLE_34, IdTrans, &ID_TRANS, &ID_TRANS> Field34; +typedef FieldTri<uint64_t, 34, 7, RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5>, &SQR_TABLE_34, &SQR2_TABLE_34, &SQR4_TABLE_34, &SQR8_TABLE_34, &SQR16_TABLE_34, &QRT_TABLE_34, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri34; +#endif + +#ifdef ENABLE_FIELD_INT_35 +// 35 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5> StatTable35; +constexpr StatTable35 SQR_TABLE_35({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x280000, 0xa00000, 0x2800000, 0xa000000, 0x28000000, 0xa0000000, 0x280000000, 0x200000005}); +constexpr StatTable35 SQR2_TABLE_35({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0xa, 0xa0, 0xa00, 0xa000, 0xa0000, 0xa00000, 0xa000000, 0xa0000000, 0x200000005, 0x44, 0x440, 0x4400, 0x44000, 0x440000, 0x4400000, 0x44000000, 0x440000000, 0x400000028, 0x2a8, 0x2a80, 0x2a800, 0x2a8000, 0x2a80000, 0x2a800000, 0x2a8000000, 0x280000011}); +constexpr StatTable35 SQR4_TABLE_35({0x1, 0x10000, 0x100000000, 0xa000, 0xa0000000, 0x4400, 0x44000000, 0x2a80, 0x2a800000, 0x1010, 0x10100000, 0xa0a, 0xa0a0000, 0x200000445, 0x4444000, 0x4400002a8, 0x2aaa800, 0x2a8000101, 0x1000100, 0x10000a0, 0xa000a0, 0xa00044, 0x440044, 0x400440028, 0x4002a8028, 0x2802a8011, 0x280101011, 0x1010100a, 0x100a0a0a, 0x20a0a0a05, 0x20a044445, 0x444444440, 0x44442aaa8, 0x2aaaaaaa8, 0x2aaa90001}); +constexpr StatTable35 SQR8_TABLE_35({0x1, 0x2aaa800, 0x44442aaa8, 0x6400006ed, 0x64e4e4e45, 0x14544000, 0x8a145454, 0x2000034df, 0x49a749a36, 0xaa0a0000, 0x10aa0aaa, 0x1ba1a, 0x393a91ba, 0x3febaaaa9, 0x285105155, 0xa0ad9ad4, 0x269ce8d3b, 0x4de74f4e6, 0x42aaa8028, 0x4002aeea8, 0x400e46eec, 0x544e4006c, 0x145440144, 0x2abede545, 0x44309e74c, 0xa74eeda4, 0x64444ee49, 0x1aa1aaaa, 0x2b90bb1b1, 0x393902109, 0x16bc47bb2, 0x271ad1511, 0x6c8f98767, 0x69d3aa74c, 0x27790dc3b}); +constexpr StatTable35 SQR16_TABLE_35({0x1, 0x4c80f98a4, 0x763684437, 0x5a1cc86a0, 0x38922db8, 0x71755e12d, 0x2ca94c627, 0x388a2bc7f, 0x406596de0, 0x1818c6958, 0x174a92efe, 0x1a80c764e, 0x2f23eacbf, 0xd611ea8, 0x64d783fd5, 0x4fdfe0798, 0x31459de8d, 0x62c889d99, 0x9c419962, 0x2d8d865b3, 0x1ac7e7ffc, 0x38a0c12f3, 0x9fbc1076, 0x6f76d3b89, 0x6e472c757, 0x5f240de42, 0x10176ecc0, 0x20c1cef8, 0x8f77f91c, 0x3f6e533b9, 0x62017c147, 0x5ce81e2fa, 0x371fe4ad9, 0x2552b5046, 0xc3f3696c}); +constexpr StatTable35 QRT_TABLE_35({0x5c2038114, 0x2bf547ee8, 0x2bf547eea, 0x2bf1074e8, 0x2bf547eee, 0x1883d0736, 0x2bf1074e0, 0x100420, 0x2bf547efe, 0x400800, 0x1883d0716, 0x5e90e4a0, 0x2bf1074a0, 0x4e70ac20, 0x1004a0, 0x2f060c880, 0x2bf547ffe, 0x37d55fffe, 0x400a00, 0x3372573de, 0x1883d0316, 0x700c20, 0x5e90eca0, 0x10604880, 0x2bf1064a0, 0x18f35377e, 0x4e708c20, 0x33f557ffe, 0x1044a0, 0x1bf557ffe, 0x2f0604880, 0x200000000, 0x2bf557ffe, 0, 0x37d57fffe}); +typedef Field<uint64_t, 35, 5, StatTable35, &SQR_TABLE_35, &SQR2_TABLE_35, &SQR4_TABLE_35, &SQR8_TABLE_35, &SQR16_TABLE_35, &QRT_TABLE_35, IdTrans, &ID_TRANS, &ID_TRANS> Field35; +typedef FieldTri<uint64_t, 35, 2, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5>, &SQR_TABLE_35, &SQR2_TABLE_35, &SQR4_TABLE_35, &SQR8_TABLE_35, &SQR16_TABLE_35, &QRT_TABLE_35, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri35; +#endif + +#ifdef ENABLE_FIELD_INT_36 +// 36 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6> StatTable36; +constexpr StatTable36 SQR_TABLE_36({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x201, 0x804, 0x2010, 0x8040, 0x20100, 0x80400, 0x201000, 0x804000, 0x2010000, 0x8040000, 0x20100000, 0x80400000, 0x201000000, 0x804000000, 0x10000402, 0x40001008, 0x100004020, 0x400010080}); +constexpr StatTable36 SQR2_TABLE_36({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x201, 0x2010, 0x20100, 0x201000, 0x2010000, 0x20100000, 0x201000000, 0x10000402, 0x100004020, 0x40001, 0x400010, 0x4000100, 0x40001000, 0x400010000, 0x100804, 0x1008040, 0x10080400, 0x100804000, 0x8040201, 0x80402010, 0x804020100, 0x40200008, 0x402000080, 0x20000004, 0x200000040, 0x2, 0x20}); +constexpr StatTable36 SQR4_TABLE_36({0x1, 0x10000, 0x100000000, 0x201000, 0x10000402, 0x4000100, 0x1008040, 0x80402010, 0x20000004, 0x200, 0x2000000, 0x4020, 0x40200000, 0x80002, 0x800020000, 0x201008000, 0x80400010, 0x4, 0x40000, 0x400000000, 0x804000, 0x40001008, 0x10000400, 0x4020100, 0x201008040, 0x80000010, 0x800, 0x8000000, 0x10080, 0x100800000, 0x200008, 0x80402, 0x804020000, 0x201000040, 0x10, 0x100000}); +constexpr StatTable36 SQR8_TABLE_36({0x1, 0x80400010, 0x804020000, 0x201008, 0x2000080, 0x20000804, 0x1008000, 0x402, 0x800000, 0x200, 0x80000010, 0x804020100, 0x40201000, 0x400010000, 0x100004, 0x201000000, 0x80400, 0x100000000, 0x40000, 0x10, 0x804000100, 0x40201008, 0x2010080, 0x20000800, 0x200008040, 0x10080000, 0x4020, 0x8000000, 0x2000, 0x800000100, 0x40200008, 0x402010000, 0x100804, 0x1000040, 0x10000402, 0x804000}); +constexpr StatTable36 SQR16_TABLE_36({0x1, 0x402000000, 0x100800020, 0x201000, 0x10080402, 0x800000000, 0x1008040, 0x400000, 0x20000800, 0x200, 0x400010080, 0x100000020, 0x40200000, 0x10080002, 0x20100, 0x201008000, 0x80000000, 0x100804, 0x40000, 0x2000080, 0x20, 0x40001008, 0x10000002, 0x4020000, 0x201008040, 0x2010, 0x20100800, 0x8000000, 0x400010000, 0x4000, 0x200008, 0x2, 0x804000000, 0x201000040, 0x402000, 0x20100804}); +constexpr StatTable36 QRT_TABLE_36({0x40200, 0x8b0526186, 0x8b0526184, 0x240001000, 0x8b0526180, 0xcb6894d94, 0x240001008, 0xdb6880c22, 0x8b0526190, 0x8000200, 0xcb6894db4, 0x500424836, 0x240001048, 0x406cb2834, 0xdb6880ca2, 0x241200008, 0x8b0526090, 0xdb05021a6, 0x8000000, 0xdb01829b2, 0xcb68949b4, 0x1001000, 0x500424036, 0x106116406, 0x240000048, 0xcb29968a4, 0x406cb0834, 0, 0xdb6884ca2, 0x110010516, 0x241208008, 0x430434520, 0x8b0536090, 0x41208040, 0xdb05221a6, 0xb6884d14}); +typedef Field<uint64_t, 36, 513, StatTable36, &SQR_TABLE_36, &SQR2_TABLE_36, &SQR4_TABLE_36, &SQR8_TABLE_36, &SQR16_TABLE_36, &QRT_TABLE_36, IdTrans, &ID_TRANS, &ID_TRANS> Field36; +typedef FieldTri<uint64_t, 36, 9, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6>, &SQR_TABLE_36, &SQR2_TABLE_36, &SQR4_TABLE_36, &SQR8_TABLE_36, &SQR16_TABLE_36, &QRT_TABLE_36, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri36; +#endif + +#ifdef ENABLE_FIELD_INT_37 +// 37 bit field +typedef RecLinTrans<uint64_t, 6, 6, 5, 5, 5, 5, 5> StatTable37; +constexpr StatTable37 SQR_TABLE_37({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0xa6, 0x298, 0xa60, 0x2980, 0xa600, 0x29800, 0xa6000, 0x298000, 0xa60000, 0x2980000, 0xa600000, 0x29800000, 0xa6000000, 0x298000000, 0xa60000000, 0x980000053, 0x60000011f, 0x180000047c}); +constexpr StatTable37 SQR2_TABLE_37({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x298, 0x2980, 0x29800, 0x298000, 0x2980000, 0x29800000, 0x298000000, 0x980000053, 0x180000047c, 0x4414, 0x44140, 0x441400, 0x4414000, 0x44140000, 0x441400000, 0x4140000a6, 0x140000ac6, 0x140000ac60, 0xac43e, 0xac43e0, 0xac43e00, 0xac43e000, 0xac43e0000, 0xc43e0011f, 0x43e00101a, 0x3e0010106, 0x1e00101033}); +constexpr StatTable37 SQR4_TABLE_37({0x1, 0x10000, 0x100000000, 0x29800, 0x298000000, 0x44140, 0x441400000, 0xac43e, 0xac43e0000, 0x1e00101033, 0x1010011000, 0x11029a980, 0x9a982b1d3, 0x2b1c45014, 0x4501005f2, 0x1005f8ef80, 0x18efa98941, 0x9897de117, 0x1de10002ad, 0x2990398, 0x190398047c, 0x180443dee4, 0x3ded94ac6, 0x194ac071fa, 0x71c56e1a, 0x56e1adff2, 0x1adffa1690, 0x1a16a9ab31, 0x9ab0957cf, 0x957d85468, 0x18547edba2, 0x1edb9fc515, 0x1fc526c1a4, 0x6c1956aab, 0x156aa5b9d4, 0x5b9f59def, 0x159de6d961}); +constexpr StatTable37 SQR8_TABLE_37({0x1, 0x18efa98941, 0x1fc526c1a4, 0x11352e16c4, 0xba7aa5340, 0x17346e075f, 0xe91c746aa, 0xe560ac1bd, 0xa4544c5d9, 0x11bd3c631f, 0xd70c4b63c, 0xfe77d107c, 0x10548e5288, 0x1183954fb3, 0x19b3aa4bb, 0x782a2943c, 0x1c19ba61de, 0x6ad01fe38, 0xa22701577, 0xb96546ca0, 0x1d7c6c8b9c, 0xffef807e2, 0x16fcc14dc2, 0x110cc4e83c, 0xc3a35629a, 0x1062330476, 0xb2e5d1de1, 0x1ca4e3d229, 0x67826b51b, 0xe7e4c36e7, 0x59f1ac963, 0x12777f22c6, 0x13963d623a, 0x9e305ac92, 0x219b91d13, 0x175bebeb0d, 0xc6b7b5572}); +constexpr StatTable37 SQR16_TABLE_37({0x1, 0xcb88f2f8b, 0x1a2a0be7af, 0xb93048ada, 0x113ed92190, 0xc95a18e2b, 0x1e1cd4a85b, 0x19584a1a66, 0x1b947c28c2, 0x1b52b48e27, 0xe64e7b169, 0x14a256d011, 0xda657196d, 0x1947c1dcb4, 0x18b2fa3851, 0xae3d4171a, 0x658f1f4b9, 0x91852c314, 0x69346cf8e, 0x8224bf36c, 0x1086c810ed, 0x10419bc782, 0x57d6a4e36, 0xfbb31a43e, 0x18b502de05, 0x786795174, 0x1de0f1b7f3, 0x1d456b87dc, 0x1aabb2f3bc, 0xc5b80ef0c, 0x1ce4fd7543, 0x7ca740ca1, 0x29eaec26a, 0x1eb0b42043, 0xca3b2b17, 0x3453101c1, 0x1714c59187}); +constexpr StatTable37 QRT_TABLE_37({0xa3c62e7ba, 0xdc7a0c16a, 0xdc7a0c168, 0x12f7484546, 0xdc7a0c16c, 0xa9803a20, 0x12f748454e, 0xda07064a4, 0xdc7a0c17c, 0x123908de8e, 0xa9803a00, 0x122a888a8e, 0x12f748450e, 0x6790add8, 0xda0706424, 0x12e0a0384c, 0xdc7a0c07c, 0xcb28a2c2, 0x123908dc8e, 0xd09f85e86, 0xa9803e00, 0x124d682b6e, 0x122a88828e, 0x1738711a, 0x12f748550e, 0x73035b8, 0x67908dd8, 0xa0702438, 0xda0702424, 0xe0a0b860, 0x12e0a0b84c, 0x1c7a1c060, 0xdc7a1c07c, 0, 0xcb2aa2c2, 0x100000002c, 0x12390cdc8e}); +typedef Field<uint64_t, 37, 83, StatTable37, &SQR_TABLE_37, &SQR2_TABLE_37, &SQR4_TABLE_37, &SQR8_TABLE_37, &SQR16_TABLE_37, &QRT_TABLE_37, IdTrans, &ID_TRANS, &ID_TRANS> Field37; +#endif + +#ifdef ENABLE_FIELD_INT_38 +// 38 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5, 5> StatTable38; +constexpr StatTable38 SQR_TABLE_38({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x63, 0x18c, 0x630, 0x18c0, 0x6300, 0x18c00, 0x63000, 0x18c000, 0x630000, 0x18c0000, 0x6300000, 0x18c00000, 0x63000000, 0x18c000000, 0x630000000, 0x18c0000000, 0x2300000063, 0xc0000014a, 0x3000000528}); +constexpr StatTable38 SQR2_TABLE_38({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x18c, 0x18c0, 0x18c00, 0x18c000, 0x18c0000, 0x18c00000, 0x18c000000, 0x18c0000000, 0xc0000014a, 0x1405, 0x14050, 0x140500, 0x1405000, 0x14050000, 0x140500000, 0x1405000000, 0x500001ef, 0x500001ef0, 0x100001ef63, 0x1ef7bc, 0x1ef7bc0, 0x1ef7bc00, 0x1ef7bc000, 0x1ef7bc0000, 0x2f7bc00129, 0x37bc00112d, 0x3bc0011027, 0x3c00110022}); +constexpr StatTable38 SQR4_TABLE_38({0x1, 0x10000, 0x100000000, 0x18c00, 0x18c000000, 0x14050, 0x140500000, 0x100001ef63, 0x1ef7bc000, 0x3bc0011027, 0x110001100, 0x110194c0, 0x194c0194c, 0x194d5455, 0xd5455154f, 0x151544a193, 0x4a18c631f, 0xc6319c6ca, 0x19c6c00014, 0x18c8d, 0x18c8d0000, 0xd00014096, 0x1409ddc00, 0x1ddc01efc6, 0x1efd5ab90, 0x15ab9110e1, 0x1110fe85b2, 0x3e85ab5465, 0x2b5445c97a, 0x5c9450993, 0x50994148f, 0x141488b12a, 0x8b134ee36, 0x34ee3a8ecc, 0x3a8ee3edc8, 0x23edeef7ed, 0x2ef7de8bf9, 0x1e8bc14041}); +constexpr StatTable38 SQR8_TABLE_38({0x1, 0x4a18c631f, 0x8b134ee36, 0x10b5c9474c, 0x3330e98ecb, 0x939897650, 0xd74b026b9, 0x860251dd9, 0x3afbe829b4, 0x3ae6afc308, 0x239ecafe00, 0x2acbc94749, 0x3a5770e19e, 0x4052e180b, 0x321fa15712, 0x3a8a4869ef, 0x1948598082, 0x3b1bd98542, 0xc1deb9112, 0x1b5c9242e, 0x338ba58e8b, 0x8abe06d20, 0x145bb1d2a9, 0x1d6e10fbf0, 0x197d522629, 0x2ff1bbe50d, 0xcc1594a16, 0xc94db1b03, 0x3b20e51c56, 0x101d1e5d07, 0x19472478f7, 0x269635a968, 0x2fd4a35802, 0x1b63e116b6, 0x19fdf9d22a, 0x2ef0e4d419, 0x3e80f730f4, 0x29869b04b9}); +constexpr StatTable38 SQR16_TABLE_38({0x1, 0x3f5fe2afaa, 0x4216541b5, 0x33b362f56a, 0x9d630d7e1, 0x11127694c1, 0x3f8daab2d6, 0x153ca20edc, 0x22a747a3de, 0xc6ab16040, 0x19cc9a7e37, 0x449d96001, 0x45a7e7c46, 0x36d11561ce, 0x114b93f52a, 0x42a87f1b3, 0x23112a30bc, 0x400df9212, 0x3aca9544df, 0x140c4b0bcf, 0x2ae2efa6d3, 0x2f7051159c, 0x19cca2f62e, 0x102023d8c0, 0xccc793f0b, 0x2ff4789b55, 0x339e4cd9ba, 0x2b02ab5052, 0x8c1b5db82, 0x2e461e4e32, 0xd93541605, 0x1acf12087, 0x33b88dca2b, 0x1e91723c8b, 0xd81047b2b, 0x2e5e54b97c, 0x85bb507d8, 0x2145b1864b}); +constexpr StatTable38 QRT_TABLE_38({0x34b0ac6430, 0x2223262fa, 0x2223262f8, 0x35554405fe, 0x2223262fc, 0x355514098a, 0x35554405f6, 0x400840, 0x2223262ec, 0x1777726532, 0x35551409aa, 0x15c06fc0, 0x35554405b6, 0x1f5303fec, 0x4008c0, 0x236a21030, 0x2223263ec, 0x1a9008c00, 0x1777726732, 0x3692c60ab6, 0x3555140daa, 0x15556007ee, 0x15c067c0, 0x14a0b030f2, 0x35554415b6, 0x227c06d168, 0x1f5301fec, 0x16c3928fc2, 0x4048c0, 0x3a942c4c0, 0x236a29030, 0x1636a2902e, 0x2223363ec, 0x3a6e898276, 0x1a9028c00, 0x6de74eb2c, 0x1777766732, 0}); +typedef Field<uint64_t, 38, 99, StatTable38, &SQR_TABLE_38, &SQR2_TABLE_38, &SQR4_TABLE_38, &SQR8_TABLE_38, &SQR16_TABLE_38, &QRT_TABLE_38, IdTrans, &ID_TRANS, &ID_TRANS> Field38; +#endif + +#ifdef ENABLE_FIELD_INT_39 +// 39 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5> StatTable39; +constexpr StatTable39 SQR_TABLE_39({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x22, 0x88, 0x220, 0x880, 0x2200, 0x8800, 0x22000, 0x88000, 0x220000, 0x880000, 0x2200000, 0x8800000, 0x22000000, 0x88000000, 0x220000000, 0x880000000, 0x2200000000, 0x800000011, 0x2000000044}); +constexpr StatTable39 SQR2_TABLE_39({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x22, 0x220, 0x2200, 0x22000, 0x220000, 0x2200000, 0x22000000, 0x220000000, 0x2200000000, 0x2000000044, 0x404, 0x4040, 0x40400, 0x404000, 0x4040000, 0x40400000, 0x404000000, 0x4040000000, 0x400000088, 0x4000000880, 0x8888, 0x88880, 0x888800, 0x8888000, 0x88880000, 0x888800000, 0x888000011, 0x880000101, 0x800001001}); +constexpr StatTable39 SQR4_TABLE_39({0x1, 0x10000, 0x100000000, 0x2200, 0x22000000, 0x404, 0x4040000, 0x400000088, 0x888800, 0x888000011, 0x100010, 0x1000100000, 0x1000022000, 0x220022000, 0x220004040, 0x40404040, 0x4040400880, 0x4008888880, 0x888888101, 0x881000001, 0x122, 0x1220000, 0x2200000022, 0x260400, 0x2604000000, 0x48c88, 0x48c880000, 0x800009889, 0x98881000, 0x810001221, 0x12201220, 0x2012200264, 0x2002604264, 0x6042604044, 0x604048c8c4, 0x48c8c8c880, 0x48c8898881, 0x988888881, 0x888802201}); +constexpr StatTable39 SQR8_TABLE_39({0x1, 0x4040400880, 0x2002604264, 0xaa8022011, 0x810049ea9, 0x100100010, 0xc04008101, 0x644048ea4c, 0x18c1764441, 0x60f8e8526c, 0x22000122, 0x48c88989a3, 0xae0032001, 0x2a7aeafae5, 0x6a76641225, 0x2036245242, 0x3e9ab0308b, 0x1c49f6fe41, 0x681b069e2d, 0x4edee8cae5, 0x898c04, 0x660daa8880, 0x69cae9ccc1, 0x4881320991, 0xd06280001, 0x1cc8c8e3d9, 0x445fc65628, 0x4c889a8a49, 0x300b8caeec, 0x50d842fc94, 0x1811acb89d, 0x9d22101c, 0x2025aa407e, 0x20370a744a, 0x3cf77cb80b, 0x54a13e66e7, 0x34c17e2e04, 0x5c19fe54c1, 0x6a72cc767d}); +constexpr StatTable39 SQR16_TABLE_39({0x1, 0x37214861ce, 0x689e897065, 0x5678d6ee60, 0x619da834c4, 0x28352752d3, 0x14fed69ec6, 0x5b3d4aa637, 0x682fb8da4d, 0x2ce48c5615, 0x1591ac539c, 0x72d4fbcd0, 0x346b547296, 0x1e7065d419, 0x4e6eb48571, 0x26615d4c2c, 0x60d1c6122e, 0x78d0e2a2eb, 0x52bb3e2980, 0x3c2592d0ab, 0x701ba76b58, 0x5fdf53b685, 0x57cfd2d120, 0x75559e4344, 0x3837a46907, 0x15f961a4ce, 0x397b9a03e9, 0x5a8dd4ab69, 0x3a6ab3356f, 0x215d39c25e, 0x5bbaf82443, 0x6759e3c88c, 0x3c0b862ca1, 0x37eec7e79e, 0x6ce865e38, 0x4a56a338c0, 0x5684636aee, 0x325a019126, 0x24f18a4ef6}); +constexpr StatTable39 QRT_TABLE_39({0x66b02a408c, 0x100420, 0x100422, 0x14206080, 0x100426, 0x5dccefab1c, 0x14206088, 0x9fc11e5b6, 0x100436, 0x5466bea62a, 0x5dccefab3c, 0x9aa110536, 0x142060c8, 0x54739ed6e2, 0x9fc11e536, 0xe7a82c080, 0x100536, 0x4002000, 0x5466bea42a, 0x6a4022000, 0x5dccefaf3c, 0x9e8118536, 0x9aa110d36, 0x5680e080, 0x142070c8, 0x7d293c5b6, 0x54739ef6e2, 0x8d680e080, 0x9fc11a536, 0x6d282c080, 0xe7a824080, 0x800000000, 0x110536, 0x2d680e080, 0x4022000, 0, 0x5466baa42a, 0x46b03a44aa, 0x6a40a2000}); +typedef Field<uint64_t, 39, 17, StatTable39, &SQR_TABLE_39, &SQR2_TABLE_39, &SQR4_TABLE_39, &SQR8_TABLE_39, &SQR16_TABLE_39, &QRT_TABLE_39, IdTrans, &ID_TRANS, &ID_TRANS> Field39; +typedef FieldTri<uint64_t, 39, 4, RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5>, &SQR_TABLE_39, &SQR2_TABLE_39, &SQR4_TABLE_39, &SQR8_TABLE_39, &SQR16_TABLE_39, &QRT_TABLE_39, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri39; +#endif + +#ifdef ENABLE_FIELD_INT_40 +// 40 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5> StatTable40; +constexpr StatTable40 SQR_TABLE_40({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x39, 0xe4, 0x390, 0xe40, 0x3900, 0xe400, 0x39000, 0xe4000, 0x390000, 0xe40000, 0x3900000, 0xe400000, 0x39000000, 0xe4000000, 0x390000000, 0xe40000000, 0x3900000000, 0xe400000000, 0x900000004b, 0x400000015e}); +constexpr StatTable40 SQR2_TABLE_40({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x39, 0x390, 0x3900, 0x39000, 0x390000, 0x3900000, 0x39000000, 0x390000000, 0x3900000000, 0x900000004b, 0x541, 0x5410, 0x54100, 0x541000, 0x5410000, 0x54100000, 0x541000000, 0x5410000000, 0x41000000dd, 0x1000000d34, 0xd379, 0xd3790, 0xd37900, 0xd379000, 0xd3790000, 0xd37900000, 0xd379000000, 0x3790000115, 0x790000111b, 0x900001111f}); +constexpr StatTable40 SQR4_TABLE_40({0x1, 0x10000, 0x100000000, 0x3900, 0x39000000, 0x541, 0x5410000, 0x41000000dd, 0xd37900, 0xd379000000, 0x111001, 0x1110010000, 0x10003aa90, 0x3aa903900, 0x903900511a, 0x51051541, 0x515410de9, 0x410de9de4d, 0xe9de437815, 0x437801010e, 0x101000038, 0x383939, 0x3839390000, 0x3900057d41, 0x57d444100, 0x444100d6e5, 0xd6ebaa79, 0xebaa7911c6, 0x7911d2791a, 0xd2791102a9, 0x1102b82901, 0xb82902a972, 0x2a96bfed1, 0x6bfed16851, 0xd16859f42e, 0x59f43f61a8, 0x3f61a43794, 0xa43791de59, 0x91de42401f, 0x424000390e}); +constexpr StatTable40 SQR8_TABLE_40({0x1, 0x515410de9, 0x2a96bfed1, 0x13ba41ea90, 0x45bffe2b75, 0x5836900, 0x3887d7e690, 0xd34b688712, 0xc7a3d51557, 0xd1151ada71, 0x51442a740, 0x41cc5cbdb6, 0xc61a5701e9, 0x8757946d91, 0xa99e8b9e65, 0x80a0aca777, 0xc242b5c0e9, 0x6826eccb25, 0xad687ebd2d, 0xad5c69d802, 0x7ed2f8390, 0x51fa78eedf, 0xc0718c96f6, 0xaf4672a8c2, 0xc67436f2fd, 0x56ddb12767, 0x535afb0326, 0xbce1edda33, 0xef36202f0f, 0x45d13015ec, 0x104ab11aef, 0xef96c86d49, 0xc1b790bfc9, 0x2fa610e77f, 0x2a10a27d6e, 0xca5bb10773, 0xfdaf2b4642, 0xb3b4b7e20d, 0xe8bbe4d22e, 0xf9986bd2df}); +constexpr StatTable40 SQR16_TABLE_40({0x1, 0xe88450a7de, 0x3a0a56c3e8, 0x1684757d36, 0xc7f40bf3e9, 0x38aa7009c0, 0x2b6f129659, 0xd1e0fc42e5, 0x96150bc554, 0x9774ef4cc1, 0xd34eebf74d, 0x2d183441ec, 0xeedf6d1c78, 0x3f93c5d217, 0xb924305809, 0xc383bb7c14, 0x3f242bb94e, 0x9313556f6b, 0x2f5e1ecc6b, 0x2e7f9df195, 0xac8b882870, 0xd14f457f55, 0xf9f936148d, 0x719190770, 0x6838b41a21, 0xb95ff30106, 0xc1527dd1c5, 0xe858b5f9b6, 0x9368a791c2, 0x7de23878af, 0x95c610d398, 0xed0edcb032, 0x9548a680b0, 0xc133469e7b, 0x68c96ccbb2, 0x7773231ebb, 0xbd5ef4207c, 0xdf8bd59374, 0xb862414268, 0xfa62b39e42}); +constexpr StatTable40 QRT_TABLE_40({0x624b3cecc, 0xbc5c3f4c6, 0xbc5c3f4c4, 0xde1603e2c, 0xbc5c3f4c0, 0xaabec06cea, 0xde1603e24, 0x6cd9f724c2, 0xbc5c3f4d0, 0xcde1743818, 0xaabec06cca, 0xa138c314ca, 0xde1603e64, 0xaafc00f01a, 0x6cd9f72442, 0xcdca11bb4, 0xbc5c3f5d0, 0xa00002001a, 0xcde1743a18, 0xdf1407b90, 0xaabec068ca, 0xc043b482c8, 0xa138c31cca, 0xcb86977e3c, 0xde1602e64, 0x604596a326, 0xaafc00d01a, 0xcc1c165d0, 0x6cd9f76442, 0x673c94da26, 0xcdca19bb4, 0x67c0940a26, 0xbc5c2f5d0, 0xa4dca19bae, 0xa00000001a, 0x1bc5c2f5d0, 0xcde1703a18, 0, 0xdf1487b90, 0x8df1487b8a}); +typedef Field<uint64_t, 40, 57, StatTable40, &SQR_TABLE_40, &SQR2_TABLE_40, &SQR4_TABLE_40, &SQR8_TABLE_40, &SQR16_TABLE_40, &QRT_TABLE_40, IdTrans, &ID_TRANS, &ID_TRANS> Field40; +#endif +} + +Sketch* ConstructClMul5Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_33 + case 33: return new SketchImpl<Field33>(implementation, 33); +#endif +#ifdef ENABLE_FIELD_INT_34 + case 34: return new SketchImpl<Field34>(implementation, 34); +#endif +#ifdef ENABLE_FIELD_INT_35 + case 35: return new SketchImpl<Field35>(implementation, 35); +#endif +#ifdef ENABLE_FIELD_INT_36 + case 36: return new SketchImpl<Field36>(implementation, 36); +#endif +#ifdef ENABLE_FIELD_INT_37 + case 37: return new SketchImpl<Field37>(implementation, 37); +#endif +#ifdef ENABLE_FIELD_INT_38 + case 38: return new SketchImpl<Field38>(implementation, 38); +#endif +#ifdef ENABLE_FIELD_INT_39 + case 39: return new SketchImpl<Field39>(implementation, 39); +#endif +#ifdef ENABLE_FIELD_INT_40 + case 40: return new SketchImpl<Field40>(implementation, 40); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri5Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_33 + case 33: return new SketchImpl<FieldTri33>(implementation, 33); +#endif +#ifdef ENABLE_FIELD_INT_34 + case 34: return new SketchImpl<FieldTri34>(implementation, 34); +#endif +#ifdef ENABLE_FIELD_INT_35 + case 35: return new SketchImpl<FieldTri35>(implementation, 35); +#endif +#ifdef ENABLE_FIELD_INT_36 + case 36: return new SketchImpl<FieldTri36>(implementation, 36); +#endif +#ifdef ENABLE_FIELD_INT_39 + case 39: return new SketchImpl<FieldTri39>(implementation, 39); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_6bytes.cpp b/src/minisketch/src/fields/clmul_6bytes.cpp new file mode 100644 index 0000000000..d0e712400a --- /dev/null +++ b/src/minisketch/src/fields/clmul_6bytes.cpp @@ -0,0 +1,170 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_6) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_41 +// 41 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5> StatTable41; +constexpr StatTable41 SQR_TABLE_41({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x4800000, 0x12000000, 0x48000000, 0x120000000, 0x480000000, 0x1200000000, 0x4800000000, 0x12000000000, 0x8000000012}); +constexpr StatTable41 SQR2_TABLE_41({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x48, 0x480, 0x4800, 0x48000, 0x480000, 0x4800000, 0x48000000, 0x480000000, 0x4800000000, 0x8000000012, 0x104, 0x1040, 0x10400, 0x104000, 0x1040000, 0x10400000, 0x104000000, 0x1040000000, 0x10400000000, 0x4000000048, 0x492, 0x4920, 0x49200, 0x492000, 0x4920000, 0x49200000, 0x492000000, 0x4920000000, 0x9200000012, 0x12000000104}); +constexpr StatTable41 SQR4_TABLE_41({0x1, 0x10000, 0x100000000, 0x480, 0x4800000, 0x8000000012, 0x104000, 0x1040000000, 0x4920, 0x49200000, 0x12000000104, 0x1001000, 0x10010000000, 0x48048, 0x480480000, 0x4800001040, 0x10410400, 0x4104000048, 0x492492, 0x4924920000, 0x9200010002, 0x100000100, 0x1000480, 0x10004800000, 0x8000048012, 0x480104000, 0x1040001040, 0x10404920, 0x4049200048, 0x12000492104, 0x4921001000, 0x10010010010, 0x100148048, 0x1480480480, 0x4804805840, 0x8058410412, 0x410410414c, 0x10414d2492, 0x14d24924920, 0x9249259202, 0x12592000004}); +constexpr StatTable41 SQR8_TABLE_41({0x1, 0x10410400, 0x100148048, 0x13040000104, 0x11044801040, 0x924da49202, 0x9680490002, 0x82510514c, 0x8481485932, 0xc83144d832, 0x134d34d6db6, 0x18010048012, 0x165db20004c, 0x4800597052, 0x10131135cd0, 0xcd6cc30d32, 0x160586101cc, 0x15c64969da8, 0x179715681cc, 0x12f3c0ffc74, 0xc7dd3dd3ce, 0x10014c968, 0x1b040048116, 0x35d6801044, 0xda4deda6d0, 0x1de94c85852, 0x1083500114c, 0xc4c9685dfa, 0x18515c6d592, 0x17de69aed7e, 0x16c6c8a6c6c, 0x165cfe1044c, 0xdb004cf018, 0x7075031c98, 0x1d9a90b0d72, 0x1bb2485caee, 0x1cbe4dfd48a, 0x1f1540b7400, 0xc62bc7fd02, 0x147b5103f2e, 0x390ee8bcc6}); +constexpr StatTable41 SQR16_TABLE_41({0x1, 0x61deee38fe, 0xe00adae2e, 0x1ea53eaa95a, 0x503e540566, 0xabc8e7f89a, 0x1bf760d86ac, 0x94cce9c722, 0x15c8006ee5c, 0x7aba20c1da, 0x12662a9603e, 0x5fe76acec4, 0x1e6beca9e42, 0x1efc8f7a000, 0x165997c6d7e, 0xee947a07ee, 0xd9bd741142, 0xaa304566c0, 0x5fe336e356, 0x11f1021b80c, 0xd34e5a1674, 0x99ed56b9dc, 0x9afae0eca, 0x1a5830b390e, 0x1be1a63eb7e, 0x141e77e141c, 0xee3be92168, 0xa93823d65c, 0x18a59f4b19c, 0xce69942af6, 0x3f7b319c0e, 0xba83a4a7b4, 0x7da4b6fcde, 0x17f79268f10, 0x1222602d048, 0x1b4b2f326b8, 0x159abff0786, 0xb35534a7a2, 0x84bbc48050, 0x173d5cbf330, 0x2897dd6f58}); +constexpr StatTable41 QRT_TABLE_41({0, 0x1599a5e0b0, 0x1599a5e0b2, 0x105c119e0, 0x1599a5e0b6, 0x1a2030452a6, 0x105c119e8, 0x1a307c55b2e, 0x1599a5e0a6, 0x1ee3f47bc8e, 0x1a203045286, 0x400808, 0x105c119a8, 0x1a3038573a6, 0x1a307c55bae, 0x4d2882a520, 0x1599a5e1a6, 0x1ffbaa0b720, 0x1ee3f47be8e, 0x4d68c22528, 0x1a203045686, 0x200006, 0x400008, 0x1b79a21b200, 0x105c109a8, 0x1ef3886a526, 0x1a3038553a6, 0x1b692209200, 0x1a307c51bae, 0x5d99a4e1a6, 0x4d28822520, 0x185e109ae, 0x1599a4e1a6, 0x4e3f43be88, 0x1ffbaa2b720, 0x4000000000, 0x1ee3f43be8e, 0x18000000006, 0x4d68ca2528, 0xa203145680, 0x1a203145686}); +typedef Field<uint64_t, 41, 9, StatTable41, &SQR_TABLE_41, &SQR2_TABLE_41, &SQR4_TABLE_41, &SQR8_TABLE_41, &SQR16_TABLE_41, &QRT_TABLE_41, IdTrans, &ID_TRANS, &ID_TRANS> Field41; +typedef FieldTri<uint64_t, 41, 3, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5>, &SQR_TABLE_41, &SQR2_TABLE_41, &SQR4_TABLE_41, &SQR8_TABLE_41, &SQR16_TABLE_41, &QRT_TABLE_41, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri41; +#endif + +#ifdef ENABLE_FIELD_INT_42 +// 42 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6> StatTable42; +constexpr StatTable42 SQR_TABLE_42({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x81, 0x204, 0x810, 0x2040, 0x8100, 0x20400, 0x81000, 0x204000, 0x810000, 0x2040000, 0x8100000, 0x20400000, 0x81000000, 0x204000000, 0x810000000, 0x2040000000, 0x8100000000, 0x20400000000, 0x1000000102, 0x4000000408, 0x10000001020}); +constexpr StatTable42 SQR2_TABLE_42({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x204, 0x2040, 0x20400, 0x204000, 0x2040000, 0x20400000, 0x204000000, 0x2040000000, 0x20400000000, 0x4000000408, 0x4001, 0x40010, 0x400100, 0x4001000, 0x40010000, 0x400100000, 0x4001000000, 0x10000081, 0x100000810, 0x1000008100, 0x10000081000, 0x810204, 0x8102040, 0x81020400, 0x810204000, 0x8102040000, 0x1020400102, 0x10204001020, 0x2040010004, 0x20400100040, 0x4001000008}); +constexpr StatTable42 SQR4_TABLE_42({0x1, 0x10000, 0x100000000, 0x2040, 0x20400000, 0x4000000408, 0x4001000, 0x10000081, 0x810204, 0x8102040000, 0x20400100040, 0x1000000100, 0x1020400, 0x10204000000, 0x200001, 0x2000010000, 0x100040800, 0x408002040, 0x20408002, 0x4080020408, 0x204000020, 0x204001, 0x2040010000, 0x100040010, 0x400102040, 0x1020408100, 0x4081020008, 0x10200000020, 0x80, 0x800000, 0x8000000000, 0x102000, 0x1020000000, 0x20008, 0x200080000, 0x800004080, 0x40810200, 0x8102000810, 0x20008000040, 0x8102, 0x81020000, 0x10200001020}); +constexpr StatTable42 SQR8_TABLE_42({0x1, 0x100040800, 0x1020000000, 0x10200001000, 0x2040810000, 0x408102040, 0x4080000408, 0x10000000, 0x8002040810, 0x20008000, 0x10200080000, 0x40000004, 0x20408000040, 0x4000020000, 0x204000, 0x8002040010, 0x408102, 0x200081020, 0x40810000, 0x2000, 0x81000408, 0x4001, 0x2040010, 0x1020008002, 0x10204081020, 0x800004, 0x20000000000, 0x4081000400, 0x10000081, 0x100040010, 0x8102, 0x10000000020, 0x40010200, 0x408000000, 0x4080000400, 0x810204000, 0x102040810, 0x1020000102, 0x4000000, 0x2000810204, 0x8002000, 0x4080020000}); +constexpr StatTable42 SQR16_TABLE_42({0x1, 0x40800204, 0x2000800, 0x20008002040, 0x408100, 0x20000, 0x10004081020, 0x10000081, 0x40010204, 0x8102000000, 0x400000000, 0x1020008102, 0x1020408, 0x204081020, 0x200001, 0x10200, 0x102000010, 0x20408100000, 0x1020408002, 0x4000020000, 0x204000000, 0x204001, 0x2000800004, 0x8100000010, 0x400102000, 0x8002, 0x4080020000, 0x10000001000, 0x80, 0x2040010200, 0x100040000, 0x400100040, 0x20408000, 0x1000000, 0x204080020, 0x800004080, 0x2000810200, 0x8100000810, 0x20000000000, 0x1000408002, 0x81020400, 0x10204081000}); +constexpr StatTable42 QRT_TABLE_42({0x810200080, 0x120810806, 0x120810804, 0x1068c1a1000, 0x120810800, 0x34005023008, 0x1068c1a1008, 0x800004080, 0x120810810, 0x162818a10, 0x34005023028, 0x42408a14, 0x1068c1a1048, 0x1001040, 0x800004000, 0xb120808906, 0x120810910, 0x34000020068, 0x162818810, 0x68c021400, 0x34005023428, 0x10004000, 0x42408214, 0x162418214, 0x1068c1a0048, 0xb002018116, 0x1003040, 0x10008180448, 0x800000000, 0x62c08b04, 0xb120800906, 0x2408d1a3060, 0x120800910, 0x34401003028, 0x34000000068, 0, 0x162858810, 0xa042058116, 0x68c0a1400, 0x8162858806, 0x34005123428, 0x3068c0a1468}); +typedef Field<uint64_t, 42, 129, StatTable42, &SQR_TABLE_42, &SQR2_TABLE_42, &SQR4_TABLE_42, &SQR8_TABLE_42, &SQR16_TABLE_42, &QRT_TABLE_42, IdTrans, &ID_TRANS, &ID_TRANS> Field42; +typedef FieldTri<uint64_t, 42, 7, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6>, &SQR_TABLE_42, &SQR2_TABLE_42, &SQR4_TABLE_42, &SQR8_TABLE_42, &SQR16_TABLE_42, &QRT_TABLE_42, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri42; +#endif + +#ifdef ENABLE_FIELD_INT_43 +// 43 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5, 5, 5> StatTable43; +constexpr StatTable43 SQR_TABLE_43({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0xb2, 0x2c8, 0xb20, 0x2c80, 0xb200, 0x2c800, 0xb2000, 0x2c8000, 0xb20000, 0x2c80000, 0xb200000, 0x2c800000, 0xb2000000, 0x2c8000000, 0xb20000000, 0x2c80000000, 0xb200000000, 0x2c800000000, 0x32000000059, 0x4800000013d, 0x20000000446}); +constexpr StatTable43 SQR2_TABLE_43({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0xb2, 0xb20, 0xb200, 0xb2000, 0xb20000, 0xb200000, 0xb2000000, 0xb20000000, 0xb200000000, 0x32000000059, 0x20000000446, 0x4504, 0x45040, 0x450400, 0x4504000, 0x45040000, 0x450400000, 0x4504000000, 0x45040000000, 0x504000002c8, 0x4000002efa, 0x4000002efa0, 0x2ef8c8, 0x2ef8c80, 0x2ef8c800, 0x2ef8c8000, 0x2ef8c80000, 0x2ef8c800000, 0x6f8c800013d, 0x78c80001025, 0xc800010117, 0x48000101129}); +constexpr StatTable43 SQR4_TABLE_43({0x1, 0x10000, 0x100000000, 0xb20, 0xb200000, 0x32000000059, 0x450400, 0x4504000000, 0x4000002efa0, 0x2ef8c8000, 0x78c80001025, 0x10110010, 0x11001000b2, 0x1000b2b920, 0xb2b920b200, 0x120b204545f, 0x20454554046, 0x45540506efa, 0x506ed4df68, 0x6d4df6a79f5, 0x76a79c80133, 0x1c801010007, 0x101000b2100, 0xb210b2b20, 0x10b2b204504, 0x320450f655d, 0x50f654106c8, 0x54106efcb4c, 0x6efcb696320, 0x369631c93e1, 0x31c93ff9d8c, 0x3ff9d91a2a2, 0x591a2b9839b, 0x2b983b98dd4, 0x3b98dc651b0, 0x5c651a971e9, 0x1a971c9c0ba, 0x1c9c0b5853e, 0xb585322d78, 0x5322d7c6430, 0x57c6416617d, 0x4166159c82c, 0x159c8000b6c}); +constexpr StatTable43 SQR8_TABLE_43({0x1, 0x20454554046, 0x591a2b9839b, 0x722ff9f6fe9, 0x6a269c1eb12, 0xa3ce9f234e, 0x4d9ba8aae0b, 0x392cf0cf99b, 0x465f8594525, 0x4f9c1fb1524, 0x3b1a1dd441c, 0x381edd42255, 0x37a599424b, 0x554caee8670, 0x5335bb91d81, 0x69288c8a1a3, 0x3df2b6e68e5, 0x75330d31d56, 0x51a604b090c, 0x32e0d5a7ca3, 0x41eb9d4896e, 0x633e2855c9f, 0x4780d70e32f, 0x73b0cd728c3, 0x16627402bad, 0x4418f2a818a, 0x5cdd06cf7e5, 0x2a8da97a3ae, 0x446864a8976, 0x5a7bcbd45ea, 0x4034a4b8b04, 0x6bdaac9c5fa, 0x18769ce3a67, 0x560278257c, 0x41c06d6b64c, 0x69f6f61bd4b, 0x16cc45f84fd, 0x53f6b42f0f0, 0x6cac3d234f3, 0x1f94e8f24d5, 0x319342c7148, 0x8685bca86d, 0x6b694a6ea66}); +constexpr StatTable43 SQR16_TABLE_43({0x1, 0x1ce77599049, 0x191715250a, 0xc1573d8dff, 0x118e73ab5e4, 0x4b6a83225fe, 0x72b4bc8e0f5, 0x4a4b2b6bb02, 0x66daf4741e9, 0x50baba19898, 0x5eb38771912, 0x6fb458aad3c, 0x5ce3b10bde9, 0x5575f3498f0, 0x5f075aa8a0a, 0x41d0aa8ee20, 0x609e3c78c28, 0xe2e45a8018, 0x523ac062837, 0x738388a569d, 0x6616ec46da9, 0x1a75cc16d96, 0x49b0b43bbc3, 0x400416b3c9a, 0x25813f41ffe, 0x309fdb9d0bc, 0x489f45b2cbf, 0xa141f4f88e, 0x739e0d11fb3, 0x44971f51cc0, 0x6490576e60e, 0x6c6674c5355, 0x6978126a4e1, 0x3d04eae5a5, 0x312eed633f2, 0x1de4b98d6b9, 0x118a106fb0a, 0x26dae025f4, 0x5c179312ebb, 0x75870ef1921, 0x60e9fed95c0, 0x209ab92427a, 0x1c5014a1937}); +constexpr StatTable43 QRT_TABLE_43({0x2bccc2d6f6c, 0x4bccc2d6f54, 0x4bccc2d6f56, 0x7cc7bc61df0, 0x4bccc2d6f52, 0x7d13b404b10, 0x7cc7bc61df8, 0x37456e9ac5a, 0x4bccc2d6f42, 0x4e042c6a6, 0x7d13b404b30, 0x4a56de9ef4c, 0x7cc7bc61db8, 0x14bc18d8e, 0x37456e9acda, 0x7c89f84fb1e, 0x4bccc2d6e42, 0x7ffae40d210, 0x4e042c4a6, 0x366f45dd06, 0x7d13b404f30, 0x496fcaf8cca, 0x4a56de9e74c, 0x370b62b6af4, 0x7cc7bc60db8, 0x1498185a8, 0x14bc1ad8e, 0x7e602c46a98, 0x37456e9ecda, 0x36ccc2c6e74, 0x7c89f847b1e, 0x7e27d06d516, 0x4bccc2c6e42, 0x7f93302c396, 0x7ffae42d210, 0x3dd3440706, 0x4e046c4a6, 0x78bbc09da36, 0x366f4ddd06, 0, 0x7d13b504f30, 0x8bbc09da00, 0x496fc8f8cca}); +typedef Field<uint64_t, 43, 89, StatTable43, &SQR_TABLE_43, &SQR2_TABLE_43, &SQR4_TABLE_43, &SQR8_TABLE_43, &SQR16_TABLE_43, &QRT_TABLE_43, IdTrans, &ID_TRANS, &ID_TRANS> Field43; +#endif + +#ifdef ENABLE_FIELD_INT_44 +// 44 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5> StatTable44; +constexpr StatTable44 SQR_TABLE_44({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x21, 0x84, 0x210, 0x840, 0x2100, 0x8400, 0x21000, 0x84000, 0x210000, 0x840000, 0x2100000, 0x8400000, 0x21000000, 0x84000000, 0x210000000, 0x840000000, 0x2100000000, 0x8400000000, 0x21000000000, 0x84000000000, 0x10000000042, 0x40000000108}); +constexpr StatTable44 SQR2_TABLE_44({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x21, 0x210, 0x2100, 0x21000, 0x210000, 0x2100000, 0x21000000, 0x210000000, 0x2100000000, 0x21000000000, 0x10000000042, 0x401, 0x4010, 0x40100, 0x401000, 0x4010000, 0x40100000, 0x401000000, 0x4010000000, 0x40100000000, 0x1000000084, 0x10000000840, 0x8421, 0x84210, 0x842100, 0x8421000, 0x84210000, 0x842100000, 0x8421000000, 0x84210000000, 0x42100000108, 0x21000001004, 0x10000010002}); +constexpr StatTable44 SQR4_TABLE_44({0x1, 0x10000, 0x100000000, 0x210, 0x2100000, 0x21000000000, 0x40100, 0x401000000, 0x10000000840, 0x8421000, 0x84210000000, 0x100001, 0x1000010000, 0x100002100, 0x21000210, 0x10002100042, 0x21000401000, 0x4010040100, 0x401008421, 0x10084210840, 0x42108421108, 0x84211000010, 0x10000000001, 0x31000, 0x310000000, 0x611, 0x6110000, 0x61100000000, 0xc4310, 0xc43100000, 0x31000001844, 0x18421100, 0x84211000021, 0x10000310001, 0x3100031000, 0x310006110, 0x61100611, 0x110061100c6, 0x61100c43100, 0xc4310c4310, 0x10c43118423, 0x31184210844, 0x42108421218, 0x84212100010}); +constexpr StatTable44 SQR8_TABLE_44({0x1, 0x21000401000, 0x84211000021, 0xa521000, 0x42108421719, 0x311e5310e55, 0x10401008c61, 0x3210031000, 0x10c43058522, 0x74110050101, 0xf5312e97201, 0x42108421109, 0x21061501611, 0x94311002961, 0xf59421000, 0x52d4b56923a, 0x201e5300e54, 0x71501c8fe71, 0xb6002131043, 0xb5e4c168432, 0xb320f5619ae, 0xf5000f97201, 0x10c43118422, 0x24010451100, 0x942113d4330, 0x8421a521001, 0x6310e130719, 0xf72fc731c6c, 0x4ae739631, 0x70008417e4d, 0x20ca6358b77, 0x64110094a51, 0xd4002e97200, 0xf7f5a13853a, 0xb12664417f6, 0x843322d6860, 0xf7c4100194d, 0x7382d17842b, 0xf67fd71a10c, 0x6efcca4b731, 0xf4e5901ea2d, 0xcb8278a46, 0xa32050401ee, 0xb7218bb6518}); +constexpr StatTable44 SQR16_TABLE_44({0x1, 0xc6cdb660138, 0x13de5a69a7b, 0x80bcafe7981, 0x60eb6f976d1, 0x677fbef6cce, 0x1549bb4cdec, 0x3b1ddf6859, 0xc01b8da28a6, 0xf3e11efbf8c, 0xd3e6faf8ee3, 0xa3dbc5712c8, 0x72361d7ca84, 0xe59e509337d, 0x15fca12a6f4, 0x33ce445498c, 0x44406de91fb, 0x9784b690571, 0xb0fb81753af, 0xb53a7c2c977, 0x34fbd3dba9b, 0xc758c22e647, 0xd5ff69aa469, 0x41e6d42b47d, 0xa4d1a3d02e7, 0x365db54ae9f, 0xd2293b8770b, 0xf1bf95c7746, 0x337fbe1d950, 0x726879e26a7, 0xa4be5ec2171, 0x7080da9df82, 0x7560017ce2, 0xd03997e34ae, 0x27ad4309a78, 0xb7b0ead892b, 0xf45bedb915d, 0xc4f0e25a52c, 0xe774a9d7fe8, 0xece6c1d7a26, 0xf20ea9ab655, 0x159bb624dc2, 0x12f2780b45f, 0x840cc52f19d}); +constexpr StatTable44 QRT_TABLE_44({0xf05334f4f6e, 0x4002016, 0x4002014, 0xf04350e6246, 0x4002010, 0x4935b379a26, 0xf04350e624e, 0xf84250c228e, 0x4002000, 0xf04300e521e, 0x4935b379a06, 0xb966838dd48, 0xf04350e620e, 0xf7b8b80feda, 0xf84250c220e, 0xf972e097d5e, 0x4002100, 0x8000020000, 0xf04300e501e, 0x430025000, 0x4935b379e06, 0xf976a09dc5e, 0xb966838d548, 0xf84218c029a, 0xf04350e720e, 0x4925f36bf06, 0xf7b8b80deda, 0xb047d3ee758, 0xf84250c620e, 0xf80350e720e, 0xf972e09fd5e, 0x8091825284, 0x4012100, 0x9015063210, 0x8000000000, 0xff31a028c5e, 0xf04300a501e, 0x44340b7100, 0x4300a5000, 0, 0x4935b279e06, 0xa976b2dce18, 0xf976a29dc5e, 0x8935b279e18}); +typedef Field<uint64_t, 44, 33, StatTable44, &SQR_TABLE_44, &SQR2_TABLE_44, &SQR4_TABLE_44, &SQR8_TABLE_44, &SQR16_TABLE_44, &QRT_TABLE_44, IdTrans, &ID_TRANS, &ID_TRANS> Field44; +typedef FieldTri<uint64_t, 44, 5, RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5>, &SQR_TABLE_44, &SQR2_TABLE_44, &SQR4_TABLE_44, &SQR8_TABLE_44, &SQR16_TABLE_44, &QRT_TABLE_44, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri44; +#endif + +#ifdef ENABLE_FIELD_INT_45 +// 45 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5> StatTable45; +constexpr StatTable45 SQR_TABLE_45({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x36, 0xd8, 0x360, 0xd80, 0x3600, 0xd800, 0x36000, 0xd8000, 0x360000, 0xd80000, 0x3600000, 0xd800000, 0x36000000, 0xd8000000, 0x360000000, 0xd80000000, 0x3600000000, 0xd800000000, 0x36000000000, 0xd8000000000, 0x16000000001b, 0x18000000005a}); +constexpr StatTable45 SQR2_TABLE_45({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0xd8, 0xd80, 0xd800, 0xd8000, 0xd80000, 0xd800000, 0xd8000000, 0xd80000000, 0xd800000000, 0xd8000000000, 0x18000000005a, 0x514, 0x5140, 0x51400, 0x514000, 0x5140000, 0x51400000, 0x514000000, 0x5140000000, 0x51400000000, 0x114000000036, 0x1400000003b8, 0x3b6e, 0x3b6e0, 0x3b6e00, 0x3b6e000, 0x3b6e0000, 0x3b6e00000, 0x3b6e000000, 0x3b6e0000000, 0x1b6e0000001b, 0x16e00000011f, 0xe0000001105}); +constexpr StatTable45 SQR4_TABLE_45({0x1, 0x10000, 0x100000000, 0xd8, 0xd80000, 0xd800000000, 0x5140, 0x51400000, 0x114000000036, 0x3b6e00, 0x3b6e000000, 0xe0000001105, 0x11011000, 0x110110000000, 0x1000000d58d8, 0xd58d58000, 0x18d58000054e, 0x5451454, 0x54514540000, 0x145400038db8, 0x38db6d8e0, 0xdb6d8e00104, 0x18e00101000a, 0x10100010100, 0x10100d8d8, 0x100d8d800d8, 0x18d800d8d85a, 0xd8d8511140, 0x18511140511a, 0x114051117b58, 0x11117b556e36, 0x1b556e3b5575, 0xe3b557f1015, 0x157f1011011e, 0x101101101c48, 0x1101c458d58, 0x1c458d58d580, 0xd58d58815d4, 0x158815d1451a, 0x15d1451452c0, 0x51452ce6f6e, 0x12ce6f6db6d6, 0xf6db6da6e3d, 0x16da6e39e00f, 0xe39e00000dd}); +constexpr StatTable45 SQR8_TABLE_45({0x1, 0x18d58000054e, 0xe3b557f1015, 0x51985198, 0xdb68cb55452, 0x1d0cc1d84f02, 0x140110c19ae, 0x11a16a14e7fe, 0x1e7ca7c62aa9, 0xe0eae26629b, 0x12182bee80ab, 0x1a38a68f28d3, 0x8581419c8c, 0x1d47f6f12ebc, 0x19fd34c3806e, 0x12ddba59f3cd, 0x10fa07f12a0e, 0x1d93eb544486, 0x1cf42cd119be, 0x1ff32d4b62c3, 0xf34ae031191, 0xada837715bf, 0xd368a753f92, 0x2ba87b17a03, 0x10374c3e4088, 0x1a6f539c11bd, 0x16548a5473c7, 0x1eb70011a8c9, 0x1ee5435ba1a3, 0x1173c0537680, 0xa1a3668dd6b, 0x119faad25e8, 0xd3909240e00, 0x1b560d018881, 0x127ecb9095ed, 0x306b507e701, 0x12b934c21ea3, 0x1a9d258c5b8b, 0x10452fbf0b1c, 0xae92fee120d, 0x183eb4b419fa, 0xc24d2391313, 0x4e6c4746f6, 0x2815fe7c395, 0xe4ab383747f}); +constexpr StatTable45 SQR16_TABLE_45({0x1, 0x14af92df932, 0x484e0190bdc, 0xda69889e16e, 0xcf70dfdb150, 0x18c6743571a8, 0x1b2c3ad7fa79, 0x5f0cbe204f6, 0xee973392a75, 0x3e86ef79673, 0xb2a9bef7181, 0x19b5347ff116, 0x1cae0ec79856, 0x69093f18f81, 0x1964382be09a, 0x92c894b073e, 0x1d99d2922eb2, 0x647905ad0eb, 0x1695971acdd3, 0x8f3292bc8c4, 0x1ee4057ad94, 0x17f02dc60e01, 0x1bb8e05ab4d5, 0x14de5d2a05d6, 0x13a019a02983, 0xcd7097c3616, 0x1bd798639b8f, 0x1cf0ca5ac7b2, 0xa93b983cf05, 0x159a955a2aa8, 0x69e5ba33397, 0x3a6b3392237, 0x26aeab71e13, 0x26fe04d38b9, 0x1fa9df0e8c45, 0x104e85c234b0, 0x1792853f8767, 0x81573b15f20, 0x127d6bfb06d3, 0x8110e6957e8, 0x11f59cbcc110, 0xad68264cad8, 0x61438575b35, 0x56e4446dc, 0x1cc9cb28b150}); +constexpr StatTable45 QRT_TABLE_45({0xede34e3e0fc, 0x1554148191aa, 0x1554148191a8, 0x1767be1dc4a6, 0x1554148191ac, 0x26bd4931492, 0x1767be1dc4ae, 0x233ab9c454a, 0x1554148191bc, 0x16939e8bb3dc, 0x26bd49314b2, 0x3c6ca8bac52, 0x1767be1dc4ee, 0x16caa5054c16, 0x233ab9c45ca, 0x14a1649628bc, 0x1554148190bc, 0x3c382881252, 0x16939e8bb1dc, 0x3c7ca0aa160, 0x26bd49310b2, 0x27f40158000, 0x3c6ca8ba452, 0x173fc092853c, 0x1767be1dd4ee, 0x16cbe284f25c, 0x16caa5056c16, 0x155559002f96, 0x233ab9c05ca, 0x26eb8908b32, 0x14a16496a8bc, 0x15440885333c, 0x1554148090bc, 0x17d60702e0, 0x3c3828a1252, 0x54548d10b2, 0x16939e8fb1dc, 0x3ac1e81b1d2, 0x3c7ca02a160, 0x166bd48310bc, 0x26bd48310b2, 0, 0x27f40358000, 0x10000000000e, 0x3c6cacba452}); +typedef Field<uint64_t, 45, 27, StatTable45, &SQR_TABLE_45, &SQR2_TABLE_45, &SQR4_TABLE_45, &SQR8_TABLE_45, &SQR16_TABLE_45, &QRT_TABLE_45, IdTrans, &ID_TRANS, &ID_TRANS> Field45; +#endif + +#ifdef ENABLE_FIELD_INT_46 +// 46 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5> StatTableTRI46; +constexpr StatTableTRI46 SQR_TABLE_TRI46({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000, 0xc0000000, 0x300000000, 0xc00000000, 0x3000000000, 0xc000000000, 0x30000000000, 0xc0000000000, 0x300000000000}); +constexpr StatTableTRI46 SQR2_TABLE_TRI46({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0xc, 0xc0, 0xc00, 0xc000, 0xc0000, 0xc00000, 0xc000000, 0xc0000000, 0xc00000000, 0xc000000000, 0xc0000000000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x500000, 0x5000000, 0x50000000, 0x500000000, 0x5000000000, 0x50000000000, 0x100000000003, 0x3c, 0x3c0, 0x3c00, 0x3c000, 0x3c0000, 0x3c00000, 0x3c000000, 0x3c0000000, 0x3c00000000, 0x3c000000000, 0x3c0000000000}); +constexpr StatTableTRI46 SQR4_TABLE_TRI46({0x1, 0x10000, 0x100000000, 0xc, 0xc0000, 0xc00000000, 0x50, 0x500000, 0x5000000000, 0x3c0, 0x3c00000, 0x3c000000000, 0x1100, 0x11000000, 0x110000000000, 0xcc00, 0xcc000000, 0xc0000000005, 0x55000, 0x550000000, 0x10000000003f, 0x3fc000, 0x3fc0000000, 0x101, 0x1010000, 0x10100000000, 0xc0c, 0xc0c0000, 0xc0c00000000, 0x5050, 0x50500000, 0x105000000003, 0x3c3c0, 0x3c3c00000, 0x3c000000011, 0x111100, 0x1111000000, 0x1100000000cc, 0xcccc00, 0xcccc000000, 0xc0000000555, 0x5555000, 0x55550000000, 0x100000003fff, 0x3fffc000, 0x3fffc0000000}); +constexpr StatTableTRI46 SQR8_TABLE_TRI46({0x1, 0xcc000000, 0x3c3c0, 0x10000000c, 0x550055000, 0x3c000111111, 0xc0050, 0x103fc000003f, 0x1111cccc00, 0x5000500390, 0x13ec1010101, 0xcccc9999955, 0x5000001100, 0xc0c01010000, 0xc003fffc555, 0x12c003c0cc00, 0x10505c5c0c0f, 0x155450003ffe, 0x1100cc054100, 0xfcc5053c3d1, 0xd3ff2c00d, 0xd059c3a5c3a, 0x2828282d21e, 0x5000000001, 0xcd010000, 0xc000003c695, 0x3c103c0000c, 0x55c095c0c, 0x169550112eee, 0x1100000c1150, 0x1c339050003f, 0x112e320c01, 0x1d50cc50cf95, 0x116d5292c2c2, 0xcccc9959954, 0x1050cc00113f, 0xc1d1002c3c0, 0xc013fafc509, 0x12fa93c59d01, 0x135c9081d11e, 0x150453cc3fae, 0x13f0d044d33, 0x688119f0a84, 0x39d2d62d29d, 0x3751370167, 0x24e4e4e4b4b4}); +constexpr StatTableTRI46 SQR16_TABLE_TRI46({0x1, 0x1fe6e1ab503e, 0xbbae1f3f55b, 0x1d51cc530c59, 0x163a6a22e14a, 0x5847feb7f81, 0x1ec9cc5fd281, 0xf6cc7b80c70, 0x8f46b31e374, 0xc13bf2ed37d, 0x148a1595bffe, 0x581ad245849, 0x1ea6920b83c1, 0x9d9a8355c7d, 0x6bcf393d5ff, 0x1d4e245085c0, 0x602a8c5e62c, 0x1922dd69197f, 0x7945d3a2aad, 0xf82a823f768, 0xdd24665599b, 0x13b43f6a29d, 0x4df114f238d, 0x1ee783c75ec0, 0xfb670f65c31, 0xf855dc973d2, 0x61ede5f2651, 0x6c1a1266403, 0x1f66ed2a96a, 0xbbbdf683148, 0x1ecc83e160c0, 0x1a2778c4bc0c, 0x10e154273753, 0x1704f8873c23, 0x1b4d3172da99, 0x2b3be805044, 0x5bb08848b9d, 0x1967d2b99be5, 0x7fa55262740, 0xe761a27cc28, 0x17dedd7181b5, 0x155b0344714a, 0x15187b38816e, 0xc5a679b5300, 0x1096cbf94c5d, 0x3f6b3cc122da}); +constexpr StatTableTRI46 QRT_TABLE_TRI46({0x211c4fd486ba, 0x100104a, 0x1001048, 0x104d0492d4, 0x100104c, 0x20005040c820, 0x104d0492dc, 0x40008080, 0x100105c, 0x24835068ce00, 0x20005040c800, 0x200000400800, 0x104d04929c, 0x100904325c, 0x40008000, 0x25da9e77daf0, 0x100115c, 0x1184e1696f0, 0x24835068cc00, 0x24825169dd5c, 0x20005040cc00, 0x3ea3241c60c0, 0x200000400000, 0x211c4e5496f0, 0x104d04829c, 0x20005340d86c, 0x100904125c, 0x24835968de5c, 0x4000c000, 0x6400a0c0, 0x25da9e775af0, 0x118cf1687ac, 0x101115c, 0x1ea1745cacc0, 0x1184e1496f0, 0x20181e445af0, 0x2483506ccc00, 0x20240060c0, 0x24825161dd5c, 0x1e21755dbd9c, 0x20005050cc00, 0x26a3746cacc0, 0x3ea3243c60c0, 0xea3243c60c0, 0x200000000000, 0}); +typedef FieldTri<uint64_t, 46, 1, StatTableTRI46, &SQR_TABLE_TRI46, &SQR2_TABLE_TRI46, &SQR4_TABLE_TRI46, &SQR8_TABLE_TRI46, &SQR16_TABLE_TRI46, &QRT_TABLE_TRI46, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri46; +#endif + +#ifdef ENABLE_FIELD_INT_47 +// 47 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5> StatTable47; +constexpr StatTable47 SQR_TABLE_47({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x42, 0x108, 0x420, 0x1080, 0x4200, 0x10800, 0x42000, 0x108000, 0x420000, 0x1080000, 0x4200000, 0x10800000, 0x42000000, 0x108000000, 0x420000000, 0x1080000000, 0x4200000000, 0x10800000000, 0x42000000000, 0x108000000000, 0x420000000000, 0x80000000042, 0x200000000108}); +constexpr StatTable47 SQR2_TABLE_47({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x42, 0x420, 0x4200, 0x42000, 0x420000, 0x4200000, 0x42000000, 0x420000000, 0x4200000000, 0x42000000000, 0x420000000000, 0x200000000108, 0x1004, 0x10040, 0x100400, 0x1004000, 0x10040000, 0x100400000, 0x1004000000, 0x10040000000, 0x100400000000, 0x4000000042, 0x40000000420, 0x400000004200, 0x42108, 0x421080, 0x4210800, 0x42108000, 0x421080000, 0x4210800000, 0x42108000000, 0x421080000000, 0x210800000108, 0x108000001004, 0x80000010002}); +constexpr StatTable47 SQR4_TABLE_47({0x1, 0x10000, 0x100000000, 0x42, 0x420000, 0x4200000000, 0x1004, 0x10040000, 0x100400000000, 0x42108, 0x421080000, 0x210800000108, 0x1000010, 0x10000100000, 0x1000004200, 0x42000420, 0x420004200000, 0x42000100400, 0x1004010040, 0x40100400420, 0x4004210842, 0x42108421080, 0x84210810002, 0x108100000004, 0x142, 0x1420000, 0x14200000000, 0x5204, 0x52040000, 0x520400000000, 0x142508, 0x1425080000, 0x250800000528, 0x5210810, 0x52108100000, 0x81000014202, 0x142001420, 0x420014200042, 0x142000520400, 0x5204052040, 0x40520401424, 0x20401425094a, 0x142509425080, 0x9425085210a, 0x508521084204, 0x21084210804a, 0x421080420010}); +constexpr StatTable47 SQR8_TABLE_47({0x1, 0x420004200000, 0x250800000528, 0x11004, 0xc6210910402, 0x142005730c10, 0x101000010, 0x1056050040, 0x55a429184204, 0x111404110002, 0x532408500562, 0x3d196251d62e, 0x420142, 0x44524611c66, 0x1142047728, 0x46205e2508, 0x67339c2519da, 0x5661384b5880, 0x434346200424, 0x392938cb55aa, 0x724b0c31058c, 0x7f4f1cf703fc, 0x4fe303d32d1e, 0x75de250d676c, 0x100400011004, 0xc6210910540, 0x102525331834, 0x1101046318, 0x1057532548, 0x37f4bd7f4b5e, 0x115021380840, 0x52674a501142, 0x297a4a1b86ae, 0x666b0c2010ca, 0x776839031b88, 0x4b5316a05622, 0x254e215e2030, 0x6733ce2009de, 0xa8609d21e86, 0x567347420874, 0x6e0d31db55ba, 0x4357182485c4, 0x2afb35ef02ba, 0x5af227961c30, 0x64faad1b0116, 0x2d5d2527ef40, 0x6e27e3bc1978}); +constexpr StatTable47 SQR16_TABLE_47({0x1, 0x421ac25c8774, 0x30581389b510, 0x423b9c671db0, 0xa4537914208, 0x38f9be0dbf38, 0x351c5a8b92a8, 0xc38b9920da2, 0x508d34674f2a, 0x1f8c359a6b76, 0x5ac4bf86daaa, 0x51d6a6616df2, 0xe2717a0378a, 0x13353e783e6e, 0x55a55ac09ec6, 0x3f17cde43402, 0x760584b64b6c, 0x6acbecc99a02, 0x16be80e45b76, 0x2d5069e0005a, 0x3388f5759aa6, 0x2f98f891f4e2, 0x7657f368d924, 0x48f81e34f5b0, 0x51a9087f072e, 0x1de01ba9001c, 0x560b4b374bfc, 0x13f576988ff0, 0x3673cd322294, 0x595959f7c5fe, 0xbfa426eb4a4, 0x2b68fd7c02c2, 0x2a3c1437913a, 0x6e4b179fcf9e, 0x69ddf09bbdee, 0x7b91973d5e52, 0x1329cefd9514, 0x6a5f380b7ab0, 0x48e6620529c4, 0x60589a4b95b6, 0x5e4bd1d1aa34, 0x4b1ec7645cc2, 0x5cfb8785aec6, 0x34e47cf10c3a, 0x7b6c363eee10, 0x1dc52d768b32, 0x3585af9113a0}); +constexpr StatTable47 QRT_TABLE_47({0, 0x1001040, 0x1001042, 0x1047043076, 0x1001046, 0x112471c241e, 0x104704307e, 0x4304e052168, 0x1001056, 0x10004000, 0x112471c243e, 0x172a09c949d6, 0x104704303e, 0x4002020, 0x4304e0521e8, 0x5400e220, 0x1001156, 0x172b08c85080, 0x10004200, 0x41200b0800, 0x112471c203e, 0x172f0cca50a0, 0x172a09c941d6, 0x7eb88a11c1d6, 0x104704203e, 0x1044042020, 0x4000020, 0x42001011156, 0x4304e0561e8, 0x172a28c95880, 0x54006220, 0x112931cc21e, 0x1011156, 0x53670f283e, 0x172b08ca5080, 0x7a80c414a03e, 0x10044200, 0x40000000000, 0x4120030800, 0x1928318801e, 0x112470c203e, 0x799283188000, 0x172f0cea50a0, 0x1eb88a91c1c8, 0x172a098941d6, 0x3ea8cc95e1f6, 0x7eb88a91c1d6}); +typedef Field<uint64_t, 47, 33, StatTable47, &SQR_TABLE_47, &SQR2_TABLE_47, &SQR4_TABLE_47, &SQR8_TABLE_47, &SQR16_TABLE_47, &QRT_TABLE_47, IdTrans, &ID_TRANS, &ID_TRANS> Field47; +typedef FieldTri<uint64_t, 47, 5, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5>, &SQR_TABLE_47, &SQR2_TABLE_47, &SQR4_TABLE_47, &SQR8_TABLE_47, &SQR16_TABLE_47, &QRT_TABLE_47, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri47; +#endif + +#ifdef ENABLE_FIELD_INT_48 +// 48 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6> StatTable48; +constexpr StatTable48 SQR_TABLE_48({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x2d, 0xb4, 0x2d0, 0xb40, 0x2d00, 0xb400, 0x2d000, 0xb4000, 0x2d0000, 0xb40000, 0x2d00000, 0xb400000, 0x2d000000, 0xb4000000, 0x2d0000000, 0xb40000000, 0x2d00000000, 0xb400000000, 0x2d000000000, 0xb4000000000, 0x2d0000000000, 0xb40000000000, 0xd0000000005a, 0x40000000011f}); +constexpr StatTable48 SQR2_TABLE_48({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x2d, 0x2d0, 0x2d00, 0x2d000, 0x2d0000, 0x2d00000, 0x2d000000, 0x2d0000000, 0x2d00000000, 0x2d000000000, 0x2d0000000000, 0xd0000000005a, 0x451, 0x4510, 0x45100, 0x451000, 0x4510000, 0x45100000, 0x451000000, 0x4510000000, 0x45100000000, 0x451000000000, 0x5100000000b4, 0x100000000bd9, 0xbdbd, 0xbdbd0, 0xbdbd00, 0xbdbd000, 0xbdbd0000, 0xbdbd00000, 0xbdbd000000, 0xbdbd0000000, 0xbdbd00000000, 0xdbd00000011f, 0xbd0000001001, 0xd0000001010f}); +constexpr StatTable48 SQR4_TABLE_48({0x1, 0x10000, 0x100000000, 0x2d, 0x2d0000, 0x2d00000000, 0x451, 0x4510000, 0x45100000000, 0xbdbd, 0xbdbd0000, 0xbdbd00000000, 0x101101, 0x1011010000, 0x1101000002d0, 0x2d2fd2d, 0x2d2fd2d0000, 0xfd2d0000454a, 0x45514551, 0x455145510000, 0x4551000bd0bd, 0xbd0b6d0bd, 0xd0b6d0bd011f, 0xd0bd0100011e, 0x10001010001, 0x10100012d00, 0x12d002d2d, 0x2d002d2d002d, 0x2d2d00295100, 0x2951045551, 0x5104555104e5, 0x555104ecbdb4, 0x4ecbdbd00bd, 0xbdbd00bdadbc, 0xbdadac1101, 0xadac11011001, 0x11011013c3fc, 0x1013c3fefd2d, 0xc3fefd2fd2a7, 0xfd2fd2baac36, 0xd2baac2d450b, 0xac2d45145ac2, 0x45145ad0f851, 0x5ad0f85adb64, 0xf85adb6cbd10, 0xdb6cbd0bd0a2, 0xbd0bd0bc003c, 0xd0bc002c001f}); +constexpr StatTable48 SQR8_TABLE_48({0x1, 0x2d2fd2d0000, 0x4ecbdbd00bd, 0x2c0000002c, 0x47882d9b95ec, 0xb9eec2eefc90, 0xbd900450, 0x9734009bfc8f, 0x93070a51e9b7, 0xb9d0aa02ec00, 0x8630787dd0ab, 0x6b3468ab98c9, 0x4100001bc1bd, 0x3ffeec1692fd, 0x2e0cce80414a, 0x11a77fc95626, 0x7c9856084ffe, 0x2f41c702f193, 0xd260e95bfeeb, 0x3b86220b54eb, 0x5735c802071a, 0x44626fc7ba84, 0x2b9cda923f5b, 0xc57d0a962e45, 0xd1685001450b, 0x6d78400145b6, 0x9114412978c1, 0x7d9ece1f5b0c, 0xfd2419988960, 0xd12ac3eaeaa5, 0x7d67e75f441d, 0xf86c2ba5457c, 0x40db7617aa6a, 0x80ee292186, 0xbd0f8525d34c, 0xa87ce27ca699, 0xacf3315d7a6d, 0x1a289bca977, 0x92975374e0f1, 0x3fcf113826ab, 0xff4d9be19a5e, 0x1412e5091900, 0x82c721f22d43, 0x1d773380ff32, 0xfed661cca7b1, 0x308072e06846, 0xd3eb44e91aa0, 0x819a669cbb14}); +constexpr StatTable48 SQR16_TABLE_48({0x1, 0x50c24311dfa9, 0xfc08c1e39482, 0xa9ff91b620a3, 0x54954a59d16c, 0xec45c0a9fb02, 0xba7004022837, 0xc1ea19828166, 0xee9a3efecffe, 0x57ed421a20bb, 0x69d387b19141, 0x9105d02d728f, 0xd2f24d4006da, 0x39195005f508, 0xd0206ff5333c, 0x8592e734a441, 0x787b36a1c435, 0x1151e6f03f85, 0xfe0429bf95ab, 0xab2b20b47651, 0x2a65fc935212, 0x7f73ae670e2e, 0x697c17f0fc4a, 0x55dc5681f013, 0xadbd09a289bc, 0x418414f64940, 0x927c737efd40, 0x38535d08fc98, 0xe811b107691c, 0x856c3bbf4cf6, 0x47e629ad3757, 0xf9c82b4b2c09, 0x64312c99e2f, 0xd4936c978dfd, 0x782ff8716675, 0xaf853e867dd7, 0x457143c1fa6d, 0x84c4dda48e91, 0xbac9aacda41e, 0x6a6e0ffb2dc1, 0xfef377f00194, 0x3a129790acc1, 0x541e49c6f92a, 0x73e821aca96d, 0x3a6f15c03f57, 0x1bf377c66f3b, 0xbff5e192fe3b, 0x346360ee74a}); +constexpr StatTable48 QRT_TABLE_48({0xc00442c284f0, 0xc16b7fda410a, 0xc16b7fda4108, 0xada3b5c79fbe, 0xc16b7fda410c, 0x16f3c18d5b0, 0xada3b5c79fb6, 0x7090a381f64, 0xc16b7fda411c, 0xcafc15d179f8, 0x16f3c18d590, 0x6630880e534e, 0xada3b5c79ff6, 0xa13dd1f49826, 0x7090a381fe4, 0xb87560f6a74, 0xc16b7fda401c, 0xaaaaffff0012, 0xcafc15d17bf8, 0xaafd15f07bf6, 0x16f3c18d190, 0x60000020000e, 0x6630880e5b4e, 0xcb977fcb401c, 0xada3b5c78ff6, 0x6663420cad0, 0xa13dd1f4b826, 0xc0045fc2f41c, 0x7090a385fe4, 0x6762e24b834, 0xb87560fea74, 0xc6351fed241c, 0xc16b7fdb401c, 0x60065622ea7a, 0xaaaafffd0012, 0xdf9562bea74, 0xcafc15d57bf8, 0x6657ea057bea, 0xaafd15f87bf6, 0xa79329ddaa66, 0x16f3c08d190, 0xa39229f0aa66, 0x60000000000e, 0x175fb4468ad0, 0x6630884e5b4e, 0, 0xcb977f4b401c, 0x2630884e5b40}); +typedef Field<uint64_t, 48, 45, StatTable48, &SQR_TABLE_48, &SQR2_TABLE_48, &SQR4_TABLE_48, &SQR8_TABLE_48, &SQR16_TABLE_48, &QRT_TABLE_48, IdTrans, &ID_TRANS, &ID_TRANS> Field48; +#endif +} + +Sketch* ConstructClMul6Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_41 + case 41: return new SketchImpl<Field41>(implementation, 41); +#endif +#ifdef ENABLE_FIELD_INT_42 + case 42: return new SketchImpl<Field42>(implementation, 42); +#endif +#ifdef ENABLE_FIELD_INT_43 + case 43: return new SketchImpl<Field43>(implementation, 43); +#endif +#ifdef ENABLE_FIELD_INT_44 + case 44: return new SketchImpl<Field44>(implementation, 44); +#endif +#ifdef ENABLE_FIELD_INT_45 + case 45: return new SketchImpl<Field45>(implementation, 45); +#endif +#ifdef ENABLE_FIELD_INT_47 + case 47: return new SketchImpl<Field47>(implementation, 47); +#endif +#ifdef ENABLE_FIELD_INT_48 + case 48: return new SketchImpl<Field48>(implementation, 48); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri6Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_41 + case 41: return new SketchImpl<FieldTri41>(implementation, 41); +#endif +#ifdef ENABLE_FIELD_INT_42 + case 42: return new SketchImpl<FieldTri42>(implementation, 42); +#endif +#ifdef ENABLE_FIELD_INT_44 + case 44: return new SketchImpl<FieldTri44>(implementation, 44); +#endif +#ifdef ENABLE_FIELD_INT_46 + case 46: return new SketchImpl<FieldTri46>(implementation, 46); +#endif +#ifdef ENABLE_FIELD_INT_47 + case 47: return new SketchImpl<FieldTri47>(implementation, 47); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_7bytes.cpp b/src/minisketch/src/fields/clmul_7bytes.cpp new file mode 100644 index 0000000000..2050dc32dd --- /dev/null +++ b/src/minisketch/src/fields/clmul_7bytes.cpp @@ -0,0 +1,170 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_7) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_49 +// 49 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable49; +constexpr StatTable49 SQR_TABLE_49({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x402, 0x1008, 0x4020, 0x10080, 0x40200, 0x100800, 0x402000, 0x1008000, 0x4020000, 0x10080000, 0x40200000, 0x100800000, 0x402000000, 0x1008000000, 0x4020000000, 0x10080000000, 0x40200000000, 0x100800000000, 0x402000000000, 0x1008000000000, 0x20000000402, 0x80000001008, 0x200000004020, 0x800000010080}); +constexpr StatTable49 SQR2_TABLE_49({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x1008, 0x10080, 0x100800, 0x1008000, 0x10080000, 0x100800000, 0x1008000000, 0x10080000000, 0x100800000000, 0x1008000000000, 0x80000001008, 0x800000010080, 0x100004, 0x1000040, 0x10000400, 0x100004000, 0x1000040000, 0x10000400000, 0x100004000000, 0x1000040000000, 0x400001008, 0x4000010080, 0x40000100800, 0x400001008000, 0x10080402, 0x100804020, 0x1008040200, 0x10080402000, 0x100804020000, 0x1008040200000, 0x80402001008, 0x804020010080, 0x40200100004, 0x402001000040, 0x20010000002, 0x200100000020}); +constexpr StatTable49 SQR4_TABLE_49({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x1008000, 0x10080000000, 0x800000010080, 0x100004000, 0x1000040000000, 0x400001008000, 0x10080402000, 0x804020010080, 0x200100000020, 0x1000000001000, 0x11008000, 0x110080000000, 0x800000110880, 0x1108004000, 0x1080040001008, 0x400011008400, 0x110084402000, 0x844020110880, 0x201108040220, 0x1080402000008, 0x20001008002, 0x10080000100, 0x800001010080, 0x10100004000, 0x1000040010080, 0x400101808000, 0x1018080402000, 0x8040210100c0, 0x210100400020, 0x1004000011080, 0x11180c020, 0x11180c0200000, 0xc020011108c0, 0x11108004010, 0x1080040111088, 0x401111808400, 0x1118084403008, 0x8440311908c0, 0x311908440220, 0x108440211008c, 0x2110184c022, 0x10184c0201108, 0xc020100904c2, 0x100904024010, 0x1040240000004}); +constexpr StatTable49 SQR8_TABLE_49({0x1, 0x800000110880, 0x210100400020, 0x1040240000004, 0x130081008002, 0x191c0a54130c8, 0x900804130890, 0x210111408020, 0x582c0402004, 0xd320910984c0, 0xb1d1ad4522e8, 0x1d80945829818, 0x1218540601128, 0x1240340000024, 0x11300c1018082, 0x193d1a4c5f0ea, 0x148a0522810, 0xe0201051c8e0, 0x2dc7c25120a8, 0x1d2205148a440, 0x33c0adc0e24a, 0x17850e987bab8, 0xa8a40071c9e0, 0x10d47d391c1a8, 0x9740b01888c2, 0x91c0e54130c8, 0x920805138892, 0x130819500b028, 0x8583c0516884, 0x1fa25934984e8, 0x1f5c2fcc5a6ec, 0x15a094483189a, 0x1014cc242300, 0xac020000582c, 0x3b05524100aa, 0x1520255145c6e, 0x4279b95aec40, 0x1f1a0951178e8, 0xbcc646004828, 0x3b055219aca8, 0x57d2fc40666e, 0xba50b987beba, 0x8aa44d913bea, 0x944717d1b9a0, 0x776562491022, 0x1701305105c4c, 0x19039cd5be842, 0x1b281d8e58082, 0x134dac80532a4}); +constexpr StatTable49 SQR16_TABLE_49({0x1, 0x790181b552e0, 0xeb19044e00a, 0xc6bf7911f7ae, 0x447f77c1a0c4, 0x19d2a0d21c480, 0x13d4e22aadedc, 0x18fa344c8f0a6, 0x1481c1bbfde92, 0x41547e22f6e0, 0xf5ad96335088, 0xd7e4db3adaa0, 0x197fc8d7b53d0, 0x37781564b82a, 0xa52ef2139cbc, 0x153c6a0949498, 0x18d7401fc152e, 0xc4b5d8597752, 0xd15cd891aa2, 0x217903427da8, 0x13ec9e269a0e0, 0xc01720774514, 0x389aeb1d788a, 0x64a914a860a4, 0xa09aebec6188, 0x15c3239e150c8, 0x38f8fe110ce, 0xc1ea415c5006, 0x3209972f2ff0, 0x41bfc6b2ad88, 0x1ccc2fd5f73c8, 0x7bed1f863c00, 0x1a46d9b9844f4, 0x12e3ca6573ff6, 0x290c26cca98c, 0x514cb03b3b2e, 0x11168909cbc2c, 0x8e6dc910afda, 0x11311def1c440, 0x3e42d62664d8, 0x1c2bb2d75fe80, 0x2db5d58b45ca, 0x3d14059fd338, 0x109e8f457ebf8, 0x43b071b62a64, 0x185242247c010, 0x5e0c7721c092, 0x1c94950e46b82, 0x1761170f76a40}); +constexpr StatTable49 QRT_TABLE_49({0, 0x10004196, 0x10004194, 0x5099461f080, 0x10004190, 0x40840600c20, 0x5099461f088, 0x58a56349cfde, 0x10004180, 0x48641a0c03fe, 0x40840600c00, 0x10084002848, 0x5099461f0c8, 0x4002048, 0x58a56349cf5e, 0x5088460a048, 0x10004080, 0x4c2852624dde, 0x48641a0c01fe, 0x14893129c280, 0x40840600800, 0x1eb23c323ace8, 0x10084002048, 0x48740a09417e, 0x5099461e0c8, 0x40852604d96, 0x4000048, 0x5cad2b29c37e, 0x58a563498f5e, 0x20000200, 0x50884602048, 0x10000000000, 0x10014080, 0x4c2a56624d96, 0x4c2852604dde, 0x1ee2347438ca0, 0x48641a0801fe, 0x480000000048, 0x14893121c280, 0x14091121c080, 0x40840700800, 0x1a5099561e17e, 0x1eb23c303ace8, 0x8740a894136, 0x10084402048, 0x18101c501ace8, 0x48740a89417e, 0x15dace6286f96, 0x5099561e0c8}); +typedef Field<uint64_t, 49, 513, StatTable49, &SQR_TABLE_49, &SQR2_TABLE_49, &SQR4_TABLE_49, &SQR8_TABLE_49, &SQR16_TABLE_49, &QRT_TABLE_49, IdTrans, &ID_TRANS, &ID_TRANS> Field49; +typedef FieldTri<uint64_t, 49, 9, RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5, 5>, &SQR_TABLE_49, &SQR2_TABLE_49, &SQR4_TABLE_49, &SQR8_TABLE_49, &SQR16_TABLE_49, &QRT_TABLE_49, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri49; +#endif + +#ifdef ENABLE_FIELD_INT_50 +// 50 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable50; +constexpr StatTable50 SQR_TABLE_50({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x1d, 0x74, 0x1d0, 0x740, 0x1d00, 0x7400, 0x1d000, 0x74000, 0x1d0000, 0x740000, 0x1d00000, 0x7400000, 0x1d000000, 0x74000000, 0x1d0000000, 0x740000000, 0x1d00000000, 0x7400000000, 0x1d000000000, 0x74000000000, 0x1d0000000000, 0x740000000000, 0x1d00000000000, 0x340000000001d, 0x1000000000053}); +constexpr StatTable50 SQR2_TABLE_50({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x74, 0x740, 0x7400, 0x74000, 0x740000, 0x7400000, 0x74000000, 0x740000000, 0x7400000000, 0x74000000000, 0x740000000000, 0x340000000001d, 0x151, 0x1510, 0x15100, 0x151000, 0x1510000, 0x15100000, 0x151000000, 0x1510000000, 0x15100000000, 0x151000000000, 0x1510000000000, 0x1100000000069, 0x10000000006e4, 0x6e34, 0x6e340, 0x6e3400, 0x6e34000, 0x6e340000, 0x6e3400000, 0x6e34000000, 0x6e340000000, 0x6e3400000000, 0x2e3400000001d, 0x234000000011f, 0x3400000001118}); +constexpr StatTable50 SQR4_TABLE_50({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x74000, 0x740000000, 0x340000000001d, 0x151000, 0x1510000000, 0x1100000000069, 0x6e3400, 0x6e34000000, 0x234000000011f, 0x1110100, 0x11101000000, 0x1010000000734, 0x7334740, 0x73347400000, 0x347400000145c, 0x14540510, 0x145405100000, 0x510000068b9, 0x68b91a34, 0x68b91a340000, 0x11a3400010106, 0x101010001, 0x1010100010000, 0x1000100074740, 0x1000747474000, 0x347474007401d, 0x340074015050d, 0x340150505101d, 0x1050510151069, 0x11015106e5a5d, 0x1106e5a5a3469, 0x25a5a346e351f, 0x2346e3510111e, 0x235101110001f, 0x111000110634, 0x1106347334, 0x1063473340074, 0x733400735301, 0x7353004541, 0x353004541014c, 0x454101446dc0, 0x101446dc1cb90, 0x6dc1cb97468d, 0x1cb97468c1a30, 0x3468c1a350009, 0x1a3500010007}); +constexpr StatTable50 SQR8_TABLE_50({0x1, 0x7334740, 0x1050510151069, 0x3468c1a350009, 0x341624173531c, 0x245791a347b50, 0x23179d1a40682, 0x1671402235203, 0x321023818751e, 0x143ca5b716dd5, 0x171633ad257de, 0x33860681a5d1d, 0x5e572f82a317, 0x10e7512224646, 0x32d6b56300005, 0x350ab39687414, 0x25c47550c1a8a, 0x23a2e2d91533f, 0x2211af19c2381, 0x352073a863a68, 0x37f43380f0ac4, 0x233516127052a, 0x25ad4785169cf, 0x237b6a609b0b6, 0x132fd372b5dac, 0x1f311727562e, 0x345bd7e275754, 0x352fe5b3d7708, 0x259a328ca3376, 0x25101aab53ece, 0x32701d9da5ace, 0x17809a9c86099, 0x72b4752a7323, 0x202d22dc33a7c, 0x5a8c0dbc19a2, 0x14a86b37416ad, 0x5c574289fe12, 0x3627f3bf0f37b, 0x27349052a4f83, 0x2436d71033de5, 0x22fab345e0bce, 0x27ea796d5a27a, 0x1e4f33562d17, 0x31a1f9c3f2154, 0x1638db7753f96, 0x2256163f33b5f, 0x11a6ecf28882e, 0x1bd4cf35f47cc, 0x25e19aeb21e64, 0x371612d0b4dcd}); +constexpr StatTable50 SQR16_TABLE_50({0x1, 0x14db3a1b1531f, 0x270a39b5e8c48, 0x26536a58442bd, 0x7f158d4b869e, 0x12663760f7d, 0x29634a2c8876, 0x15271f7ec5d31, 0x17fbb0726d0f0, 0x7f0f7bf826bb, 0x115135d3c7c4c, 0x348ffaaa125e5, 0x1887695a20d9, 0x25e41181c0de, 0x2670d7f17fb35, 0x356079737f513, 0x22bebda8a1574, 0x315f9649d2b50, 0x13abe45aa6ac8, 0x723d536b5242, 0x24263520a22a9, 0x15860c0156a69, 0x271d0bbeed892, 0x146920f281d19, 0x117d5d46e7991, 0x278d8273551fc, 0x15d73a9745614, 0x7e5e966bbfe0, 0x687b14e62abb, 0x178acea79fa5c, 0x3363c557e9662, 0x3153c79bf06ef, 0x15c8ff9daf7ce, 0x243b030f4617a, 0x20663fbd2383a, 0x25c5dbd448872, 0x21fc8dfbd2429, 0x229f9fb8f01b0, 0x17a180ae72359, 0x1c8e2f554ad9, 0x174596d1e774f, 0x3264c5da47f53, 0x333817d45b05c, 0x321907ec10dfd, 0x3a12b2018ada, 0x23ab0599cd08, 0x23028d60c00e5, 0x8ca05e2a1eab, 0x3537bf673a228, 0x32f8cf8611080}); +constexpr StatTable50 QRT_TABLE_50({0xfbdfa3ae9d4c, 0x38143245a4878, 0x38143245a487a, 0x38527487e7492, 0x38143245a487e, 0x3124c61f56d2a, 0x38527487e749a, 0xfa8c91b087c0, 0x38143245a486e, 0x3eca48c6196be, 0x3124c61f56d0a, 0x380000040080a, 0x38527487e74da, 0x976b2d8b39b4, 0xfa8c91b08740, 0xfa8cd5b02724, 0x38143245a496e, 0x316291dd013fe, 0x3eca48c6194be, 0x10344122064, 0x3124c61f5690a, 0x68c5f006ee40, 0x380000040000a, 0x852749fe64d0, 0x38527487e64da, 0x37ef8e9d0e9da, 0x976b2d8b19b4, 0x37fabd1cef34a, 0xfa8c91b0c740, 0x96282d9159b4, 0xfa8cd5b0a724, 0x464a8249dd0, 0x38143245b496e, 0x37eaa8ddc94be, 0x316291dd213fe, 0x392446035690a, 0x3eca48c6594be, 0x974b258b4964, 0x103441a2064, 0x385a7c87fb4da, 0x3124c61e5690a, 0xeb8ad5d9a724, 0x68c5f026ee40, 0x3724c61e5690a, 0x380000000000a, 0x3a8c5f026ee4a, 0x8527497e64d0, 0, 0x38527497e64da, 0x2fbdfa2ae8d0a}); +typedef Field<uint64_t, 50, 29, StatTable50, &SQR_TABLE_50, &SQR2_TABLE_50, &SQR4_TABLE_50, &SQR8_TABLE_50, &SQR16_TABLE_50, &QRT_TABLE_50, IdTrans, &ID_TRANS, &ID_TRANS> Field50; +#endif + +#ifdef ENABLE_FIELD_INT_51 +// 51 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable51; +constexpr StatTable51 SQR_TABLE_51({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x96, 0x258, 0x960, 0x2580, 0x9600, 0x25800, 0x96000, 0x258000, 0x960000, 0x2580000, 0x9600000, 0x25800000, 0x96000000, 0x258000000, 0x960000000, 0x2580000000, 0x9600000000, 0x25800000000, 0x96000000000, 0x258000000000, 0x960000000000, 0x2580000000000, 0x160000000004b, 0x580000000012c, 0x6000000000426}); +constexpr StatTable51 SQR2_TABLE_51({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x96, 0x960, 0x9600, 0x96000, 0x960000, 0x9600000, 0x96000000, 0x960000000, 0x9600000000, 0x96000000000, 0x960000000000, 0x160000000004b, 0x6000000000426, 0x4114, 0x41140, 0x411400, 0x4114000, 0x41140000, 0x411400000, 0x4114000000, 0x41140000000, 0x411400000000, 0x4114000000000, 0x1140000000258, 0x1400000002516, 0x40000000251f6, 0x251d38, 0x251d380, 0x251d3800, 0x251d38000, 0x251d380000, 0x251d3800000, 0x251d38000000, 0x251d380000000, 0x51d380000012c, 0x1d3800000100e, 0x538000001003d, 0x380000010011e}); +constexpr StatTable51 SQR4_TABLE_51({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x96000, 0x960000000, 0x160000000004b, 0x411400, 0x4114000000, 0x1140000000258, 0x251d380, 0x251d3800000, 0x1d3800000100e, 0x10010110, 0x100101100000, 0x1011000009600, 0x960969f6, 0x960969f60000, 0x169f60004110b, 0x6000411015132, 0x4110151054000, 0x1510540251f60, 0x540251f6ba760, 0x51f6ba74eb92c, 0x3a74eb900001f, 0x6b90000010033, 0x100010860, 0x1000108600000, 0x1086000096000, 0x960092874, 0x160092874004b, 0x128740041144b, 0x40041144304e2, 0x1144304c78258, 0x304c78251d1d8, 0x78251d1c38368, 0x1d1c38352800e, 0x3835280011188, 0x280011197096e, 0x1119709787000, 0x709787009fb46, 0x7009fb7861c9, 0x1fb7861cae24b, 0x61cae2456109, 0x2e245610a7bc3, 0x5610a7bd61498, 0x27bd614b79d2b, 0x614b79d3a74de, 0x79d3a74e9f68a, 0x274e9f6b06011, 0x1f6b06000000f}); +constexpr StatTable51 SQR8_TABLE_51({0x1, 0x960969f6, 0x40041144304e2, 0x79d3a74e9f68a, 0x61005961939d4, 0x2e2108dfdafb5, 0xe7e61b897f73, 0x493a58b330d18, 0x7882105dc65ec, 0x5f00774200d11, 0x63ef4cd371ef3, 0x660b24b8d214b, 0x7ab791e669e3d, 0x10821820969f6, 0x1544b9d4c3f3e, 0x831185e3da14, 0x1eb0831983187, 0x1d8699ae87312, 0x586e000eb5f1a, 0x3ea794ef9c821, 0x2ab1c63209cc1, 0x7f434bcc29855, 0x673d370c40117, 0x6a668249ddd8b, 0x48be019e56bbe, 0x57d1a751be823, 0x5621931ca6d5f, 0x68c5a37844a68, 0xefa69123b6b1, 0x5804da97df62a, 0x30c29b82f3986, 0x5b808f6ddc779, 0x2c8b4e7596cbe, 0x2c5a432ec7a14, 0x7f178a4d63277, 0x77112a07b99b7, 0x56cf47ad50529, 0x73a2180190a41, 0x25cbc68f1f1a8, 0x1c27dc22e6950, 0x2fbf4aafee2ad, 0x554b728a595ca, 0x52726d34627e, 0x6dcc716c9e860, 0x36ade274d5eff, 0x1fa23a55b359a, 0x1bc6260896059, 0x53a74c5798bc1, 0x50e671fc54a4a, 0x251a72b3c4c3c, 0x6d9623f5d3a1e}); +constexpr StatTable51 SQR16_TABLE_51({0x1, 0x27b32044e9663, 0x528c08dd195bf, 0x5d461228d5764, 0x616db8f131bf6, 0x9d910988ca4, 0x1e7a7a29c55fd, 0x512a2e6297818, 0x688d44453ead0, 0x70e0b6e1b3be2, 0x4313e5612d70, 0x132241d43589d, 0x7ca688c29c89a, 0x1d6b8caeb8958, 0x36d06e8e76e3b, 0x18ebafc89388e, 0x1cb5f93b2c29c, 0x5137bd7b7b6ec, 0x6e3ae8731000b, 0x359203e5e12fe, 0x1822ded1f1e16, 0x3ee9c50cbcb89, 0x5cc0b4564ab4, 0x695b235bd9236, 0x283c619a1ecb, 0x6f37f1f6ef70d, 0x7f394b6fbdd53, 0x3f482b36793f, 0x4055274e56dfa, 0x1a85d9d434f33, 0x37aa8f3df2031, 0x5f4e77b2bb063, 0x6e9702d84f07b, 0x25f16f8ffd4c2, 0x22c591d8277cb, 0x59435d9bae242, 0x46eaf9f69ddd9, 0x3098c1e26bd6e, 0x6c6544847a1d, 0x254946c0c33ce, 0x23970a6118811, 0x67f6c55082b49, 0x6592c83ebde46, 0x716418f089ed8, 0x8cb8de463166, 0x37cb1794fac42, 0x94ac55c1ac68, 0x3ab0d33bb4fdf, 0x1669c2f7ae3c5, 0x4d4e4f61d1f04, 0x476980d17eef5}); +constexpr StatTable51 QRT_TABLE_51({0x778bf2703d152, 0x2aaaafbff2092, 0x2aaaafbff2090, 0x4d2119c7e7780, 0x2aaaafbff2094, 0x65de1df8ae194, 0x4d2119c7e7788, 0x67d63d7ba262c, 0x2aaaafbff2084, 0x28ff003f4167c, 0x65de1df8ae1b4, 0x658397fb1d034, 0x4d2119c7e77c8, 0x4d7c9284526ba, 0x67d63d7ba26ac, 0x6666333fc0cbe, 0x2aaaafbff2184, 0x295b807ab55ee, 0x28ff003f4147c, 0x2aaabfffe0016, 0x65de1df8ae5b4, 0x209210349d60, 0x658397fb1d834, 0x4d215dc7cf1c8, 0x4d2119c7e67c8, 0x662b2b3d7b4be, 0x4d7c9284506ba, 0x255af00b36e0, 0x67d63d7ba66ac, 0x65de1fb8ac1a6, 0x6666333fc8cbe, 0x662f3b3ded4be, 0x2aaaafbfe2184, 0x663a9dbc3a426, 0x295b807a955ee, 0x4cdc9ec128928, 0x28ff003f0147c, 0x28a0c93cd511c, 0x2aaabfff60016, 0x65d73cf8e78d4, 0x65de1df9ae5b4, 0x4d5eddc44f1c8, 0x209210149d60, 0x357fcc506c8a, 0x658397ff1d834, 0, 0x4d215dcfcf1c8, 0x63f536f5d4554, 0x4d2119d7e67c8, 0x4000000000022, 0x662b2b1d7b4be}); +typedef Field<uint64_t, 51, 75, StatTable51, &SQR_TABLE_51, &SQR2_TABLE_51, &SQR4_TABLE_51, &SQR8_TABLE_51, &SQR16_TABLE_51, &QRT_TABLE_51, IdTrans, &ID_TRANS, &ID_TRANS> Field51; +#endif + +#ifdef ENABLE_FIELD_INT_52 +// 52 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable52; +constexpr StatTable52 SQR_TABLE_52({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x90000, 0x240000, 0x900000, 0x2400000, 0x9000000, 0x24000000, 0x90000000, 0x240000000, 0x900000000, 0x2400000000, 0x9000000000, 0x24000000000, 0x90000000000, 0x240000000000, 0x900000000000, 0x2400000000000, 0x9000000000000, 0x4000000000012}); +constexpr StatTable52 SQR2_TABLE_52({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x9, 0x90, 0x900, 0x9000, 0x90000, 0x900000, 0x9000000, 0x90000000, 0x900000000, 0x9000000000, 0x90000000000, 0x900000000000, 0x9000000000000, 0x41, 0x410, 0x4100, 0x41000, 0x410000, 0x4100000, 0x41000000, 0x410000000, 0x4100000000, 0x41000000000, 0x410000000000, 0x4100000000000, 0x1000000000024, 0x249, 0x2490, 0x24900, 0x249000, 0x2490000, 0x24900000, 0x249000000, 0x2490000000, 0x24900000000, 0x249000000000, 0x2490000000000, 0x4900000000012, 0x9000000000104}); +constexpr StatTable52 SQR4_TABLE_52({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x9000, 0x90000000, 0x900000000000, 0x4100, 0x41000000, 0x410000000000, 0x2490, 0x24900000, 0x249000000000, 0x1001, 0x10010000, 0x100100000000, 0x1000000000900, 0x9009000, 0x90090000000, 0x900000000410, 0x4104100, 0x41041000000, 0x410000000249, 0x2492490, 0x24924900000, 0x9249000000104, 0x1000001, 0x10000010000, 0x100000090, 0x1000000900000, 0x9000009000, 0x90000041, 0x900000410000, 0x4100004100, 0x1000041000024, 0x410000249000, 0x2490002490, 0x4900024900012, 0x249000100100, 0x1001001001, 0x10010010009, 0x100100090090, 0x1000900900900, 0x9009009009000, 0x90090041041, 0x900410410410, 0x4104104104100, 0x1041041024924, 0x410249249249, 0x2492492492490, 0x4924924910002, 0x9249100000004}); +constexpr StatTable52 SQR8_TABLE_52({0x1, 0x1000000000900, 0x900000410000, 0x410249249249, 0x349100000000, 0x1900000d1, 0x10d10d1065965, 0x51010000, 0x2d924909000, 0x4114114114109, 0xe591, 0x1000007c96591, 0x1903981d13981, 0x249000000001, 0x1000100000990, 0x990090451041, 0x41022cb49249, 0x37d824910000, 0x90191890190d8, 0x10d10d106edf5, 0x54114101, 0x102f4b400bd90, 0x5c04114114108, 0x8f5900000ebcc, 0x1e59107b5f401, 0x8f5ab89f5abcc, 0x24924900001, 0x1010010010909, 0x90000041d100, 0x41024f7df7d9, 0x34a591003491, 0x9001900000d0, 0x4c10d106522c, 0xb49051500100, 0x43dafdb40249, 0x428c0c5114109, 0x59001f590e576, 0x49f59f25facf6, 0x19039c4c17881, 0x26fdb4902491, 0x10110010998, 0x109009045cc51, 0x90022a8867d9, 0x9527ffcb5a6dc, 0x9bc01190190d9, 0x14c119006e608, 0x42fd9e0d55042, 0x143f711b5bfd9, 0x5fa58e1809008, 0x23d647ac81eb2, 0x57ac8f223a466, 0x865abc8b4abcc}); +constexpr StatTable52 SQR16_TABLE_52({0x1, 0x4f881c2d96599, 0xd7eb53011fc41, 0x81d7387961fef, 0xd9afe338982c3, 0x17590c140da98, 0x141a99a87a04e, 0x10036ba4083d9, 0x8f4f72ffb12c7, 0xc8b70df241e1b, 0x18bd9e5d46c40, 0x18331d76266bd, 0x4d915f264a4e0, 0x46aeffb8e4037, 0x4800042de37b5, 0xdb172953272e8, 0x17a9c2edf826a, 0x191cf7053e3f2, 0x82036da842cea, 0x5891da126c1e, 0x1e536e9e4af49, 0x451b5638f5449, 0x5a006c6c4f8c8, 0x5ac71a535fb44, 0xd39a4d489ebd0, 0x4704e31bc006d, 0xc4b327f6ffae1, 0x46980b709bd00, 0xd755405154c11, 0x5741be2d0b797, 0xcb3d48ed630cb, 0x98a66c9f4f599, 0x4caa324b91629, 0x816b5015eeeaf, 0xa595e92a8ed4, 0xc93c6d9f5a073, 0x4250068b39e2, 0x105add98997b5, 0x408b030c0bce0, 0xced5e4a4a2028, 0x1eb59d68e7f25, 0x189756a5b6db0, 0xc49c5a7c98b01, 0x18c9a496767cb, 0xde650554b3d49, 0x11077035fd81c, 0x8b37c5e95a659, 0x45b9226c2c25e, 0xdd2b5e20c7c8b, 0x6de972f0e7025, 0x84e80092f5681, 0x8dfcf97183cbc}); +constexpr StatTable52 QRT_TABLE_52({0xc108165459b0e, 0x10004086, 0x10004084, 0xc00000100104e, 0x10004080, 0x2041810a545b0, 0xc000001001046, 0x1181e055efc0, 0x10004090, 0x40810214390, 0x2041810a54590, 0xc000141019106, 0xc000001001006, 0x10816045ab40, 0x1181e055ef40, 0xc000111015196, 0x10004190, 0xe045c19af44a2, 0x40810214190, 0xe045809ad0532, 0x2041810a54190, 0xdb387a03fe646, 0xc000141019906, 0x2000000800000, 0xc000001000006, 0x2486548199c34, 0x108160458b40, 0x2041808a50534, 0x1181e055af40, 0xc0408312153d6, 0xc00011101d196, 0x21499f0e0eed0, 0x10014190, 0xe15dff9faabe2, 0xe045c19ad44a2, 0xdb787b01ea7d6, 0x40810254190, 0xe484409180532, 0xe045809a50532, 0xc14095164d896, 0x2041810b54190, 0x217dee8fb7a74, 0xdb387a01fe646, 0x441810b54190, 0xc000141419906, 0xc3386e15e7f46, 0x2000000000000, 0x1000141419900, 0xc000000000006, 0, 0x248654a199c34, 0xa48654a199c32}); +typedef Field<uint64_t, 52, 9, StatTable52, &SQR_TABLE_52, &SQR2_TABLE_52, &SQR4_TABLE_52, &SQR8_TABLE_52, &SQR16_TABLE_52, &QRT_TABLE_52, IdTrans, &ID_TRANS, &ID_TRANS> Field52; +typedef FieldTri<uint64_t, 52, 3, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5>, &SQR_TABLE_52, &SQR2_TABLE_52, &SQR4_TABLE_52, &SQR8_TABLE_52, &SQR16_TABLE_52, &QRT_TABLE_52, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri52; +#endif + +#ifdef ENABLE_FIELD_INT_53 +// 53 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5> StatTable53; +constexpr StatTable53 SQR_TABLE_53({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x8e, 0x238, 0x8e0, 0x2380, 0x8e00, 0x23800, 0x8e000, 0x238000, 0x8e0000, 0x2380000, 0x8e00000, 0x23800000, 0x8e000000, 0x238000000, 0x8e0000000, 0x2380000000, 0x8e00000000, 0x23800000000, 0x8e000000000, 0x238000000000, 0x8e0000000000, 0x2380000000000, 0x8e00000000000, 0x3800000000047, 0xe00000000011c, 0x18000000000437}); +constexpr StatTable53 SQR2_TABLE_53({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x238, 0x2380, 0x23800, 0x238000, 0x2380000, 0x23800000, 0x238000000, 0x2380000000, 0x23800000000, 0x238000000000, 0x2380000000000, 0x3800000000047, 0x18000000000437, 0x4054, 0x40540, 0x405400, 0x4054000, 0x40540000, 0x405400000, 0x4054000000, 0x40540000000, 0x405400000000, 0x4054000000000, 0x54000000008e, 0x54000000008e0, 0x14000000008e8e, 0x8ea56, 0x8ea560, 0x8ea5600, 0x8ea56000, 0x8ea560000, 0x8ea5600000, 0x8ea56000000, 0x8ea560000000, 0x8ea5600000000, 0xea5600000011c, 0xa560000001015, 0x560000001000b, 0x1600000010003e}); +constexpr StatTable53 SQR4_TABLE_53({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x23800, 0x238000000, 0x2380000000000, 0x40540, 0x405400000, 0x4054000000000, 0x8ea56, 0x8ea560000, 0x8ea5600000000, 0x1600000010003e, 0x1000111000, 0x10001110000000, 0x11100000238000, 0x2380219b80, 0x380219b800047, 0x19b8000405447, 0x4054441114, 0x54441114008e, 0x41114008ea5ee, 0x14008ea5e6c1b8, 0xea5e6c193611c, 0x6c1936100000d, 0x13610000000124, 0x1010338, 0x10103380000, 0x1033800000238, 0x180000023a3e0f, 0x23a3e3d4000, 0x1a3e3d40000437, 0x1d400004014997, 0x40149af1600, 0x149af160008e0, 0xf160008e2a4a3, 0x8e2a4bc4710, 0x2a4bc47101015, 0x1c4710101022bb, 0x101010228120a8, 0x102281208ba380, 0x1208ba3a3c26c, 0xba3a3c26e7e1c, 0x3c26e7e0ad413, 0xe7e0ad414deb9, 0xad414dea4a7d0, 0x14dea4a7c40960, 0x4a7c4094acd8b, 0x4094acd82b43a, 0xacd82b432f376, 0x2b432f3623804, 0x12f36238010027}); +constexpr StatTable53 SQR8_TABLE_53({0x1, 0x11100000238000, 0x1a3e3d40000437, 0x4a7c4094acd8b, 0x1eea7d6a679bbe, 0x1c423906384897, 0x2168da5f2c08c, 0x1b8259ec8ea11, 0x19d1f388b2d3d6, 0x1959a5720001a7, 0x1a6c8fa147c79, 0x191868056d58df, 0x19b717beaf7eb0, 0x1d37e92df66e7f, 0x16ec165c8535b7, 0x7da5e73dba0b3, 0x14d2bece5702b1, 0xadfaa30a5cf7e, 0x101934bec0d066, 0xaae7f006690ce, 0x6bfdbd85eb297, 0x6b2cb00d8c2b5, 0x52ee73aae547e, 0x67461baa976c2, 0xf8a44f459c7d, 0x2579abd0b5fc6, 0x6e7e5e9b82057, 0x1ab0a0fcf2d91c, 0x385dc87020053, 0xfc75a891c9df0, 0xca67b851d0c1a, 0x4c8d3234fd4f7, 0xf3ea564798c7f, 0x16881f479c0d6b, 0x60b0e8e33fe90, 0x18259a2869066d, 0xc52b463fb1475, 0x8229075c3475d, 0x6725108ff0948, 0xd5edf67d5a509, 0xbf52bb2383664, 0xd5b84ac7ed2ab, 0xbb5901d009d56, 0xcb380bfcebc5, 0x5f411d4594745, 0x18bdcb9f9d25da, 0xf0d3abe76ec15, 0x8b1a6404ca3b5, 0x15b7c7c793b65f, 0x11ff16f08569ff, 0x19c1d4c5eb3442, 0x5deb92ff5fc40, 0xa8009f9410cbc}); +constexpr StatTable53 SQR16_TABLE_53({0x1, 0x5a65e677a526f, 0x142b8f50195f72, 0x12b0ca8e8b1225, 0x1b892547f268cc, 0x1239ca3a4824b6, 0x4249dac026ea8, 0x38080cba150e5, 0x903882481cefb, 0x1ad11e5cf99bf0, 0x14fa149116ab75, 0x6cbd888de21e5, 0x1388c718c37a69, 0x89d1eb38e9978, 0xf12019f00f91f, 0xb377986c7da1f, 0x1c780b06da5cb9, 0x1e10c7eee3249d, 0xe1afb7bd8111d, 0xc821f2a6fa090, 0x1a26caa65e1d59, 0x4280741c8cc4c, 0xb9c507337dad8, 0x65bffa0a097b6, 0x12068bb8ed4ac0, 0x6d751e7056355, 0xbccc3fbdcf084, 0x17ed82d58ea927, 0x125a30b543b4b8, 0xaf1ce3f5f84ce, 0x1082e42090b649, 0xf8d6a6212c41a, 0x1f89211d4982d, 0x1910bdfe092d07, 0x9363da9b9b9d3, 0x8a7196ef7b84e, 0x33fe46ddf1dc, 0x1f3f3291cf719d, 0x91a5da69f1035, 0x5a8dc6eb62cfb, 0xaf99fcc57728a, 0x15e73f1aa49f47, 0x2d82e50796b97, 0x1072fcbb074200, 0x15180f0fc7904, 0xa3a194b750f79, 0xb053c3eea9bb3, 0x1e58da5ae123de, 0x10b47afec00861, 0x17cd9ea910639d, 0x1ecf806dbf8c36, 0xf93d00fe6145b, 0x1247d788a3eda}); +constexpr StatTable53 QRT_TABLE_53({0xf940b90844076, 0x1f940b90844052, 0x1f940b90844050, 0x9d2a063b43e64, 0x1f940b90844054, 0x936f69323ec14, 0x9d2a063b43e6c, 0xe12270a88898, 0x1f940b90844044, 0x1f917f00bb5a3c, 0x936f69323ec34, 0x1f622df85b46ee, 0x9d2a063b43e2c, 0x9bc65ab040b66, 0xe12270a88818, 0x958330b931986, 0x1f940b90844144, 0x98e2a06e32e0, 0x1f917f00bb583c, 0x1f877970dc1024, 0x936f69323e834, 0x16cc3c9b1558c2, 0x1f622df85b4eee, 0x16de1c3351dae8, 0x9d2a063b42e2c, 0x1fecdc7855f8ee, 0x9bc65ab042b66, 0x933821b1cb6fe, 0xe12270a8c818, 0x1f675958641c0e, 0x958330b939986, 0x9d97e050e960, 0x1f940b90854144, 0x1f820fa0e38adc, 0x98e2a06c32e0, 0x1650f0e358a010, 0x1f917f00bf583c, 0x1643af4b037a3a, 0x1f877970d41024, 0x1ffe2c281d8c16, 0x936f69333e834, 0xf00d50ffccf8, 0x16cc3c9b3558c2, 0x16bc31cbca943a, 0x1f622df81b4eee, 0xa6cbd8007232, 0x16de1c33d1dae8, 0x15d2a062b42e10, 0x9d2a062b42e2c, 0x1aa77896586ca, 0x1fecdc7a55f8ee, 0, 0x9bc65af042b66}); +typedef Field<uint64_t, 53, 71, StatTable53, &SQR_TABLE_53, &SQR2_TABLE_53, &SQR4_TABLE_53, &SQR8_TABLE_53, &SQR16_TABLE_53, &QRT_TABLE_53, IdTrans, &ID_TRANS, &ID_TRANS> Field53; +#endif + +#ifdef ENABLE_FIELD_INT_54 +// 54 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6> StatTable54; +constexpr StatTable54 SQR_TABLE_54({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x201, 0x804, 0x2010, 0x8040, 0x20100, 0x80400, 0x201000, 0x804000, 0x2010000, 0x8040000, 0x20100000, 0x80400000, 0x201000000, 0x804000000, 0x2010000000, 0x8040000000, 0x20100000000, 0x80400000000, 0x201000000000, 0x804000000000, 0x2010000000000, 0x8040000000000, 0x20100000000000, 0x400000000402, 0x1000000001008, 0x4000000004020, 0x10000000010080}); +constexpr StatTable54 SQR2_TABLE_54({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x804, 0x8040, 0x80400, 0x804000, 0x8040000, 0x80400000, 0x804000000, 0x8040000000, 0x80400000000, 0x804000000000, 0x8040000000000, 0x400000000402, 0x4000000004020, 0x40001, 0x400010, 0x4000100, 0x40001000, 0x400010000, 0x4000100000, 0x40001000000, 0x400010000000, 0x4000100000000, 0x1000000201, 0x10000002010, 0x100000020100, 0x1000000201000, 0x10000002010000, 0x20100804, 0x201008040, 0x2010080400, 0x20100804000, 0x201008040000, 0x2010080400000, 0x20100804000000, 0x1008040001008, 0x10080400010080, 0x804000100004, 0x8040001000040, 0x400010000002, 0x4000100000020}); +constexpr StatTable54 SQR4_TABLE_54({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x80400, 0x804000000, 0x8040000000000, 0x400010, 0x4000100000, 0x1000000201, 0x10000002010000, 0x20100804000, 0x1008040001008, 0x400010000002, 0x100000000100, 0x1008040, 0x10080400000, 0x804000000804, 0x8000001, 0x80000010000, 0x100004020, 0x1000040200000, 0x402000080400, 0x20000804020100, 0x8040200008000, 0x2000080400010, 0x804000000800, 0x8040001, 0x80400010000, 0x4000100004020, 0x1000040001000, 0x400010080400, 0x100804020100, 0x8040201008040, 0x2010080000010, 0x800000000004, 0x200, 0x2000000, 0x20000000000, 0x1008, 0x10080000, 0x100800000000, 0x8000000008040, 0x80002000, 0x800020000000, 0x200000040200, 0x402010080, 0x4020100800000, 0x1008000200008, 0x2000000002, 0x20000000020000, 0x201008000, 0x2010080000000, 0x800000100004}); +constexpr StatTable54 SQR8_TABLE_54({0x1, 0x10080400000, 0x100804020100, 0x1008000200008, 0x10080002000000, 0x4000000804, 0x8000201000040, 0x402000000000, 0x20100004020, 0x1000000000, 0x80000010, 0x20100800000100, 0x201008, 0x80002010080, 0x804020100000, 0x201000040, 0x2000080000, 0x4020100004000, 0x8040000, 0x10000402000, 0x20000, 0x1008000001008, 0x10080402010080, 0x4000000004, 0x40001000040, 0x10080402, 0x4020000004000, 0x40001, 0x2000080402010, 0x20000000000000, 0x1000000200000, 0x10000000000080, 0x4020100000, 0x40201008040, 0x402000080002, 0x4020000800000, 0x1000000201, 0x2000080400010, 0x100800000000, 0x8040001008, 0x400000000, 0x20000004, 0x8040200000040, 0x80402, 0x20000804020, 0x201008040000, 0x80400010, 0x800020000, 0x1008040001000, 0x2010000, 0x4000100800, 0x8000, 0x402000000402, 0x4020100804020}); +constexpr StatTable54 SQR16_TABLE_54({0x1, 0x80002010000, 0x4020000000000, 0x1000000201008, 0x402010000402, 0x20100804020000, 0x8000001000040, 0x2000000000000, 0x804020000800, 0x1000000201, 0x80002000080, 0x804020, 0x8, 0x400010080000, 0x20100000000000, 0x8000001008040, 0x2010080002010, 0x804020100804, 0x8000001, 0x10000000000000, 0x4020100004000, 0x8000001008, 0x400010000400, 0x4020100, 0x40, 0x2000080400000, 0x800000000804, 0x8040001, 0x10080400010080, 0x4020100804020, 0x40000008, 0x402, 0x20100800020000, 0x40000008040, 0x2000080002000, 0x20100800, 0x200, 0x10000402000000, 0x4000000004020, 0x40200008, 0x402000080002, 0x20100804020100, 0x200000040, 0x2010, 0x804000100804, 0x200000040200, 0x10000400010000, 0x100804000, 0x1000, 0x2010000402, 0x20000000020100, 0x201000040, 0x2010000400010, 0x804020100004}); +constexpr StatTable54 QRT_TABLE_54({0x201008000200, 0x26c10916494994, 0x26c10916494996, 0x40008008, 0x26c10916494992, 0x141a2434c12d12, 0x40008000, 0x36c00110594c22, 0x26c10916494982, 0x200000040200, 0x141a2434c12d32, 0x10010816104534, 0x40008040, 0x36da60b01308b2, 0x36c00110594ca2, 0x48200209000, 0x26c10916494882, 0x41b6da2d86106, 0x200000040000, 0x32db2c228965b0, 0x141a2434c12932, 0x9000000200048, 0x10010816104d34, 0x32db68b2832da4, 0x40009040, 0x40045928b4902, 0x36da60b01328b2, 0x1000040000, 0x36c00110590ca2, 0x101b69865a4120, 0x48200201000, 0x22da6434912884, 0x26c10916484882, 0x9000240208008, 0x41b6da2da6106, 0x22c14484c20180, 0x200000000000, 0x4016db29b6812, 0x32db2c228165b0, 0x9008200201048, 0x141a2434d12932, 0x32c36ca2c264b0, 0x9000000000048, 0x140a65b48a2c32, 0x10010816504d34, 0, 0x32db68b2032da4, 0x404490824814, 0x41009040, 0x14da60a4536126, 0x40045908b4902, 0x8000041009008, 0x36da60b41328b2, 0x6db68b2032c12}); +typedef Field<uint64_t, 54, 513, StatTable54, &SQR_TABLE_54, &SQR2_TABLE_54, &SQR4_TABLE_54, &SQR8_TABLE_54, &SQR16_TABLE_54, &QRT_TABLE_54, IdTrans, &ID_TRANS, &ID_TRANS> Field54; +typedef FieldTri<uint64_t, 54, 9, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6>, &SQR_TABLE_54, &SQR2_TABLE_54, &SQR4_TABLE_54, &SQR8_TABLE_54, &SQR16_TABLE_54, &QRT_TABLE_54, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri54; +#endif + +#ifdef ENABLE_FIELD_INT_55 +// 55 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable55; +constexpr StatTable55 SQR_TABLE_55({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x102, 0x408, 0x1020, 0x4080, 0x10200, 0x40800, 0x102000, 0x408000, 0x1020000, 0x4080000, 0x10200000, 0x40800000, 0x102000000, 0x408000000, 0x1020000000, 0x4080000000, 0x10200000000, 0x40800000000, 0x102000000000, 0x408000000000, 0x1020000000000, 0x4080000000000, 0x10200000000000, 0x40800000000000, 0x2000000000102, 0x8000000000408, 0x20000000001020}); +constexpr StatTable55 SQR2_TABLE_55({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x102, 0x1020, 0x10200, 0x102000, 0x1020000, 0x10200000, 0x102000000, 0x1020000000, 0x10200000000, 0x102000000000, 0x1020000000000, 0x10200000000000, 0x2000000000102, 0x20000000001020, 0x10004, 0x100040, 0x1000400, 0x10004000, 0x100040000, 0x1000400000, 0x10004000000, 0x100040000000, 0x1000400000000, 0x10004000000000, 0x40000000102, 0x400000001020, 0x4000000010200, 0x40000000102000, 0x1020408, 0x10204080, 0x102040800, 0x1020408000, 0x10204080000, 0x102040800000, 0x1020408000000, 0x10204080000000, 0x2040800000102, 0x20408000001020, 0x4080000010004, 0x40800000100040, 0x8000001000008}); +constexpr StatTable55 SQR4_TABLE_55({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x10200, 0x102000000, 0x1020000000000, 0x10004, 0x100040000, 0x1000400000000, 0x4000000010200, 0x102040800, 0x1020408000000, 0x4080000010004, 0x100000010, 0x1000000100000, 0x1000010200, 0x10000102000000, 0x1020000102000, 0x1020010004, 0x10200100040000, 0x1000400100040, 0x4001000410200, 0x10004102040800, 0x41020408102000, 0x4081020418004, 0x10204180000010, 0x41800000000040, 0x10300, 0x103000000, 0x1030000000000, 0x10106, 0x101060000, 0x1010600000000, 0x6000000010302, 0x103040c00, 0x103040c000000, 0x40c0000010106, 0x101020418, 0x1010204180000, 0x2041800010302, 0x18000103000008, 0x1030000103000, 0x1030010106, 0x10300101060000, 0x1010600101060, 0x6001010610302, 0x10106103040c00, 0x6103040c103020, 0x40c103041c106, 0x103041c1020418, 0x41c10204081060, 0x2040810214282, 0x8102142800008, 0x21428000000020}); +constexpr StatTable55 SQR8_TABLE_55({0x1, 0x1000010200, 0x101060000, 0x6103040c103020, 0x1001400010200, 0x4081121478004, 0x7903050f103028, 0x1100010200, 0x1020101162000, 0x6703040c113322, 0x113055c1030618, 0x50a102353a804, 0x3e83050f11336a, 0x103040f102071e, 0x101070200, 0x7123050c143020, 0x3100c010200, 0x60c193166c286, 0x6d2b040f15302c, 0x103140f010200, 0x2070f112772e2, 0x7621050c153322, 0x143341cd420418, 0x70e193270ee9e, 0xbe9840f14334e, 0x143355fe53051e, 0x3050e103555fc, 0x5153e50f143c00, 0x1001500050200, 0x450a152957a004, 0x7b071d0f11332a, 0x1000040200, 0x5000040b060000, 0x64061a0c103020, 0x153c44f147c71e, 0x4008142b428a04, 0x2ca75c8f103078, 0x113341f117371e, 0x5402040b162000, 0x62064aac143336, 0x50c003f51ff06, 0x7cf1f3d7ef2e6, 0x6e2d180714332e, 0x103150f050100, 0x43350b1a3152e2, 0x74261e06143020, 0x113f54fd17c65e, 0x56301c3b66cd98, 0x10dff9c7953054, 0x153055fe47360e, 0x46340a1b2277fc, 0x4574bfb5753c14, 0x50f003345fc52, 0x41f653264c9962, 0x7612375be93322}); +constexpr StatTable55 SQR16_TABLE_55({0x1, 0x4ba7488f00015a, 0x30ce9d3a61c1e4, 0x4a2e76980aff84, 0x4e44f5b9d2f610, 0x7b479e4450115c, 0x248c18e86b39b2, 0x1ba74c8406015a, 0x35e93420af76aa, 0x7f282c7c68ad54, 0x7f8b356ad7bc5a, 0x527272878d3b24, 0x587495a40395a4, 0x43c4d0fd51f96c, 0x57ce893a71f0c6, 0x62c68a94803da, 0x1b32bc920e6546, 0x5073c39b469c78, 0x2fba08c009b110, 0x10bd0559ba45c, 0x3bbbd0ca4b3246, 0x243ad4b7c193b8, 0x335d7f186b5db2, 0x5590f3a0fd73f0, 0x72953f208233ba, 0x5210b9a31e6c62, 0x744bb124e351da, 0x4929f00a730244, 0x736ff5bdc1c63c, 0x4c1da1fb246e2e, 0x553c18b46d95cc, 0x268f5c8c143376, 0x438f5a59cbf094, 0x6a718b25fd3946, 0x67053b1bf54fe0, 0x441c5323cb0288, 0x5def8fcd41d5a8, 0x40446cdfcdb062, 0x1043009fb20072, 0xef08d6ed9e9c6, 0xbdf8adea645be, 0x76b092b499c072, 0xd754f98b724c2, 0x5a21d55c8f8752, 0x4f0f36a62eeb0c, 0x262f651fb93b18, 0x3336d340aa0aaa, 0x69375d0e9970fa, 0x2e0997225afe66, 0x6692008b83364e, 0x230856519bc3ae, 0x2c0a54962f8378, 0x2a6460de8a4266, 0x2f14d8fa237452, 0x25934cd7ae0030}); +constexpr StatTable55 QRT_TABLE_55({0, 0x121d57b6623fde, 0x121d57b6623fdc, 0x68908340d10e00, 0x121d57b6623fd8, 0x100300510e20, 0x68908340d10e08, 0x10004096, 0x121d57b6623fc8, 0x100010000, 0x100300510e00, 0x7ea8c890a088e8, 0x68908340d10e48, 0x68809540871648, 0x10004016, 0x68808000808068, 0x121d57b6623ec8, 0x68909240d41c48, 0x100010200, 0x6884c170ad0216, 0x100300510a00, 0x68848160a50200, 0x7ea8c890a080e8, 0x7eecbca04ab4b6, 0x68908340d11e48, 0x120c54b62234c8, 0x68809540873648, 0x69929240d61c48, 0x10000016, 0x68808060800000, 0x68808000800068, 0x80000080, 0x121d57b6633ec8, 0x7ea8cb90a18ae8, 0x68909240d61c48, 0x16284090200080, 0x100050200, 0x474302a345e, 0x6884c170a50216, 0x166cbca0cab4de, 0x100300410a00, 0x1000000000000, 0x68848160850200, 0x688cc1f0a50296, 0x7ea8c890e080e8, 0x7e8cc1f0a50280, 0x7eecbca0cab4b6, 0x68000000000068, 0x68908341d11e48, 0x7880954487365e, 0x120c54b42234c8, 0x9929248d61c20, 0x68809544873648, 0x41121208561c20, 0x69929248d61c48}); +typedef Field<uint64_t, 55, 129, StatTable55, &SQR_TABLE_55, &SQR2_TABLE_55, &SQR4_TABLE_55, &SQR8_TABLE_55, &SQR16_TABLE_55, &QRT_TABLE_55, IdTrans, &ID_TRANS, &ID_TRANS> Field55; +typedef FieldTri<uint64_t, 55, 7, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5>, &SQR_TABLE_55, &SQR2_TABLE_55, &SQR4_TABLE_55, &SQR8_TABLE_55, &SQR16_TABLE_55, &QRT_TABLE_55, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri55; +#endif + +#ifdef ENABLE_FIELD_INT_56 +// 56 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable56; +constexpr StatTable56 SQR_TABLE_56({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x95, 0x254, 0x950, 0x2540, 0x9500, 0x25400, 0x95000, 0x254000, 0x950000, 0x2540000, 0x9500000, 0x25400000, 0x95000000, 0x254000000, 0x950000000, 0x2540000000, 0x9500000000, 0x25400000000, 0x95000000000, 0x254000000000, 0x950000000000, 0x2540000000000, 0x9500000000000, 0x25400000000000, 0x95000000000000, 0x5400000000012a, 0x5000000000043d, 0x40000000001061}); +constexpr StatTable56 SQR2_TABLE_56({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x95, 0x950, 0x9500, 0x95000, 0x950000, 0x9500000, 0x95000000, 0x950000000, 0x9500000000, 0x95000000000, 0x950000000000, 0x9500000000000, 0x95000000000000, 0x5000000000043d, 0x4111, 0x41110, 0x411100, 0x4111000, 0x41110000, 0x411100000, 0x4111000000, 0x41110000000, 0x411100000000, 0x4111000000000, 0x41110000000000, 0x11100000000254, 0x110000000025d5, 0x10000000025dc5, 0x25dcc5, 0x25dcc50, 0x25dcc500, 0x25dcc5000, 0x25dcc50000, 0x25dcc500000, 0x25dcc5000000, 0x25dcc50000000, 0x25dcc500000000, 0x5dcc500000012a, 0xdcc50000001061, 0xcc500000010079, 0xc500000010016c, 0x5000000100103c}); +constexpr StatTable56 SQR4_TABLE_56({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x9500, 0x95000000, 0x950000000000, 0x4111, 0x41110000, 0x411100000000, 0x110000000025d5, 0x25dcc500, 0x25dcc5000000, 0xdcc50000001061, 0x10010101, 0x100101010000, 0x1010100000950, 0x1000009509595, 0x95095959500, 0x5095959500043d, 0x95950004115111, 0x41151505011, 0x11515050110254, 0x505011025de985, 0x11025de9a93c10, 0x5de9a93c19c42a, 0xa93c19c400005d, 0x19c4000001000c, 0x100010094, 0x1000100940000, 0x1009400009500, 0x94000095009500, 0x950095418400, 0x954184004111, 0x41840041114111, 0x411141349dd4, 0x1141349dd425d5, 0x349dd425dce0d5, 0xd425dce0cce1b9, 0xdce0cce1ddd461, 0xcce1ddd4011160, 0xddd401110941f5, 0x1110941959dc4, 0x941959dc49cc5, 0x959dc49cc118d5, 0xc49cc1189454b9, 0xc1189454d4d12c, 0x9454d4d14358f8, 0xd4d14358b9aa44, 0x4358b9aa20a205, 0xb9aa20a221d7b8, 0x20a221d7ed10a2, 0x21d7ed10b0f90a, 0xed10b0f918507b, 0xb0f91850000050, 0x1850000001000d}); +constexpr StatTable56 SQR8_TABLE_56({0x1, 0x1010100000950, 0x950095418400, 0xd4d14358b9aa44, 0x1135dd851025d5, 0x2c3e45b8a8a9d9, 0xcc39c4d816cc89, 0x51109400, 0x8496c8edb8f151, 0x1c2d7d88406199, 0x3856af0918b2ea, 0x2c26c02be43364, 0x7c13f0a9492898, 0x887abc757e3b3c, 0x10100411009c5, 0x850b98e029a995, 0x18309e7d346f24, 0x49e692600134d8, 0xf902789abce101, 0xed998d1d57187b, 0xa5488e663e846e, 0x84267921a952d0, 0x3d464f2d15176e, 0x801aac9d710b04, 0xfc00d6eb916343, 0x5c7fb78f391c1, 0x745ee236e80ea0, 0x81f8c65be65eac, 0x1940095415941, 0x1188025e2103d0, 0x49c166e0b13f34, 0xbd26558f28a2b3, 0xe147d131ae4b81, 0x25b501ad8ba86d, 0x75fb4e24c70a79, 0x88172901f1684d, 0xb090520bb570a2, 0x963c9b97aad59, 0x39b1e5f12c85a6, 0xd90de4c2bf3055, 0x9c921257e4a1b, 0x45f1f318fef834, 0x48161e1eb09635, 0x10685b397538ce, 0xbc8d4a0c6bc62a, 0xdce738247bfad9, 0x1115b3337d25a4, 0x195bf5a6f0999b, 0xb85101388b2f37, 0x8ee1b2833544cf, 0x49bc1efef7da90, 0x346e404662e355, 0x8c0dab6a1217d6, 0x3cb782ec54c968, 0x5efe07d4f59f4, 0x55f19c0f482900}); +constexpr StatTable56 SQR16_TABLE_56({0x1, 0x2563e0b70105c4, 0x48ce07ef5576bb, 0xb94064d844f117, 0x207d2f511ffe3c, 0xf8f6dd1e2a3e6b, 0xe4cc405e0c6cdb, 0xd053f9b827b2bf, 0x550ae8d22edcbf, 0x29f7570f88728b, 0xa06a9e2dfd84a6, 0x55567b9483b3ff, 0x197c6c0d004df6, 0xe106c03f218a16, 0xc50dd2aaf0a388, 0x39473f6702a06c, 0xc8c1736b312ded, 0x992dc692bb707d, 0x24bb9a8dcad06f, 0x9cc45f9e3c2075, 0x455e7271eb130b, 0x847157a5326f59, 0xdc8ccb4ab3f5bd, 0x9463c02c46910d, 0xe1debd0b794514, 0x4c5128db660cde, 0x11910a685416a3, 0x11dfa5b9552a3d, 0x5d902ced822708, 0x794ff735e94601, 0xf1dc5fd7efcf7e, 0x19fb7ff8d06993, 0x7069119ac28a09, 0x98ba5a77d83e7f, 0xf4923dbc1b24e5, 0x7c2dcc84668312, 0xc27e2f5f2243f, 0x78c6d8ebe4bede, 0xad39495debf1a5, 0xa1564b894b50f0, 0x5898ae4e965be9, 0x28aa991e046567, 0x585e95889bb734, 0xc59e73661cf916, 0xed70696926d95d, 0xcca6630954309a, 0x8c4b12ac111264, 0xe8413ad0493e05, 0x1acea73bc9166, 0x9a7f11cd38d12d, 0x390dd1972139ec, 0x9146bc1a4fbff0, 0xd5a1c594335b01, 0x2566272e74ef1a, 0xd4a8baf259e7d0, 0x71e7efd8f20703}); +constexpr StatTable56 QRT_TABLE_56({0x10004084, 0xd058f12fd5925e, 0xd058f12fd5925c, 0x41a60b5566d9f0, 0xd058f12fd59258, 0xbda60a142740ba, 0x41a60b5566d9f8, 0xd059f1afc5e688, 0xd058f12fd59248, 0xfc040841615a22, 0xbda60a1427409a, 0xbda60b5426c1ca, 0x41a60b5566d9b8, 0x1a60b4166b950, 0xd059f1afc5e608, 0xfc000041409822, 0xd058f12fd59348, 0xd1ee7a4ef4185c, 0xfc040841615822, 0x9049759b80b4a4, 0xbda60a1427449a, 0xd258e06f301e18, 0xbda60b5426c9ca, 0x6dfeeb3bf6d7d2, 0x41a60b5566c9b8, 0xbdef3ed4ae398a, 0x1a60b41669950, 0xd1ef3f8eeff04c, 0xd059f1afc5a608, 0xbda203340783de, 0xfc000041401822, 0x2dfefbaff2b27a, 0xd058f12fd49348, 0xfdb788a0706776, 0xd1ee7a4ef6185c, 0x2e5de0ae41337a, 0xfc040841655822, 0x41eb17d5ceecf8, 0x9049759b88b4a4, 0x40048874211afc, 0xbda60a1437449a, 0xd04a720f93400c, 0xd258e06f101e18, 0xbc559cf5ac7fce, 0xbda60b5466c9ca, 0x6dc9759b88b4d6, 0x6dfeeb3b76d7d2, 0x92feea7b275af0, 0x41a60b5466c9b8, 0, 0xbdef3ed6ae398a, 0x2811d5edd8ee2a, 0x1a60b45669950, 0xb1a60b5466c9ca, 0xd1ef3f86eff04c, 0xec493582c8f032}); +typedef Field<uint64_t, 56, 149, StatTable56, &SQR_TABLE_56, &SQR2_TABLE_56, &SQR4_TABLE_56, &SQR8_TABLE_56, &SQR16_TABLE_56, &QRT_TABLE_56, IdTrans, &ID_TRANS, &ID_TRANS> Field56; +#endif +} + +Sketch* ConstructClMul7Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_49 + case 49: return new SketchImpl<Field49>(implementation, 49); +#endif +#ifdef ENABLE_FIELD_INT_50 + case 50: return new SketchImpl<Field50>(implementation, 50); +#endif +#ifdef ENABLE_FIELD_INT_51 + case 51: return new SketchImpl<Field51>(implementation, 51); +#endif +#ifdef ENABLE_FIELD_INT_52 + case 52: return new SketchImpl<Field52>(implementation, 52); +#endif +#ifdef ENABLE_FIELD_INT_53 + case 53: return new SketchImpl<Field53>(implementation, 53); +#endif +#ifdef ENABLE_FIELD_INT_54 + case 54: return new SketchImpl<Field54>(implementation, 54); +#endif +#ifdef ENABLE_FIELD_INT_55 + case 55: return new SketchImpl<Field55>(implementation, 55); +#endif +#ifdef ENABLE_FIELD_INT_56 + case 56: return new SketchImpl<Field56>(implementation, 56); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri7Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_49 + case 49: return new SketchImpl<FieldTri49>(implementation, 49); +#endif +#ifdef ENABLE_FIELD_INT_52 + case 52: return new SketchImpl<FieldTri52>(implementation, 52); +#endif +#ifdef ENABLE_FIELD_INT_54 + case 54: return new SketchImpl<FieldTri54>(implementation, 54); +#endif +#ifdef ENABLE_FIELD_INT_55 + case 55: return new SketchImpl<FieldTri55>(implementation, 55); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_8bytes.cpp b/src/minisketch/src/fields/clmul_8bytes.cpp new file mode 100644 index 0000000000..8dc1089fee --- /dev/null +++ b/src/minisketch/src/fields/clmul_8bytes.cpp @@ -0,0 +1,175 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_8) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_57 +// 57 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable57; +constexpr StatTable57 SQR_TABLE_57({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x22, 0x88, 0x220, 0x880, 0x2200, 0x8800, 0x22000, 0x88000, 0x220000, 0x880000, 0x2200000, 0x8800000, 0x22000000, 0x88000000, 0x220000000, 0x880000000, 0x2200000000, 0x8800000000, 0x22000000000, 0x88000000000, 0x220000000000, 0x880000000000, 0x2200000000000, 0x8800000000000, 0x22000000000000, 0x88000000000000, 0x20000000000011, 0x80000000000044}); +constexpr StatTable57 SQR2_TABLE_57({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x88, 0x880, 0x8800, 0x88000, 0x880000, 0x8800000, 0x88000000, 0x880000000, 0x8800000000, 0x88000000000, 0x880000000000, 0x8800000000000, 0x88000000000000, 0x80000000000044, 0x404, 0x4040, 0x40400, 0x404000, 0x4040000, 0x40400000, 0x404000000, 0x4040000000, 0x40400000000, 0x404000000000, 0x4040000000000, 0x40400000000000, 0x4000000000022, 0x40000000000220, 0x2222, 0x22220, 0x222200, 0x2222000, 0x22220000, 0x222200000, 0x2222000000, 0x22220000000, 0x222200000000, 0x2222000000000, 0x22220000000000, 0x22200000000011, 0x22000000000101, 0x20000000001001}); +constexpr StatTable57 SQR4_TABLE_57({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x880, 0x8800000, 0x88000000000, 0x80000000000044, 0x404000, 0x4040000000, 0x40400000000000, 0x22220, 0x222200000, 0x2222000000000, 0x20000000001001, 0x10001000, 0x100010000000, 0x100000000088, 0x880088, 0x8800880000, 0x88008800000000, 0x88000000040400, 0x404040400, 0x4040404000000, 0x4040000002222, 0x22222222, 0x222222220000, 0x22222200000101, 0x22000001000001, 0x10000000100, 0x100000001000000, 0x10000088000, 0x100000880000000, 0x8800000088000, 0x880004040, 0x8800040400000, 0x404000004040, 0x40000040400220, 0x404002222000, 0x40022220000220, 0x22200002222011, 0x22220100010, 0x22201000100011, 0x10001000100010, 0x10001000108800, 0x10001088008800, 0x10880088008800, 0x880088008c04, 0x88008c040404, 0x8c0404040404, 0x4040404040426, 0x4040404262222, 0x4042622222222, 0x26222222222222, 0x22222222232201, 0x22222322000001, 0x23220000000001}); +constexpr StatTable57 SQR8_TABLE_57({0x1, 0x100010000000, 0x100000880000000, 0x88008c040404, 0x80000000022264, 0x26262604000101, 0x50023220100230, 0x222aa222000001, 0x20000404041401, 0x100404003222000, 0x32aa22aa23aa01, 0x2326048800088, 0x808100222ea722, 0x508e36ec548e34, 0x26222000022223, 0x22000223200001, 0x32001001108801, 0x91001000108844, 0x85048c04880044, 0x4c86ae64c80220, 0x6ea40546662003, 0x66662726ae22ab, 0x40622aa2aa40c8, 0x26063ea6364477, 0x2406744c950437, 0x8a33606aa727aa, 0xd09332ca7e2d9a, 0x7e2c14ce3e6c17, 0x22aa2626260405, 0x22200002626011, 0x27260000002223, 0x22208812aa3011, 0x2ba30040488001, 0x48c8c9caebe26, 0x28898040489001, 0x1048c008922aa26, 0x30c0c08a22b801, 0x12222666fa72601, 0x9afa60a8048eaa, 0x6aa1400afba131, 0xbc9c168c00800d, 0xc083aa60588bb0, 0xbeeefeae26e6e7, 0x8c44ee22226ae2, 0x8c44ee32205042, 0x2667443228d143, 0x36af14ba381d17, 0x72eb33981a3f35, 0x72c911aab20d9d, 0x72ca2b2aaacccd, 0x66463fae3f44ff, 0xac345eee3b4077, 0xe4977caefbe1fd, 0xf01b70a0dd0f9a, 0xf40f74bc580bbd, 0x17e2d1c6c5c0f35, 0x3624140232aa33}); +constexpr StatTable57 SQR16_TABLE_57({0x1, 0xaeeec814065447, 0x110889c99ba3004, 0x1c59ed582798e0e, 0x1c9c766272a2a74, 0x1422928a5250a00, 0x2c97eb48f402a1, 0x10c6d916b128dea, 0x3f2e6ca66ebf67, 0x93ac75fcd63ec8, 0x19263128e42246, 0x1fd1ca54b556091, 0x60ff38200c4e09, 0x1381808ede9982b, 0xc7a9ace2f9dc2a, 0x6c2ee414271c57, 0x3c16f4cffdbe17, 0xc627ec6fe179ee, 0x178f994adf6525b, 0x18be0b635ca1650, 0x4afcb2ae2e98b6, 0x6f81f53a7688dd, 0x45319b3e02c15c, 0x1044be090058910, 0xaa02d012fca063, 0x11fba4c5b80dbfa, 0xf9f44be142268b, 0x1e351a44eb480bf, 0xacf5c17bd0aedf, 0x6f2d74bab851eb, 0x1b8ac3589da9915, 0x1afeb885d3fdd67, 0x7d7d596dd60bbd, 0x1329567316f5723, 0xfdfe23b549fcef, 0xc985ed1e7a009e, 0x71f794bbac1b03, 0xc740582125f7d0, 0x1b3584e031e3b77, 0x29978a3c559ed3, 0xde04d46b4ae516, 0x2f6d6e1c749405, 0x1ec95b44a4251f3, 0xb95b0a5f451f2d, 0x1dc80aedaab9bf2, 0xd0354d3ff74808, 0x180889b484b0364, 0x196895708367d90, 0x104575064a09414, 0x19e88f14fc111ec, 0x1cf4088d3cffd88, 0x1e6c28b9a76c6d5, 0x81ba060c9e485e, 0x12b811107188d68, 0x5e6f10ca82cd88, 0x120882748af043d, 0x145fb82467c596e}); +constexpr StatTable57 QRT_TABLE_57({0xd0c3a82c902426, 0x232aa54103915e, 0x232aa54103915c, 0x1763e291e61699c, 0x232aa541039158, 0x1f424d678bb15e, 0x1763e291e616994, 0x26fd8122f10d36, 0x232aa541039148, 0x1e0a0206002000, 0x1f424d678bb17e, 0x5d72563f39d7e, 0x1763e291e6169d4, 0x1519beb9d597df4, 0x26fd8122f10db6, 0x150c3a87c90e4aa, 0x232aa541039048, 0x15514891f6179d4, 0x1e0a0206002200, 0x14ec9ba7a94c6aa, 0x1f424d678bb57e, 0x1e0f4286382420, 0x5d72563f3957e, 0x4000080000, 0x1763e291e6179d4, 0x1ac0e804882000, 0x1519beb9d595df4, 0x1f430d6793b57e, 0x26fd8122f14db6, 0x3c68e806882000, 0x150c3a87c9064aa, 0x1484fe18b915e, 0x232aa541029048, 0x14f91eb9b595df4, 0x15514891f6379d4, 0x48f6a82380420, 0x1e0a0206042200, 0x14b1beb99595df4, 0x14ec9ba7a9cc6aa, 0x4cf2a82b00420, 0x1f424d679bb57e, 0x26aa0002000000, 0x1e0f4286182420, 0x173f1039dd17df4, 0x5d72563b3957e, 0x4aa0002000000, 0x4000880000, 0x16d31eb9b595df4, 0x1763e291f6179d4, 0x20000000000000, 0x1ac0e806882000, 0x2caa0002000000, 0x1519beb99595df4, 0, 0x1f430d6f93b57e, 0x73e90d6d93b57e, 0x26fd8132f14db6}); +typedef Field<uint64_t, 57, 17, StatTable57, &SQR_TABLE_57, &SQR2_TABLE_57, &SQR4_TABLE_57, &SQR8_TABLE_57, &SQR16_TABLE_57, &QRT_TABLE_57, IdTrans, &ID_TRANS, &ID_TRANS> Field57; +typedef FieldTri<uint64_t, 57, 4, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5>, &SQR_TABLE_57, &SQR2_TABLE_57, &SQR4_TABLE_57, &SQR8_TABLE_57, &SQR16_TABLE_57, &QRT_TABLE_57, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri57; +#endif + +#ifdef ENABLE_FIELD_INT_58 +// 58 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable58; +constexpr StatTable58 LOAD_TABLE_58({0x1, 0x77dd188d5d600, 0x41a7213270def0, 0x10921661867b40a, 0x880b92a6f74da3, 0x143e72cde8f4484, 0x1863cb65c3eee0e, 0x2a1aa4a82154057, 0x22265135db9e135, 0x1446f023770d6d0, 0x183c4be7b4fae6, 0x1cc8a1187e99bd4, 0x3a1ea282e1e2ff1, 0x2700aae9dbcd275, 0x1571f84428a416e, 0xc8eb0b234b8a57, 0x23227afc0d9ba75, 0x1de9497779018c7, 0x5898e896d43329, 0x1501bd1b83bb55f, 0xbb56c28ce180b, 0x188e087d2dbf7e0, 0x36eee77125d8ec9, 0xde4235cbe95166, 0x1c71e4d57306163, 0x2a7e1b1ae5d87ee, 0x3a685560450c909, 0x1cd0545bc185c4b, 0x151779b11f09892, 0x2ab069803c4d787, 0x3bf279c0b825ad5, 0x15edc1ef3d2513c, 0x37bf223b4d0d045, 0x262a786b0324cd3, 0x27658294b696713, 0x33771167b0137e8, 0x86a73ef2dc3271, 0xc64453d2ff05, 0x14c55bc975ce8c, 0x3581164b1e4826e, 0x461dde5468bc26, 0x3f31528346e9451, 0x3f2669a5324a555, 0xd0b1c042854400, 0x32cc9899ea263e3, 0x2a423a8a96f4e95, 0x3e64acd727b470b, 0x2f1f1c1de9c997d, 0x268f2df0a8ab060, 0x14fd82231712442, 0x106e14e5e9e8f8c, 0x2686a15d911a24c, 0x182f831c5142b40, 0x36ca3e60fc7c678, 0x22ba581841d83ff, 0x11539696ce13d17, 0x558c3670aecb8e, 0x2b9b2d828d6d864}); +constexpr StatTable58 SAVE_TABLE_58({0x1, 0xde56167ca60c8e, 0x166391328282dfb, 0x2c6ed660535e701, 0xf83470e499e0e, 0x3686f752cff05fd, 0x1ada23d28022d0b, 0x2a2ac069b41ffd4, 0x2d40f316b40053e, 0x3fb69372877a1f3, 0x13def6e665e9b30, 0x23eb4222bc98b90, 0x2991c5ab618f62c, 0x1c4b63ee1e37a86, 0xdcb10179c77602, 0x708837c2f0ee59, 0x151fe1b533a6d99, 0x44613653cb9d83, 0x33dc64f2b5abae6, 0x27d704726f1f9ba, 0x2fdef2de96892ad, 0x3fd032a21834dbe, 0x1ce2548191e42ab, 0x431410a40ab44b, 0x206f1338c9a75e1, 0x130035675a32179, 0xdf781bb8adbd09, 0x1aaaf085ea624e0, 0x1df0605123c28e9, 0x28d3b3631320c9c, 0x81951a3af55e95, 0x388c776adc88ca1, 0x3ebed178f719885, 0x3c4546b19b0fe51, 0x129564a29700d09, 0x3f642277d82c520, 0x3a46d24ff0ac3fd, 0x1e75e367d627740, 0x33b01746a0f4aad, 0x2af930ca2fa61f, 0x3fcea0ca3af7aac, 0x230722de56e3f4a, 0x3541e58cc5afefd, 0x32cf711ae15ba7e, 0x11d3670d510fc6f, 0x6ddd78eec82112, 0x216210641885856, 0x87535f37c08809, 0x1fa464b5f82155b, 0xdbd43e91708494, 0x1500e23396dd2c4, 0x16cf4098632235f, 0x37e9117da8979ba, 0x6f8bfa04f66a7, 0x18dff008060e626, 0x196286fd9dbad1c, 0x35078156610f8ab, 0x7a669ff8398fea}); +constexpr StatTable58 SQR_TABLE_58({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x63, 0x18c, 0x630, 0x18c0, 0x6300, 0x18c00, 0x63000, 0x18c000, 0x630000, 0x18c0000, 0x6300000, 0x18c00000, 0x63000000, 0x18c000000, 0x630000000, 0x18c0000000, 0x6300000000, 0x18c00000000, 0x63000000000, 0x18c000000000, 0x630000000000, 0x18c0000000000, 0x6300000000000, 0x18c00000000000, 0x63000000000000, 0x18c000000000000, 0x230000000000063, 0xc000000000014a, 0x300000000000528}); +constexpr StatTable58 SQR2_TABLE_58({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x18c, 0x18c0, 0x18c00, 0x18c000, 0x18c0000, 0x18c00000, 0x18c000000, 0x18c0000000, 0x18c00000000, 0x18c000000000, 0x18c0000000000, 0x18c00000000000, 0x18c000000000000, 0xc000000000014a, 0x1405, 0x14050, 0x140500, 0x1405000, 0x14050000, 0x140500000, 0x1405000000, 0x14050000000, 0x140500000000, 0x1405000000000, 0x14050000000000, 0x140500000000000, 0x50000000001ef, 0x50000000001ef0, 0x10000000001ef63, 0x1ef7bc, 0x1ef7bc0, 0x1ef7bc00, 0x1ef7bc000, 0x1ef7bc0000, 0x1ef7bc00000, 0x1ef7bc000000, 0x1ef7bc0000000, 0x1ef7bc00000000, 0x1ef7bc000000000, 0x2f7bc0000000129, 0x37bc0000000112d, 0x3bc000000011027, 0x3c0000000110022}); +constexpr StatTable58 SQR4_TABLE_58({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x18c0, 0x18c00000, 0x18c000000000, 0xc000000000014a, 0x1405000, 0x14050000000, 0x140500000000000, 0x1ef7bc, 0x1ef7bc0000, 0x1ef7bc00000000, 0x3bc000000011027, 0x110001100, 0x1100011000000, 0x11000000194c, 0x1000000194c018c, 0x194c0194c000, 0x14c0194c000014a, 0x194c00001545500, 0x15455154550, 0x154551545500000, 0x1154550001f18df, 0x150001f18c63193, 0x1f18c6318c7c00, 0xc6318c7c01010a, 0x18c7c0101000014, 0x1010000000101, 0x1000000010118c0, 0x10118d8c000, 0x10118d8c0000000, 0xd8c0000018d98a, 0x18d9811050, 0x18d98110500000, 0x18110500001411a, 0x500001410eb94c, 0x1410eb94bbc00, 0x10eb94bbc001ef0, 0x14bbc001ee85ab2, 0x1ee85aac1111, 0x2e85aac11110129, 0x2ac11110111097a, 0x111011109445c8c, 0x11109445c9554c0, 0x1445c9554d95406, 0x9554d954189419, 0xd954189414901f, 0x1894149014051f, 0x149014051e478f, 0x14051e478ee2ec, 0x11e478ee2edef63, 0x38ee2edef7aded3, 0x2edef7adef6bdc8, 0x37adef6bdf07c2d, 0x2f6bdf07c0018f9, 0x1f07c0018c018d1}); +constexpr StatTable58 SQR8_TABLE_58({0x1, 0x1100011000000, 0x10118d8c0000000, 0xd954189414901f, 0xc018def7a2f6f6, 0x1e19cc6d44444e, 0x2b2e8450d1ef706, 0x196294c624791e5, 0x2a9441aa2b74da8, 0x1c6810fa7a2fe66, 0x4e0f0eff6badbb, 0x26faf3a76e59127, 0x11aa58d6498919f, 0x3b3ce4e04f23b30, 0x2ed3f70684ae8d7, 0x8d64c737fc5014, 0x1516c589c4fd458, 0x5ba5cee14ea182, 0x368ad344d93d4ae, 0x15e2547ea25ba2a, 0xdecb4283969d9a, 0x2f2a95e5c791149, 0x3fc958586bc93d2, 0x3216bfeab663783, 0xced412d3f6e530, 0x85fb7bcb26d797, 0x19be97fdbcede01, 0x192a5409529ebf4, 0x98b4f8527795b1, 0x192e8188bbc9aac, 0x322f07e9abdf6d0, 0x2a4a5cd6239de91, 0x4c97dec82e63a, 0x37d6397e1d26aee, 0x1939dc6d77a98db, 0x2e23b8e5b0a982a, 0x2787751f5aa0dba, 0x3f81252033f3cc8, 0x1171b73d009f511, 0x8811f0328040bb, 0x3a659ae0b1d2417, 0xc8b454d91baa72, 0x197b01428520b86, 0x1872c8c17f7fe81, 0x143f7913f4c7f5, 0x3a71b7542e7ec68, 0x3e60d3d49155d34, 0x11d5f10402402c8, 0x2be8db11809a1df, 0x3667129f17b1d6a, 0x2749715db24cd0a, 0x185d6130cdfee96, 0x3abdc4d78640154, 0x39bd5ea2e22f89a, 0x3f9a113c1095209, 0xdb4c8bd4f72f4e, 0x32ae35f0ad0b4ee, 0x2f3770997f16cc}); +constexpr StatTable58 SQR16_TABLE_58({0x1, 0x2fcd2228a7c16ab, 0x846ef4a277243e, 0x1d2bf9061084cfb, 0x23598fddcd64f64, 0xcc36f3a7174e2a, 0x365a50c11b89583, 0x611bed1afae48, 0x22a03fae7957244, 0x45e6546308ff3e, 0x3aebd6f3893b9b4, 0x2bf4a9d5586f8db, 0x32fd7d2d5d6f867, 0x14819feeb813a6f, 0x100ab4d9ad808fc, 0x11c0fd674209c71, 0x3701211690581e7, 0x5c33087013a39a, 0x188935ebbc048bb, 0x10787f930a52538, 0xd49849206986b2, 0x17d1298ba5b565, 0x5d465e006f3142, 0x569a5ce90e9bff, 0x2b591716524b4cb, 0x32f7d39faa352cf, 0x10f701fea440dc0, 0x11c5f10a9d3c9d5, 0x18457154a0bf6ea, 0x15516f140726673, 0x1cf780781353aa4, 0x2a7d7e0e83c4bbc, 0x276c009e3198958, 0x220b8531adc2c11, 0x937d7effc370ab, 0x27632fc1b91dac1, 0x3b36628aa135d3f, 0x37230eddd77f21a, 0x1c1b5e0f410eca9, 0x3200c9c78a9127f, 0x3a55e6fb19e6dc4, 0x150cb064eb271f7, 0x5c74759db43ae1, 0x37046240fba02a9, 0x1937118eb920f04, 0x2795ad9a663a0c9, 0x1d4297ad3d62e8a, 0x3b927d82816e04d, 0x15b56f89c278c21, 0x2b8e4ef675619d6, 0x2e0823575b9bb28, 0xdeb4b405ed7e9c, 0x83d627c04e5155, 0x391134c52f7ae67, 0x9e2c9657999608, 0x3b1e574e9a4eb3a, 0x2b58dd062cd0021, 0x38d1fb86f1978ab}); +constexpr StatTable58 QRT_TABLE_58({0x21b9dfe73454bc2, 0x351ca3a13788360, 0x351ca3a13788362, 0x1ad5a042934094, 0x351ca3a13788366, 0x48f62c33f34cc, 0x1ad5a04293409c, 0x14b1f9a41eb8342, 0x351ca3a13788376, 0x3682437996f7786, 0x48f62c33f34ec, 0x21ad5a152920174, 0x1ad5a0429340dc, 0x3766ef998858a86, 0x14b1f9a41eb83c2, 0x151ca3a437843c2, 0x351ca3a13788276, 0x1e5ac7c1aff42c, 0x3682437996f7586, 0x3767ee558c7856a, 0x48f62c33f30ec, 0x39fb408a690330, 0x21ad5a152920974, 0x372f1d7dbf4255a, 0x1ad5a0429350dc, 0x39bb888af33330, 0x3766ef99885aa86, 0x27b58e0ba2df00, 0x14b1f9a41ebc3c2, 0x1540d06c191bcf2, 0x151ca3a4378c3c2, 0x39ee0d0a17f4c0, 0x351ca3a13798276, 0x2049f6c5379fdb4, 0x1e5ac7c1adf42c, 0x1ac5a182d64bf0, 0x3682437996b7586, 0x16cbe3c0a2c7c1e, 0x3767ee558cf856a, 0x372a1d35b20aa6a, 0x48f62c32f30ec, 0x26ab144a891cdc, 0x39fb408a490330, 0x205df3712ae76a8, 0x21ad5a152d20974, 0x34fb58f12e386b6, 0x372f1d7db74255a, 0x21b4a5f53871674, 0x1ad5a0439350dc, 0x1d602e40318fdc, 0x39bb8888f33330, 0x179bb8888f3332e, 0x3766ef99c85aa86, 0x2cec9eb2f5d0aa8, 0x27b58e03a2df00, 0x6caa1452491cdc, 0x14b1f9a51ebc3c2, 0}); +typedef Field<uint64_t, 58, 99, StatTable58, &SQR_TABLE_58, &SQR2_TABLE_58, &SQR4_TABLE_58, &SQR8_TABLE_58, &SQR16_TABLE_58, &QRT_TABLE_58, StatTable58, &LOAD_TABLE_58, &SAVE_TABLE_58> Field58; +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTableTRI58; +constexpr StatTableTRI58 SQR_TABLE_TRI58({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x80001, 0x200004, 0x800010, 0x2000040, 0x8000100, 0x20000400, 0x80001000, 0x200004000, 0x800010000, 0x2000040000, 0x8000100000, 0x20000400000, 0x80001000000, 0x200004000000, 0x800010000000, 0x2000040000000, 0x8000100000000, 0x20000400000000, 0x80001000000000, 0x200004000000000, 0x10000100002, 0x40000400008, 0x100001000020, 0x400004000080, 0x1000010000200, 0x4000040000800, 0x10000100002000, 0x40000400008000, 0x100001000020000}); +constexpr StatTableTRI58 SQR2_TABLE_TRI58({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x200004, 0x2000040, 0x20000400, 0x200004000, 0x2000040000, 0x20000400000, 0x200004000000, 0x2000040000000, 0x20000400000000, 0x200004000000000, 0x40000400008, 0x400004000080, 0x4000040000800, 0x40000400008000, 0x4000000001, 0x40000000010, 0x400000000100, 0x4000000001000, 0x40000000010000, 0x180001, 0x1800010, 0x18000100, 0x180001000, 0x1800010000, 0x18000100000, 0x180001000000, 0x1800010000000, 0x18000100000000, 0x180001000000000, 0x10000300006, 0x100003000060, 0x1000030000600, 0x10000300006000, 0x100003000060000, 0x30000400004, 0x300004000040, 0x3000040000400, 0x30000400004000, 0x300004000040000, 0x4000020000c, 0x4000020000c0, 0x4000020000c00, 0x4000020000c000}); +constexpr StatTableTRI58 SQR4_TABLE_TRI58({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x2000040, 0x20000400000, 0x200004000000000, 0x40000400008000, 0x4000000001000, 0x18000100, 0x180001000000, 0x10000300006, 0x100003000060000, 0x30000400004000, 0x4000020000c00, 0x200004000100, 0x40001400008, 0x14000000001, 0x140000000010000, 0x380005000, 0x3800050000000, 0x5000070000e0, 0x70000400014, 0x3000040001c0001, 0x40001a0001c000, 0x1a000140001000, 0x1400024000680, 0x240004000050, 0x40000180009, 0x1800010001, 0x18000100010000, 0x1000130000600, 0x1300004000040, 0x4000220004c, 0x22000440001, 0x220004400010000, 0x44000540008800, 0x5400000001100, 0x1b800150, 0x1b8001500000, 0x380015000300006, 0x15000370006e000, 0x37000440005400, 0x440003a000dc0, 0x3a0005400110, 0x20005400160000e, 0x140016400068001, 0x164000400015000, 0x4000398005900, 0x3980051000100, 0x5100063000e6, 0x100063000460014, 0x2300044001c4001, 0x44001820018c00, 0x18200104001100, 0x1040021400608, 0x214004000041, 0x140040000010008}); +constexpr StatTableTRI58 SQR8_TABLE_TRI58({0x1, 0x40001400008, 0x1300004000040, 0x4000398005900, 0x21a004140001000, 0x15007370046e014, 0x14004001b810158, 0x18022100441001, 0x1005130063600e6, 0x4200004181059, 0x447a1a3f41ccd0, 0x383d151573100e6, 0x101600544019940, 0x270027c0059c000, 0x4818070100e1, 0x20545402560168e, 0x76071e40419414, 0x38001501bb00157, 0x18054101401149, 0x104343116260a0c, 0x17b9c06c9180809, 0x35b793b6107d791, 0x2e706624276b452, 0x3a543cf805118, 0x278004063619444, 0x22aae45555d4155, 0x105597ba5075e80, 0x364504676052ce, 0x35800790000401, 0x264044141418809, 0x3313274051a405d, 0x52c1b85195848, 0x57f03b8205e9e, 0x22c5044070044e0, 0x1547370047f115, 0x10402383a848d51, 0x16220024510a1, 0x4050579e30c9e7, 0x15e201b4605c018, 0x297e7fd6e672cc2, 0x286f01429f08ff7, 0x31c56646279854c, 0x36fd34ece6e98e6, 0x31e6939431f00b9, 0x311386d18673a0a, 0x2b6524f5cf195aa, 0x2dd63711ff50016, 0x1585649073391ae, 0x1004431143e1ab5, 0x13be61cf659d4d9, 0x98a87036371777, 0x66673706472d14, 0x273867fcbd99159, 0x27c4c58464098e9, 0x347304213c56db, 0x721f05c140cc15, 0x38144503ed007d9, 0x2e054541404549}); +constexpr StatTableTRI58 SQR16_TABLE_TRI58({0x1, 0x3f4d56f7779e1f0, 0xe27368ee2eeacd, 0x135c653e9699a2f, 0x6b0f78c5b96a46, 0x25fa3044c7e0248, 0x2a078335aa8c788, 0x2b2fb5e8ec09222, 0x214fe2bd0b14a22, 0x10b6f34977f0f41, 0x3dc4a1564361cee, 0xa2ae7c793a9fcf, 0x7fc45e1a362304, 0x3ec19729047ce58, 0x1ef9b26acd27396, 0x225a72a9b2db21a, 0xaaa90ccba715d8, 0x2da6362d54cd62, 0x37dae1e3484d433, 0x1ced37972ce3594, 0x164d907773ab8b9, 0xbeaf6f3fc883a1, 0x1d8ac7ee4682652, 0x102fa1481f0470a, 0x3e17062fd515fba, 0x21652276c35fe65, 0x57862a59d3fa78, 0x36b077a8057cde3, 0x287ce593d9cee2f, 0x290b965ae5d215a, 0x2cc2a18d887125c, 0xc46c603fd8423b, 0xdcd705a0e16776, 0x3307e00c6585a3f, 0x2d82d4b6c18532d, 0x28efe74f174d530, 0x2ddbc57b95adaac, 0x31d41679a107eb4, 0x1f24f6f872cb97f, 0x32718f9b0a03ff6, 0x1f283546f68ca0c, 0x158f309c150c885, 0x1ccaf78ea1873ea, 0x30e3b732bf1875f, 0xcce47efdb9ecb1, 0xcf3954987b5601, 0xebdc136185c456, 0x388046727963e11, 0x22e117909faee51, 0x3215b67613a2a60, 0x172480d3a2f11de, 0x382552280610b4d, 0x3c53c5d9c350cce, 0x6edc0d3330295e, 0x3452a6b8c868f37, 0x398cd7e93017ecc, 0x2e1ec37c30a741e, 0xb00d11006ffa14}); +constexpr StatTableTRI58 QRT_TABLE_TRI58({0x2450096792a5c5c, 0x610014271011c, 0x610014271011e, 0x1f0cb811314ea88, 0x610014271011a, 0x8000000420, 0x1f0cb811314ea80, 0x265407ad8a20bcc, 0x610014271010a, 0x3d18be98392ebd0, 0x8000000400, 0xc29b930e407056, 0x1f0cb811314eac0, 0x1fcef001154dee8, 0x265407ad8a20b4c, 0xc69b924c61f94a, 0x610014271000a, 0x211006895845190, 0x3d18be98392e9d0, 0x54007accac09cc, 0x8000000000, 0xc08b934e107854, 0xc29b930e407856, 0x275407adc220bcc, 0x1f0cb811314fac0, 0x1f6db815164ea8a, 0x1fcef001154fee8, 0x1b2db801945e396, 0x265407ad8a24b4c, 0x21100ec95865590, 0xc69b924c61794a, 0x273507b1e530ad6, 0x610014270000a, 0x1b4cb835b34e29c, 0x211006895865190, 0x3839bf20d47e016, 0x3d18be98396e9d0, 0x3858bd34f36e01c, 0x54007acca409cc, 0, 0x8000100000, 0xc29a130e507856, 0xc08b934e307854, 0x13253921d448296, 0xc29b930e007856, 0x13c60935f6486bc, 0x275407adca20bcc, 0x3571be8c5e6c9da, 0x1f0cb811214fac0, 0x410014261011c, 0x1f6db815364ea8a, 0x13a50921d1486b6, 0x1fcef001554fee8, 0x64001249245a5c, 0x1b2db801145e396, 0x8610014670200a, 0x265407ac8a24b4c, 0x1a5cbfbdeb0f30c}); +typedef FieldTri<uint64_t, 58, 19, StatTableTRI58, &SQR_TABLE_TRI58, &SQR2_TABLE_TRI58, &SQR4_TABLE_TRI58, &SQR8_TABLE_TRI58, &SQR16_TABLE_TRI58, &QRT_TABLE_TRI58, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri58; +#endif + +#ifdef ENABLE_FIELD_INT_59 +// 59 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5> StatTable59; +constexpr StatTable59 SQR_TABLE_59({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x12a, 0x4a8, 0x12a0, 0x4a80, 0x12a00, 0x4a800, 0x12a000, 0x4a8000, 0x12a0000, 0x4a80000, 0x12a00000, 0x4a800000, 0x12a000000, 0x4a8000000, 0x12a0000000, 0x4a80000000, 0x12a00000000, 0x4a800000000, 0x12a000000000, 0x4a8000000000, 0x12a0000000000, 0x4a80000000000, 0x12a00000000000, 0x4a800000000000, 0x12a000000000000, 0x4a8000000000000, 0x2a000000000012a, 0x28000000000043d, 0x200000000001061}); +constexpr StatTable59 SQR2_TABLE_59({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x12a, 0x12a0, 0x12a00, 0x12a000, 0x12a0000, 0x12a00000, 0x12a000000, 0x12a0000000, 0x12a00000000, 0x12a000000000, 0x12a0000000000, 0x12a00000000000, 0x12a000000000000, 0x2a000000000012a, 0x200000000001061, 0x10444, 0x104440, 0x1044400, 0x10444000, 0x104440000, 0x1044400000, 0x10444000000, 0x104440000000, 0x1044400000000, 0x10444000000000, 0x104440000000000, 0x4440000000012a, 0x4440000000012a0, 0x440000000012ea8, 0x40000000012ee28, 0x12ee628, 0x12ee6280, 0x12ee62800, 0x12ee628000, 0x12ee6280000, 0x12ee62800000, 0x12ee628000000, 0x12ee6280000000, 0x12ee62800000000, 0x2ee62800000012a, 0x6e6280000001061, 0x662800000010079, 0x62800000010016c, 0x28000000100103c}); +constexpr StatTable59 SQR4_TABLE_59({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x12a0, 0x12a00000, 0x12a000000000, 0x2a000000000012a, 0x1044400, 0x10444000000, 0x104440000000000, 0x40000000012ee28, 0x12ee628000, 0x12ee6280000000, 0x662800000010079, 0x100101010, 0x1001010100000, 0x101010000012a0, 0x10000012a12b2a, 0x12a12b2b2a00, 0x2a12b2b2a00012a, 0x32b2a0001045461, 0x200010454541421, 0x104545414044000, 0x45414044012ef02, 0x4044012ef4d49aa, 0x12ef4d49e0ce28, 0x74d49e0ce290079, 0x1e0ce290000011d, 0x62900000100016d, 0x100010013a0, 0x100010013a00000, 0x10013a00012a000, 0x3a00012a012a12a, 0x12a012a116e400, 0x12a116e4010444, 0x116e40104450444, 0x40104450457ea6c, 0x4450457ea2692a0, 0x457ea2692ee7020, 0x22692ee706f7059, 0x2ee706f707e73ba, 0x6f707e73901116, 0x7e7390111013b6, 0x390111013b12b16, 0x11013b12b299b2a, 0x3b12b299b398b2a, 0x3299b398b17de61, 0x3398b17de543b4f, 0x317de543b7b1065, 0x6543b7b1053bb27, 0x37b1053bb4d0b6b, 0x53bb4d0b5b95ca, 0x34d0b5b95cfbf5b, 0x35b95cfbf6885b5, 0x5cfbf688587c89a, 0x7688587c8cf3adb, 0x587c8cf3aa00050, 0xcf3aa00001000d}); +constexpr StatTable59 SQR8_TABLE_59({0x1, 0x1001010100000, 0x10013a00012a000, 0x3398b17de543b4f, 0x2a00116b8c2812a, 0x7cbf06ffa4d5cd6, 0x1288f1cf576c2e0, 0x3047cfc394d3391, 0x322d00452b2c451, 0x226dcb1999949d1, 0x2e2e5ab30351bc0, 0x10b2afcfca2edc6, 0x7ff39b98a3372a8, 0x2d7b439441ae332, 0x5603b26a2dae616, 0x3a13899c470338a, 0x16e8a14f0113f3c, 0x754f4aa46d3bb2, 0x38aa45436b16334, 0x634468d6b3f47b5, 0x248ca58bd03241d, 0x255d1fbddf51ae7, 0x7f4b46a330ef6bc, 0x3b3159b37b1a654, 0x7bbff798b50cf3e, 0x568afef7a72512a, 0x701d7955e599ab3, 0x3e7aed5ec2e2c82, 0x5c4d118847ff477, 0x21264d599c12421, 0x4d287fc89bb5a71, 0x6d1f30202fff956, 0x6c54d2de7c68bf8, 0x350c930ed65aed3, 0x5630ddede4ba32c, 0x7c18282af602d36, 0x198a362bf3c8a07, 0x40dde880541e01c, 0x49c0e7e7438c0c7, 0x3ade2abe6845a50, 0x6ffad83e7ac09c4, 0x52185a0d23e667a, 0x2e8c821b63a858a, 0x770e59a57577b23, 0x2fe0ea55e7032a6, 0x23cf0c9a1565a09, 0x1c53d32d80a4427, 0x23164f78db9fa8b, 0x691c4ffab038e2a, 0x33fc91a8a831d85, 0x48039e34eec4e05, 0x2581dbb898c10b5, 0x374067097dfc9f9, 0x241611fdbd3f8e7, 0x1b9f2934941d831, 0x1940c046b9b4a62, 0x5333ac5e7a608f6, 0xe9fa1f11b06830, 0x3d3bbe0ab819c34}); +constexpr StatTable59 SQR16_TABLE_59({0x1, 0x857cdd2d43d447, 0xf829d2f68520b5, 0x19fe843a13f84fd, 0xcac85f3b30aa13, 0x5c7d9cd6997e169, 0x21e7ab9693a08f3, 0xe5cda6478df23f, 0xc3e206ed797b25, 0x755908ad7cca1c1, 0x16236a14b269480, 0x5fedfd73877a5e3, 0x6d66cb2c634cab2, 0x1b60fade310cb41, 0x5dcfd76c147e4ff, 0x2e686c220dcdc6a, 0x1d348a9dfc46113, 0x4e97ec4ce1b1081, 0x20ccc4ae0ada275, 0x5ec224932d09f73, 0x385cecd0572d2a0, 0x520f6a5162503d4, 0x3ff8003ba0976e, 0x5a314f7726ffcb7, 0x505c4f556b43e5a, 0x259ddd3f8c27783, 0x25441858e820409, 0x2714ab44ef6c58b, 0x53437cae5c3011c, 0x122c6454cb53ac0, 0x349b57934525af9, 0x394e01a9ab9a786, 0x665a91eb8e73f0d, 0x4c4e86cc5c98631, 0x7983a92ec037fe2, 0x67919ad3e0a3d69, 0x685c3d6c72af62e, 0x4eafca0e4b49fd7, 0x69534a8afbbeee, 0x720f8307d28c8cb, 0x49828239c03d1b7, 0x4c7e6edd9907a53, 0x1fe81ca4466f8fb, 0x19a865c194c7a23, 0x518bbfec9151454, 0x5b7bfbc756a7e4d, 0x146cc66da8b0754, 0x58e7cba08f0b29b, 0x1b578332a8f1985, 0x72d1c4f9eacac25, 0x6fc4f312025b99a, 0x199f6741974302b, 0x3edcb2e16193874, 0x38b45862414392c, 0x3a6669ab6604f52, 0x227da450a65496e, 0x4e85a5c57a7f719, 0x36b5dbf304b88be, 0x2ba8a1264ef68a0}); +constexpr StatTable59 QRT_TABLE_59({0x38d905ab028567a, 0x789fa6ed3b44d72, 0x789fa6ed3b44d70, 0x74ec857e93d828c, 0x789fa6ed3b44d74, 0x116b3c1203c96, 0x74ec857e93d8284, 0xc25ebc3871e280, 0x789fa6ed3b44d64, 0x47a37c3d910b6, 0x116b3c1203cb6, 0xc7322d7a8f48de, 0x74ec857e93d82c4, 0xb509a0ea52e496, 0xc25ebc3871e200, 0x74fdee4681d3e0c, 0x789fa6ed3b44c64, 0x7ffbbd080b2f09a, 0x47a37c3d912b6, 0xd5c937bae506c8, 0x116b3c12038b6, 0xb173c76987625e, 0xc7322d7a8f40de, 0x7591ff36b3a682c, 0x74ec857e93d92c4, 0x72b253bfbfc90c4, 0xb509a0ea52c496, 0x79f2e7b10e6d452, 0xc25ebc3871a200, 0x78c86e951086aac, 0x74fdee4681dbe0c, 0x78c96eb514c602c, 0x789fa6ed3b54c64, 0xc34818b95658e8, 0x7ffbbd080b0f09a, 0x7399f563b1980f2, 0x47a37c3dd12b6, 0xa29e0e28c58880, 0xd5c937baed06c8, 0x788ac23520ac82c, 0x116b3c13038b6, 0xa2c857e83d92b6, 0xb173c769a7625e, 0x608da990122e48, 0xc7322d7acf40de, 0xa3a89269eebefe, 0x7591ff36bba682c, 0xa25ebc2871a200, 0x74ec857e83d92c4, 0x11f62e419f1cfe, 0x72b253bf9fc90c4, 0x7425ebc2871a272, 0xb509a0ee52c496, 0x4ed8555979c8de, 0x79f2e7b18e6d452, 0x6c3580d5915d4d2, 0xc25ebc2871a200, 0, 0x78c86e971086aac}); +typedef Field<uint64_t, 59, 149, StatTable59, &SQR_TABLE_59, &SQR2_TABLE_59, &SQR4_TABLE_59, &SQR8_TABLE_59, &SQR16_TABLE_59, &QRT_TABLE_59, IdTrans, &ID_TRANS, &ID_TRANS> Field59; +#endif + +#ifdef ENABLE_FIELD_INT_60 +// 60 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6> StatTableTRI60; +constexpr StatTableTRI60 SQR_TABLE_TRI60({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000, 0xc0000000, 0x300000000, 0xc00000000, 0x3000000000, 0xc000000000, 0x30000000000, 0xc0000000000, 0x300000000000, 0xc00000000000, 0x3000000000000, 0xc000000000000, 0x30000000000000, 0xc0000000000000, 0x300000000000000, 0xc00000000000000}); +constexpr StatTableTRI60 SQR2_TABLE_TRI60({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x3, 0x30, 0x300, 0x3000, 0x30000, 0x300000, 0x3000000, 0x30000000, 0x300000000, 0x3000000000, 0x30000000000, 0x300000000000, 0x3000000000000, 0x30000000000000, 0x300000000000000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x500000, 0x5000000, 0x50000000, 0x500000000, 0x5000000000, 0x50000000000, 0x500000000000, 0x5000000000000, 0x50000000000000, 0x500000000000000, 0xf, 0xf0, 0xf00, 0xf000, 0xf0000, 0xf00000, 0xf000000, 0xf0000000, 0xf00000000, 0xf000000000, 0xf0000000000, 0xf00000000000, 0xf000000000000, 0xf0000000000000, 0xf00000000000000}); +constexpr StatTableTRI60 SQR4_TABLE_TRI60({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x30, 0x300000, 0x3000000000, 0x30000000000000, 0x500, 0x5000000, 0x50000000000, 0x500000000000000, 0xf000, 0xf0000000, 0xf00000000000, 0x11, 0x110000, 0x1100000000, 0x11000000000000, 0x330, 0x3300000, 0x33000000000, 0x330000000000000, 0x5500, 0x55000000, 0x550000000000, 0x50000000000000f, 0xff000, 0xff0000000, 0xff00000000000, 0x101, 0x1010000, 0x10100000000, 0x101000000000000, 0x3030, 0x30300000, 0x303000000000, 0x30000000000005, 0x50500, 0x505000000, 0x5050000000000, 0x5000000000000f0, 0xf0f000, 0xf0f0000000, 0xf0f00000000000, 0x1111, 0x11110000, 0x111100000000, 0x111000000000003, 0x33330, 0x333300000, 0x3333000000000, 0x330000000000055, 0x555500, 0x5555000000, 0x55550000000000, 0x500000000000fff, 0xffff000, 0xffff0000000, 0xffff00000000000}); +constexpr StatTableTRI60 SQR8_TABLE_TRI60({0x1, 0x110000, 0x10100000000, 0x111000000000003, 0x300030, 0x33003300000, 0x30303000000005, 0x330000000555555, 0x50000000500, 0x50000005500000f, 0x5050000f0f000, 0x5000ffff0000fff, 0xf000f000f011, 0xff00ff1010101, 0xf0e11111111111, 0x31, 0x3210000, 0x313100000000, 0x221000000000056, 0x5300530, 0x563056300000, 0x5353530000000f5, 0x63000000faaaaaa, 0xf5000000f500, 0x500000fa500010e, 0xf5f50011e1f000, 0x5010ffef0010ffe, 0x11f011f011f321, 0x10ef10ec1313131, 0x1e2d22222222222, 0x501, 0x55110000, 0x5040100000000, 0x411000000000ffc, 0xf030f030, 0xff33ff3300000, 0xc0c03000001114, 0x330000100555554, 0x11050000110500, 0x50001015500303f, 0x114050333c0f003, 0x5300fcff0300fcf, 0x330f330f330a511, 0x3fc03af4040404, 0x395b44444444444, 0xf531, 0xfa6210000, 0xf5c43100000000, 0x721000000010fa8, 0x11f521f530, 0x10ea73ea6300000, 0x4d4c530000322d7, 0x63000310faaaa9b, 0x321f5000321f500, 0x500313ea505343e, 0x2d4f55677d1f056, 0x310acef5310ace, 0x621a621a62e562e, 0x43bc4cb34c4c4c4, 0x878788888888888}); +constexpr StatTableTRI60 SQR16_TABLE_TRI60({0x1, 0x563055110000, 0x111010233c0f003, 0x1200afffa8baffc, 0x5356030553000c5, 0x7145cf221744a77, 0x5748045489aaaaf, 0x7d52fcee4febdb3, 0x221f633c000a012, 0x41431fb55d4f4c8, 0x7f126132f4be5d5, 0x323da1f43c3a7e0, 0x373b24844474766, 0x6cc378a25584eb, 0x7ef66648aae4aca, 0x33003000031, 0xc0c03fb7c0f1fb, 0x174757777d10536, 0x2116210a52facb3, 0x5316fc100c1fb35, 0x7aae07597d161e1, 0x6752c4decfb6b7f, 0xf590fa78d56bf3, 0x1be67573275f157, 0xe3e0e9e0d61817, 0x25ac0012251ff6c, 0x407de1e40e3a849, 0x7a7264848fdf67e, 0x3bb8ba7d3879348, 0x498941f57060c6c, 0x5000000f0f501, 0x10fa8cfc1213ac0, 0x51a500f5501aab9, 0x73ef9049dcace64, 0x526a202f322f6e7, 0x2789a852500ca93, 0x4d1346684907509, 0x7d02bcfe4febdb2, 0x330a0329aba0521, 0x50a33c66415f5eb, 0x2e99dced402a73d, 0xf78f2f1a2dbcfe, 0x793a675db461a6a, 0x73848cd4c2f25d2, 0x54fa22d244aa9c6, 0xfae22e13e01501, 0x538ead296f222e5, 0x4da65592d2a750a, 0x40f91ebc14fcd2a, 0x5e73ff2f3c21c03, 0x4c72dce55551460, 0x3ffa59f8e5aef0a, 0x30057fa7b802f82, 0x36efe87d58aa6e4, 0x3bc96a196d71957, 0x5a82cfde2ad602f, 0x1f9bce94df9d3bf, 0x43c91d9b6bcabba, 0x2193c1833502ba3, 0xd28f516c1311d3d}); +constexpr StatTableTRI60 QRT_TABLE_TRI60({0x6983c00fe00104a, 0x804570322e054e6, 0x804570322e054e4, 0x15673387e0a4e4, 0x804570322e054e0, 0x100010110, 0x15673387e0a4ec, 0x920d01f34442a70, 0x804570322e054f0, 0x7a8dc0f2e4058f0, 0x100010130, 0x120c01f140462f0, 0x15673387e0a4ac, 0x7bdbb2ca9a4fe5c, 0x920d01f34442af0, 0xe9c6b039ce0c4ac, 0x804570322e055f0, 0xfac8b080ca20c00, 0x7a8dc0f2e405af0, 0x7a8dc4b2e4a59f0, 0x100010530, 0x10000100000, 0x120c01f14046af0, 0x131a02d91c5db6c, 0x15673387e0b4ac, 0x15623387d0b4ac, 0x7bdbb2ca9a4de5c, 0x7ffbbbca0a8ee5c, 0x920d01f34446af0, 0x800000020000000, 0xe9c6b039ce044ac, 0x81130302500f000, 0x804570322e155f0, 0x935b72eb3a48e9c, 0xfac8b080ca00c00, 0x120c016140563c0, 0x7a8dc0f2e445af0, 0x7bcbb3ca8a4ee5c, 0x7a8dc4b2e4259f0, 0xc4000a0300, 0x100110530, 0x11623285c1b19c, 0x10000300000, 0x420890090c3000, 0x120c01f14446af0, 0x68d7b33b9e0b4ac, 0x131a02d9145db6c, 0xe8ccb1e18a56fc0, 0x15673386e0b4ac, 0x7aadc8f2e485af0, 0x15623385d0b4ac, 0x4a0990093c3000, 0x7bdbb2cada4de5c, 0xf9d6b3389e0b4ac, 0x7ffbbbca8a8ee5c, 0xdf6ba38cec84ac, 0x920d01f24446af0, 0x520d01f24446af0, 0x800000000000000, 0}); +typedef FieldTri<uint64_t, 60, 1, StatTableTRI60, &SQR_TABLE_TRI60, &SQR2_TABLE_TRI60, &SQR4_TABLE_TRI60, &SQR8_TABLE_TRI60, &SQR16_TABLE_TRI60, &QRT_TABLE_TRI60, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri60; +#endif + +#ifdef ENABLE_FIELD_INT_61 +// 61 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable61; +constexpr StatTable61 SQR_TABLE_61({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x138000, 0x4e0000, 0x1380000, 0x4e00000, 0x13800000, 0x4e000000, 0x138000000, 0x4e0000000, 0x1380000000, 0x4e00000000, 0x13800000000, 0x4e000000000, 0x138000000000, 0x4e0000000000, 0x1380000000000, 0x4e00000000000, 0x13800000000000, 0x4e000000000000, 0x138000000000000, 0x4e0000000000000, 0x1380000000000000, 0xe0000000000004e, 0x180000000000011f}); +constexpr StatTable61 SQR2_TABLE_61({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x1000000000000000, 0x138, 0x1380, 0x13800, 0x138000, 0x1380000, 0x13800000, 0x138000000, 0x1380000000, 0x13800000000, 0x138000000000, 0x1380000000000, 0x13800000000000, 0x138000000000000, 0x1380000000000000, 0x180000000000011f, 0x1054, 0x10540, 0x105400, 0x1054000, 0x10540000, 0x105400000, 0x1054000000, 0x10540000000, 0x105400000000, 0x1054000000000, 0x10540000000000, 0x105400000000000, 0x1054000000000000, 0x540000000000138, 0x14000000000013ce, 0x13d96, 0x13d960, 0x13d9600, 0x13d96000, 0x13d960000, 0x13d9600000, 0x13d96000000, 0x13d960000000, 0x13d9600000000, 0x13d96000000000, 0x13d960000000000, 0x13d9600000000000, 0x1d9600000000011f, 0x196000000000101a, 0x1600000000010004}); +constexpr StatTable61 SQR4_TABLE_61({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x138, 0x1380000, 0x13800000000, 0x138000000000000, 0x10540, 0x105400000, 0x1054000000000, 0x540000000000138, 0x13d9600, 0x13d96000000, 0x13d960000000000, 0x1600000000010004, 0x100111000, 0x1001110000000, 0x11100000000138, 0x10000000013812b8, 0x13812ab8000, 0x13812ab80000000, 0x12ab800000010540, 0x105514114, 0x1055141140000, 0x551411400000138, 0x1140000013d84f6, 0x13d84f72f60, 0x13d84f72f600000, 0x4f72f6000010004, 0xf6000010000010f, 0x1000001010100, 0x10101000138, 0x101010001380000, 0x100013800013938, 0x138000139393800, 0x1393938010540, 0x193938010540011f, 0x180105400104445f, 0x540010444454138, 0x1044445413d9600, 0x445413d96013cae, 0x13d96013caaab96, 0x16013caaab970004, 0x1caaab970011111f, 0xb9700111101100b, 0x11110110010028, 0x11011001002812b8, 0x1001002812aab938, 0x2812aab92b8138, 0x12aab92b81382ec0, 0x192b81382ed1400b, 0x1382ed140105514, 0xed1401055150567, 0x105515056890f6, 0x1515056890f613ce, 0x56890f613d84e58, 0x10f613d84e5db85c, 0x13d84e5db84f6010, 0xe5db84f6000000e, 0x184f600000010123}); +constexpr StatTable61 SQR8_TABLE_61({0x1, 0x100111000, 0x10101000138, 0x1001002812aab938, 0x1000001390478, 0x113916c2d28792b8, 0x1904457c4545aa5f, 0x13aa7f0f280c5e20, 0x1047900101540, 0x13be84504128808e, 0x839d72c6e39c0f1, 0x16a18bbeafc6bac6, 0x7290382d6ea1584, 0x1d7d80a66b181691, 0x19d2aaa6110c5d47, 0x1b613d85f602c96f, 0x3812870738, 0x113dbce704cbbd40, 0xd92856e5392f94b, 0x84f76c3d7c304a3, 0x1a519225fe5ce8cf, 0x1704aca0c7190b8e, 0xb7fb1620ed7d025, 0x12831368539314f6, 0x748fb7c048744be, 0x78cc8029440fcba, 0x10eb05b6015eb730, 0xfd3c38351ebc6bd, 0x1665bcfabbfbe624, 0x136549cb4738e1ec, 0x6db6139d4b707f2, 0x1000057853aeac78, 0x104401500109340, 0x554c25992c8f3d8, 0x192dd4b6c0886747, 0x219c35ac73165fc, 0xdf27daa47ee296b, 0x73ab415a10863d2, 0x1f06884b4f2dc1dd, 0xb56c8c3efd7847f, 0x7a6a82768a4a3f2, 0x8773791c3b9f69f, 0x1e4d128bbd8fa105, 0x16977fb4d8984d86, 0xb9a5106882f60bf, 0xc5102ee91822469, 0xdab44dc3cdf7a0b, 0x18d48e2841f63e4, 0x165b8e4d03de40d4, 0x11a7aec6ef42385a, 0x17064ddd9b5041ea, 0xf89b61f74d1f401, 0x18583a8c57e6cb7f, 0x607279105fda3be, 0x905e9c0d58240c7, 0x1ed3c0319519fa7d, 0xa3227b6d1cc17a1, 0xf6cb7bb2aa84563, 0xdda77eb9b649e97, 0x15480a00ec829caf, 0x62cb6da6128c272}); +constexpr StatTable61 SQR16_TABLE_61({0x1, 0x1c7cd18a3a216933, 0xd201ddad374eb4, 0xee4694049c47289, 0x40db9f51130a1e6, 0x134cab3c67ec43f4, 0x97823873a2fc00f, 0xc08b772e8161a43, 0x128159f3d3611eac, 0x1f002f36181d6c4, 0x9de899abbd8d18f, 0x1a6ecb093fbb558b, 0xa6a1251b5961643, 0x1b285c169fb6616d, 0x9c04f5fcf0a4ce5, 0xd050c0ab89025ad, 0xdab152bf63418d9, 0xad3e33af7686059, 0x1561180155ac0dc8, 0x1d9e862521ab7d29, 0xa21b06e1e7632b5, 0x29b84e35cfc95ac, 0x17a27c78dac90e2c, 0x1312fa5f7b1e4ea2, 0xfe66bf53de6a93d, 0x182041e17dde85e9, 0x1289eb06f1803a2e, 0x129449a509af818c, 0x1f308057c81ab449, 0x419981420870054, 0x19f853b859910eb1, 0x9b422c0e9d60871, 0x9e6aec92bfcfa99, 0x15a788f1748b8f44, 0x1fa9a9c171dd83a1, 0x14096af6c0840cc6, 0x1bbe256976515067, 0x14f853fd9e5c0002, 0xf6256b0235f7a8, 0x37e727448043cf6, 0xbb0f467dd137c3f, 0x2538d574ceec19e, 0x15ff26c652c82188, 0x1c22b1e2a9ed31f3, 0x1f56b4b705c21301, 0x1502df3e9aa51832, 0x89c3dec02a6a543, 0x15eac5a464a4f736, 0x1d5023636fc14fa7, 0x499c5d458f9699e, 0x355b147c1703428, 0x1864a11df3efee51, 0x9af0f612e9c1265, 0x9c613962a1c08d9, 0x1cee6fc68f73b3f7, 0x185720007e663719, 0x101dd90a4502bf06, 0x1569af254da87eb0, 0x1781376276013a90, 0x10d2bf3d5e191483, 0x6215713bdc7d250}); +constexpr StatTable61 QRT_TABLE_61({0x171d34fcdac955d0, 0x12cfc8c049e1c96, 0x12cfc8c049e1c94, 0x71d34fcdac955c2, 0x12cfc8c049e1c90, 0x631c871de564852, 0x71d34fcdac955ca, 0x129fa6407f27300, 0x12cfc8c049e1c80, 0x7094f6fdd0a3b12, 0x631c871de564872, 0xdb28cee59c8256a, 0x71d34fcdac9558a, 0xc8a0be15a915472, 0x129fa6407f27380, 0x12dfcb4058e0b80, 0x12cfc8c049e1d80, 0x117d7f04ad0118, 0x7094f6fdd0a3912, 0x621b576dbe35b6a, 0x631c871de564c72, 0x13c808a013a1ee0, 0xdb28cee59c82d6a, 0x113d79842a0272, 0x71d34fcdac9458a, 0x719776b580b6a98, 0xc8a0be15a917472, 0x6633498d6db760a, 0x129fa6407f23380, 0xbd4ae9e8c3e7560, 0x12dfcb4058e8b80, 0x8000000a, 0x12cfc8c049f1d80, 0x634ce9add3b26ea, 0x117d7f04af0118, 0xda3f19c5d66258a, 0x7094f6fdd0e3912, 0xb87427e85e71560, 0x621b576dbeb5b6a, 0xc8b0b085b8c4e0a, 0x631c871de464c72, 0x1538fc8649458a, 0x13c808a011a1ee0, 0xcddbca6d1cfe360, 0xdb28cee59882d6a, 0xae80f550d1ffff2, 0x113d7984aa0272, 0xda7770f5f195912, 0x71d34fcdbc9458a, 0x137c8a049a1ee0, 0x719776b5a0b6a98, 0xded39a9d236ba78, 0xc8a0be15e917472, 0x6732488ca7ce0a, 0x6633498dedb760a, 0xc0406d0527cb80a, 0x129fa6417f23380, 0x3d4ae9eac3e756a, 0xbd4ae9eac3e7560, 0, 0x12dfcb4458e8b80}); +typedef Field<uint64_t, 61, 39, StatTable61, &SQR_TABLE_61, &SQR2_TABLE_61, &SQR4_TABLE_61, &SQR8_TABLE_61, &SQR16_TABLE_61, &QRT_TABLE_61, IdTrans, &ID_TRANS, &ID_TRANS> Field61; +#endif + +#ifdef ENABLE_FIELD_INT_62 +// 62 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable62; +constexpr StatTable62 SQR_TABLE_62({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x20000001, 0x80000004, 0x200000010, 0x800000040, 0x2000000100, 0x8000000400, 0x20000001000, 0x80000004000, 0x200000010000, 0x800000040000, 0x2000000100000, 0x8000000400000, 0x20000001000000, 0x80000004000000, 0x200000010000000, 0x800000040000000, 0x2000000100000000, 0x440000002, 0x1100000008, 0x4400000020, 0x11000000080, 0x44000000200, 0x110000000800, 0x440000002000, 0x1100000008000, 0x4400000020000, 0x11000000080000, 0x44000000200000, 0x110000000800000, 0x440000002000000, 0x1100000008000000}); +constexpr StatTable62 SQR2_TABLE_62({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x1000000000000000, 0x80000004, 0x800000040, 0x8000000400, 0x80000004000, 0x800000040000, 0x8000000400000, 0x80000004000000, 0x800000040000000, 0x440000002, 0x4400000020, 0x44000000200, 0x440000002000, 0x4400000020000, 0x44000000200000, 0x440000002000000, 0x400000000000001, 0x20000011, 0x200000110, 0x2000001100, 0x20000011000, 0x200000110000, 0x2000001100000, 0x20000011000000, 0x200000110000000, 0x2000001100000000, 0x11100000008, 0x111000000080, 0x1110000000800, 0x11100000008000, 0x111000000080000, 0x1110000000800000, 0x1100000088000004, 0x1000000800000044, 0x8080000444, 0x80800004440, 0x808000044400, 0x8080000444000, 0x80800004440000, 0x808000044400000, 0x80000404000002, 0x800004040000020, 0x40440000202, 0x404400002020, 0x4044000020200, 0x40440000202000, 0x404400002020000}); +constexpr StatTable62 SQR4_TABLE_62({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x80000004, 0x800000040000, 0x440000002, 0x4400000020000, 0x20000011, 0x200000110000, 0x2000001100000000, 0x11100000008000, 0x1000000800000044, 0x8080000444000, 0x800004040000020, 0x40440000202000, 0x400000000000101, 0x20001011000, 0x200010110000000, 0x101110000000800, 0x1100008088000404, 0x80808004044400, 0x80044404000202, 0x444044002020200, 0x440002002001110, 0x20002011101100, 0x20110011000080, 0x1100111000800080, 0x1110080000804400, 0x800080844004440, 0x808400044402000, 0x404400002021, 0x44000000210001, 0x300010110, 0x3000101100000, 0x101118000000c, 0x1118000800c0004, 0x8084c0040446, 0x84c00444460002, 0x4440460020213, 0x404600022130011, 0x2000201120111011, 0x2011301110118000, 0x3011001900008044, 0x1918080044c044, 0x1808004840440064, 0x484c4000646020, 0xc40004040200121, 0x40460001213100, 0x600010111000101, 0x101120001011800, 0x1200018198000404, 0x181910004044800, 0x110004c488000606, 0x4c4808006064400, 0x80046404001312, 0x464044013120200, 0x440112002001190, 0x1120002011901100, 0x20190011004480, 0x1900111044800080, 0x1110480000806400}); +constexpr StatTable62 SQR8_TABLE_62({0x1, 0x400000000000101, 0x44000000210001, 0x40460001213100, 0x404500002021, 0xe40014150200121, 0x80b400145512000, 0x1495c4000646820, 0x8000808c4004445, 0xd0800c8c8440561, 0x1045c80080ad6405, 0x1988b0805419944, 0x190110048480008e, 0x2891049dcc008662, 0x8ac190411026482, 0x241574511233a020, 0x1120002031901110, 0x3040203922110044, 0x1110792020b25580, 0x282a4830647355, 0x2c60001037102032, 0x26e4507065221080, 0x2036c57040579390, 0x3450409552c0cc02, 0x5c4000c66824017, 0x5508ce8e2845301, 0x4934eca8d59343, 0x1c28a918f7c9c0d1, 0xb080581194c8e4, 0x1018495dc440e46a, 0x1ac80985d8604226, 0xc7044545722023, 0x145120003031900, 0x4440003a0200005, 0x134447a19a002514, 0x510e645a31f1135, 0xae4834446175200, 0x264f451435730311, 0x7220c2004155891, 0x2153045891358c65, 0x154154800ca02904, 0x54dc0c88ce92565, 0xdc54bc04d28bc20, 0x8c54b0401283d8b, 0x29088e8109411f28, 0x12d0dc41982620a, 0xc0030c89a712640, 0x1dc8192422907592, 0x145554681022a075, 0x470792013225580, 0x346c6a7130667300, 0x39147d7004b077b2, 0x6c83e2d354461c6, 0xcf6d0046247a030, 0x3221c0f063a45c80, 0x303645fc20539787, 0x21004cf150409b10, 0x1dd380444d78042, 0x1c709df8b7145381, 0x185834a4e8d51327, 0x1420e118b389a4d1, 0xf0c41811b4e8c4}); +constexpr StatTable62 SQR16_TABLE_62({0x1, 0x147095f0731417c5, 0x3189fad107702e11, 0x3d3937fd86a460ab, 0x3ff26c959b47c587, 0x1e2ecbec4bf22bd6, 0x168ebaeceaf71b82, 0x216d6c4471f75c10, 0x1f6d31ccabfaee58, 0x1652ef2066ec0c61, 0x3d62ef6847f808fc, 0x26a33c99ec1b43d4, 0x32f26e79367c91ed, 0x361dcdd0d1e73240, 0xe2d494d081269e2, 0x33d231b9098b6045, 0x3c4e93c22fb78a3, 0x2f655fa56e578df3, 0x3a2b9600532c2609, 0x864e125951bbdb7, 0x2e2fca705bb62c58, 0x28e0629106401eaa, 0x7ac20f0ed6cdc1f, 0x3bd50add28a35850, 0x1a6e5ea19a59ab5d, 0x2add6d1d8c0aaefb, 0x2c3cf9842e6956a3, 0x1906944685f2c7c, 0x925997c95ed1de2, 0xcb9eb5d43c6f2e9, 0x1795f2b48a0fa71d, 0x19de5de41acc2100, 0x2e30c3a8444ef165, 0x29433812a3c4b1cd, 0xcbfa65dcdae6d63, 0x2580f2100e56c068, 0x25ce14544acc08cb, 0x24fa7059a7c87e18, 0x2a01d608b5d57d70, 0x3cefa2f54bdabc51, 0x29225fd40de84dea, 0x2d2276d8df087f20, 0x1a077580d9c5e840, 0x33b71879319b7de1, 0x16017e84617bddf4, 0x2596d6b0bd1a954c, 0x10267caddadbf666, 0x22c43bd90eaa3e05, 0xcaf6704a39c29fc, 0x25a0b38132106551, 0x1a78d1fcfd98f2a2, 0x1924d0b08fe1cc34, 0x3ea0a05c4cb14ee5, 0xa9b505540022072, 0x1e65cd1d5556d710, 0x3682cccd684103f1, 0x20a58fb864d70967, 0x35bfeeacb88f9b9b, 0x3b72dce9c4b09b87, 0x839908c285aaa64, 0x2ed676dc722e9732, 0x3dd67b08dc071450}); +constexpr StatTable62 QRT_TABLE_62({0x30268b6fba455d2c, 0x200000006, 0x200000004, 0x3d67cb6c1fe66c76, 0x200000000, 0x3fc4f1901abfa400, 0x3d67cb6c1fe66c7e, 0x35e79b6c0a66bcbe, 0x200000010, 0x1e9372bc57a9941e, 0x3fc4f1901abfa420, 0x21ec9d424957a5b0, 0x3d67cb6c1fe66c3e, 0x1cb35a6e52f5fb0e, 0x35e79b6c0a66bc3e, 0x215481024c13a730, 0x200000110, 0x1c324a6c52f75b08, 0x1e9372bc57a9961e, 0x3764a9d00f676820, 0x3fc4f1901abfa020, 0x355481020e132730, 0x21ec9d424957adb0, 0x3c43c32c0f34301e, 0x3d67cb6c1fe67c3e, 0x1496122c45259728, 0x1cb35a6e52f5db0e, 0x15e418405b72ec20, 0x35e79b6c0a66fc3e, 0x30268b6e3a445c38, 0x215481024c132730, 0x100010114, 0x200010110, 0, 0x1c324a6c52f55b08, 0x215581044d133776, 0x1e9372bc57ad961e, 0x2155810e4d133766, 0x3764a9d00f6f6820, 0x2157833c4d12323e, 0x3fc4f1901aafa020, 0x1c324a4252f55b58, 0x355481020e332730, 0x28332fc0509d41e, 0x21ec9d424917adb0, 0x215783be4d12332e, 0x3c43c32c0fb4301e, 0x2157822c4d06363e, 0x3d67cb6c1ee67c3e, 0x23f6b9d2484afb78, 0x1496122c47259728, 0x14b8184047648a80, 0x1cb35a6e56f5db0e, 0x3fe4f1901aefa820, 0x15e418405372ec20, 0x3d5fd72c1be276be, 0x35e79b6c1a66fc3e, 0x14b038d24774cf10, 0x30268b6e1a445c38, 0x1d17022e43a7172e, 0x215481020c132730, 0x2157022e4d07372e}); +typedef Field<uint64_t, 62, 536870913, StatTable62, &SQR_TABLE_62, &SQR2_TABLE_62, &SQR4_TABLE_62, &SQR8_TABLE_62, &SQR16_TABLE_62, &QRT_TABLE_62, IdTrans, &ID_TRANS, &ID_TRANS> Field62; +typedef FieldTri<uint64_t, 62, 29, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5>, &SQR_TABLE_62, &SQR2_TABLE_62, &SQR4_TABLE_62, &SQR8_TABLE_62, &SQR16_TABLE_62, &QRT_TABLE_62, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri62; +#endif + +#ifdef ENABLE_FIELD_INT_63 +// 63 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTableTRI63; +constexpr StatTableTRI63 SQR_TABLE_TRI63({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4000000000000000, 0x6, 0x18, 0x60, 0x180, 0x600, 0x1800, 0x6000, 0x18000, 0x60000, 0x180000, 0x600000, 0x1800000, 0x6000000, 0x18000000, 0x60000000, 0x180000000, 0x600000000, 0x1800000000, 0x6000000000, 0x18000000000, 0x60000000000, 0x180000000000, 0x600000000000, 0x1800000000000, 0x6000000000000, 0x18000000000000, 0x60000000000000, 0x180000000000000, 0x600000000000000, 0x1800000000000000, 0x6000000000000000}); +constexpr StatTableTRI63 SQR2_TABLE_TRI63({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x1000000000000000, 0x6, 0x60, 0x600, 0x6000, 0x60000, 0x600000, 0x6000000, 0x60000000, 0x600000000, 0x6000000000, 0x60000000000, 0x600000000000, 0x6000000000000, 0x60000000000000, 0x600000000000000, 0x6000000000000000, 0x14, 0x140, 0x1400, 0x14000, 0x140000, 0x1400000, 0x14000000, 0x140000000, 0x1400000000, 0x14000000000, 0x140000000000, 0x1400000000000, 0x14000000000000, 0x140000000000000, 0x1400000000000000, 0x4000000000000006, 0x78, 0x780, 0x7800, 0x78000, 0x780000, 0x7800000, 0x78000000, 0x780000000, 0x7800000000, 0x78000000000, 0x780000000000, 0x7800000000000, 0x78000000000000, 0x780000000000000, 0x7800000000000000}); +constexpr StatTableTRI63 SQR4_TABLE_TRI63({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x6, 0x60000, 0x600000000, 0x6000000000000, 0x14, 0x140000, 0x1400000000, 0x14000000000000, 0x78, 0x780000, 0x7800000000, 0x78000000000000, 0x110, 0x1100000, 0x11000000000, 0x110000000000000, 0x660, 0x6600000, 0x66000000000, 0x660000000000000, 0x1540, 0x15400000, 0x154000000000, 0x1540000000000000, 0x7f80, 0x7f800000, 0x7f8000000000, 0x7f80000000000000, 0x10100, 0x101000000, 0x1010000000000, 0x100000000000006, 0x60600, 0x606000000, 0x6060000000000, 0x600000000000014, 0x141400, 0x1414000000, 0x14140000000000, 0x1400000000000078, 0x787800, 0x7878000000, 0x78780000000000, 0x7800000000000110, 0x1111000, 0x11110000000, 0x111100000000000, 0x1000000000000666, 0x6666000, 0x66660000000, 0x666600000000000, 0x6000000000001554, 0x15554000, 0x155540000000, 0x1555400000000000, 0x4000000000007ffe, 0x7fff8000, 0x7fff80000000, 0x7fff800000000000}); +constexpr StatTableTRI63 SQR8_TABLE_TRI63({0x1, 0x110, 0x10100, 0x1111000, 0x100010000, 0x11001100000, 0x1010101000000, 0x111111110000000, 0x100000006, 0x11000000660, 0x1010000060600, 0x111100006666000, 0x1000600060006, 0x110066006600660, 0x106060606060606, 0x1666666666666666, 0x12, 0x1320, 0x121200, 0x13332000, 0x1200120000, 0x132013200000, 0x12121212000000, 0x1333333320000000, 0x120000006c, 0x132000006ac0, 0x121200006c6c00, 0x133320006aaac000, 0x12006c006c006c, 0x13206ac06ac06ac0, 0x126c6c6c6c6c6c6c, 0x4aaaaaaaaaaaaaaa, 0x104, 0x11440, 0x1050400, 0x115544000, 0x10401040000, 0x1144114400000, 0x105050504000000, 0x1555555440000006, 0x10400000618, 0x1144000067980, 0x1050400061e1800, 0x155440067ff98006, 0x104061806180618, 0x1446798679867986, 0x21e1e1e1e1e1e1e, 0x3fffffffffffffec, 0x1248, 0x136c80, 0x125a4800, 0x137fec8000, 0x124812480000, 0x136c936c800000, 0x125a5a5a48000000, 0x7fffffec8000006a, 0x124800006db0, 0x136c80006b6b00, 0x125a48006dddb000, 0x7fec806b006b006a, 0x12486db06db06db0, 0x6ceb6b6b6b6b6b6a, 0x25dddddddddddddc}); +constexpr StatTableTRI63 SQR16_TABLE_TRI63({0x1, 0x10006, 0x100000014, 0x1000600140078, 0x116, 0x1160674, 0x11600001538, 0x116067415387e90, 0x10114, 0x101120678, 0x1011400141510, 0x112066c15687e66, 0x1170338, 0x117054a0a90, 0x1170338152c3f60, 0x54a1fbc41888532, 0x100010110, 0x1000701160660, 0x1010400141546, 0x102060c153e7f92, 0x11601170760, 0x116076301121340, 0x1171258152c6df4, 0x142a78fc131d6a4a, 0x1011500050540, 0x113067b055e1f86, 0x1110440042477e, 0x102261da46f39362, 0x117022e054b0b80, 0x45c09af143a3f72, 0x106721d847ee9ae4, 0x408a833f0a833f0a, 0x100010106, 0x1000701000614, 0x101120014147e, 0x114067814067902, 0x11601171074, 0x116076316066138, 0x117054c152d40e4, 0x33e0a853e0b842a, 0x1011500131278, 0x113066d12126d16, 0x7077c017b681e, 0x76e12736f057056, 0x117022e12493290, 0x45c1ead5f26a912, 0x76518c96bc5efa4, 0xb97397297387286, 0x1700171666, 0x17006516147554, 0x17174a012d3f8a, 0x173872913964814e, 0x160216157534, 0x16026219014b3eb8, 0x16144d1d3902f39c, 0x3964974c65925d30, 0x17163b005d59f8, 0x164974c75837d462, 0x17062a404d28cfa, 0x65854b0a96152d3c, 0x16152c2a5943b390, 0x5854b1be6419dd1e, 0x6045c19c854b1fba}); +constexpr StatTableTRI63 QRT_TABLE_TRI63({0, 0x100010114, 0x100010116, 0x1001701051372, 0x100010112, 0x1000040220, 0x100170105137a, 0x5107703453bba, 0x100010102, 0x101130117155a, 0x1000040200, 0x40000200800, 0x100170105133a, 0x103151a137276d8, 0x5107703453b3a, 0x134e65fc7c222be0, 0x100010002, 0x100030103115a, 0x101130117175a, 0x106052d103f4de2, 0x1000040600, 0x15122707691d3a, 0x40000200000, 0x4530770bc57b3a, 0x100170105033a, 0x103011a131256d8, 0x103151a137256d8, 0x176f29eb55c7a8da, 0x5107703457b3a, 0x130b158b7767d0da, 0x134e65fc7c22abe0, 0x7bcaf59d2f62d3e2, 0x100000002, 0x1001401041260, 0x100030101115a, 0x5107e03443ab8, 0x101130113175a, 0x1043701251b3a, 0x106052d10374de2, 0x134e657d7c232be2, 0x1000140600, 0x106073d103b4be2, 0x15122707491d3a, 0x4438600ac07800, 0x40000600000, 0x176a199c5682d3e0, 0x4530770b457b3a, 0x7bca759c2f62d3e0, 0x100170005033a, 0x6116d02572de2, 0x103011a111256d8, 0x1346656d7c372de2, 0x103151a177256d8, 0x643c600aa07800, 0x176f29eb5dc7a8da, 0x7b4b758b2f67d0da, 0x5107713457b3a, 0x104570776b457b3a, 0x130b158b5767d0da, 0x734e65fc3c22abe0, 0x134e65fc3c22abe0, 0x4000000000000000, 0x7bcaf59daf62d3e2}); +typedef FieldTri<uint64_t, 63, 1, StatTableTRI63, &SQR_TABLE_TRI63, &SQR2_TABLE_TRI63, &SQR4_TABLE_TRI63, &SQR8_TABLE_TRI63, &SQR16_TABLE_TRI63, &QRT_TABLE_TRI63, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri63; +#endif + +#ifdef ENABLE_FIELD_INT_64 +// 64 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable64; +constexpr StatTable64 SQR_TABLE_64({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4000000000000000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0x1b00000, 0x6c00000, 0x1b000000, 0x6c000000, 0x1b0000000, 0x6c0000000, 0x1b00000000, 0x6c00000000, 0x1b000000000, 0x6c000000000, 0x1b0000000000, 0x6c0000000000, 0x1b00000000000, 0x6c00000000000, 0x1b000000000000, 0x6c000000000000, 0x1b0000000000000, 0x6c0000000000000, 0x1b00000000000000, 0x6c00000000000000, 0xb00000000000001b, 0xc00000000000005a}); +constexpr StatTable64 SQR2_TABLE_64({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x1000000000000000, 0x1b, 0x1b0, 0x1b00, 0x1b000, 0x1b0000, 0x1b00000, 0x1b000000, 0x1b0000000, 0x1b00000000, 0x1b000000000, 0x1b0000000000, 0x1b00000000000, 0x1b000000000000, 0x1b0000000000000, 0x1b00000000000000, 0xb00000000000001b, 0x145, 0x1450, 0x14500, 0x145000, 0x1450000, 0x14500000, 0x145000000, 0x1450000000, 0x14500000000, 0x145000000000, 0x1450000000000, 0x14500000000000, 0x145000000000000, 0x1450000000000000, 0x450000000000001b, 0x50000000000001dc, 0x1db7, 0x1db70, 0x1db700, 0x1db7000, 0x1db70000, 0x1db700000, 0x1db7000000, 0x1db70000000, 0x1db700000000, 0x1db7000000000, 0x1db70000000000, 0x1db700000000000, 0x1db7000000000000, 0xdb7000000000001b, 0xb70000000000011f, 0x7000000000001105}); +constexpr StatTable64 SQR4_TABLE_64({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x1b, 0x1b0000, 0x1b00000000, 0x1b000000000000, 0x145, 0x1450000, 0x14500000000, 0x145000000000000, 0x1db7, 0x1db70000, 0x1db700000000, 0x1db7000000000000, 0x11011, 0x110110000, 0x1101100000000, 0x101100000000001b, 0x1ab1ab, 0x1ab1ab0000, 0x1ab1ab00000000, 0xb1ab00000000015e, 0x1514515, 0x15145150000, 0x151451500000000, 0x4515000000001c6b, 0x1c6db6c7, 0x1c6db6c70000, 0x1c6db6c700000000, 0xb6c700000001010f, 0x101000101, 0x1010001010000, 0x10001010000001b, 0x1010000001b1b00, 0x1b1b001b1b, 0x1b1b001b1b0000, 0x1b001b1b00000145, 0x1b1b000001444500, 0x14445014445, 0x144450144450000, 0x4501444500001dac, 0x444500001daab71b, 0x1daab71daab7, 0x1daab71daab70000, 0xb71daab70001110e, 0xaab700011101101f, 0x1110110110111, 0x110110110111001b, 0x10110111001aab1b, 0x111001aab1ab1ab, 0x1aab1ab1ab1aab, 0xab1ab1ab1aab015e, 0xb1ab1aab0150145e, 0x1aab015014514515, 0x150145145145015, 0x1451451450151c70, 0x451450151c71db6b, 0x50151c71db6db6dc, 0x1c71db6db6db71c7, 0xdb6db6db71c6000b, 0xb6db71c60001000f, 0x71c6000100000005}); +constexpr StatTable64 SQR8_TABLE_64({0x1, 0x11011, 0x101000101, 0x1110110110111, 0x100000001001a, 0x10110001100aa1a1, 0x100011a1b1a011a, 0x100baa100bb1aa0a, 0x1a00000144, 0x1ba1ba01505504, 0x1a001b5f4401441a, 0xa0eb1eea544fee41, 0x15e0144001a1ce8, 0xf5ee551fbc9d4f5d, 0x1b4543b0eee81b44, 0xb89a98b89a98b894, 0x10dbc, 0x11d76167c, 0x10cb1bd0cb1bc, 0x1c6b617617606a67, 0xdbc00010da6ad43, 0x167d1d6d105be392, 0xbd170ae2484f0af7, 0x162bc80d36e8d468, 0x1aad58014ae5f0, 0x63df9865e4bbbb5, 0x43fc5a4cbafe0d17, 0xe3d18fd6f8de2666, 0x49e2e5eab134a710, 0x1c78a1664f19bdd8, 0xf0829cea9886f08a, 0x4d8f634d8f625cdd, 0x100514550, 0x1104554401050, 0x15115140114154b, 0x10050551444aec57, 0x4551004b4277f24b, 0xef2afe861bdfb, 0x1d64ceb6c85ed2c9, 0x4975810172576524, 0x73cf4644451101e, 0x4fd1b234005fb6a7, 0x1bddd12e486f9a6f, 0xaa3c6f23ad5e9724, 0xa02b0a9206ef4923, 0x18a08533d5a4e65e, 0x1fc83ef027d0132b, 0x5e54f45f48c9a13c, 0x10deeff7bf8c0, 0x1d21c38d4f8874db, 0x10886029449884cd, 0xfe25b26c0190be86, 0xf5345525adfcb67e, 0xb606f05c0f274ae6, 0x49303a49c3147e89, 0xe3dec1f0cb3467b8, 0xf3dd197b59b91bb7, 0x6e062ec482dfc7e, 0xc24c087e94b8c9c, 0x42e75f2649a63926, 0x4646807e89775aa9, 0xca57e67631079503, 0xf738d302cd26e621, 0xda8702da9702da9d}); +constexpr StatTable64 SQR16_TABLE_64({0x1, 0x15f0144001a114f, 0x1aad43011ba1e5, 0xe34916e80106e21d, 0x11cefef6be466, 0xab943b855d3d776b, 0x1c77b6cf4edf1bd0, 0x46923ddea5ce4e34, 0x5455145e48670f13, 0xfb7d34d8e2b804bb, 0xbbe0dfe164a4d5b4, 0x431d528b1f73a8a2, 0xc259794b79e2607, 0x5945c54c76a8d132, 0xf5cb8b3860386917, 0xb345180ffd7a5551, 0xbaf1bebe1ae4ad02, 0x45562dad588c6260, 0x55b2852b76a728c4, 0xb5908b73d457d739, 0xa5a058173d115951, 0x11e605f10dd49e16, 0xb122096fef2a82a8, 0xfb95933559736ac7, 0x42652cf9ded5daa5, 0xe9a56590d5ab5301, 0xb8cef5ec20abb26f, 0xb50edcd1421d92e0, 0x12ac73f1d2f67094, 0x1c5815d4c184bd2, 0xe227a4ef0cd1165c, 0xe8d4a3a319b07491, 0xb0ef530df44bb042, 0xfbcbf52ff08d7ea3, 0xa0eaea8c7f69bf70, 0xedc22185164a14b1, 0xbfb9f37fc5eb3abc, 0x3712083e323193a, 0xe7bdca1397a3c26c, 0xf2d44dcbd1d02306, 0xa8fcad00bc810b9c, 0x4f7014f9d2186ea, 0x1b4d4ccc40f8060f, 0xe9ecf1e0105dab78, 0xe34e682846de9f1d, 0xace6cd21bf5ef658, 0x10f0cfa8cf3326ff, 0x71a97b1c73b8a63, 0xe1398cba3a3345d1, 0xa439e4c62ecb0615, 0x4bcce9efcca8db40, 0x176e95394759914e, 0xb5c7335e43a80f7f, 0xeb5439d8e177d64d, 0xa6af064a2d733f41, 0x5efc52c7e2f99007, 0x4a6efe65d270460b, 0xfe0ff44f5baa9a6a, 0x104c70edd05ffd6f, 0xf07d029f554aa763, 0x1c3c3cc0aca30a16, 0x7a0a5f6c85237d50, 0x1b862fb6b961ed37, 0xdcd1bd32f8a7d3ba}); +constexpr StatTable64 QRT_TABLE_64({0x19c9369f278adc02, 0x84b2b22ab2383ee4, 0x84b2b22ab2383ee6, 0x9d7b84b495b3e3f6, 0x84b2b22ab2383ee2, 0x37c470b49213f790, 0x9d7b84b495b3e3fe, 0x1000a0105137c, 0x84b2b22ab2383ef2, 0x368e964a8edce1fc, 0x37c470b49213f7b0, 0x19c9368e278fdf4c, 0x9d7b84b495b3e3be, 0x2e4da23cbc7d4570, 0x1000a010513fc, 0x84f35772bac24232, 0x84b2b22ab2383ff2, 0x37c570ba9314e4fc, 0x368e964a8edce3fc, 0xb377c390213cdb0e, 0x37c470b49213f3b0, 0x85ed5a3aa99c24f2, 0x19c9368e278fd74c, 0xaabff0000780000e, 0x9d7b84b495b3f3be, 0x84b6b3dab03038f2, 0x2e4da23cbc7d6570, 0x511ea03494ffc, 0x1000a010553fc, 0xae0c0220343c6c0e, 0x84f35772bac2c232, 0x800000008000000e, 0x84b2b22ab2393ff2, 0xb376c29c202bc97e, 0x37c570ba9316e4fc, 0x9c3062488879e6ce, 0x368e964a8ed8e3fc, 0x41e42c08e47e70, 0xb377c3902134db0e, 0x85b9b108a60f56ce, 0x37c470b49203f3b0, 0x19dd3b6e21f3cb4c, 0x85ed5a3aa9bc24f2, 0x198ddf682c428ac0, 0x19c9368e27cfd74c, 0x4b7c68431ca84b0, 0xaabff0000700000e, 0x8040655489ffefbe, 0x9d7b84b494b3f3be, 0x18c1354e32bfa74c, 0x84b6b3dab23038f2, 0xaaf613cc0f74627e, 0x2e4da23cb87d6570, 0x3248b3d6b3342a8c, 0x511ea0b494ffc, 0xb60813c00e70700e, 0x1000a110553fc, 0x1e0d022a05393ffc, 0xae0c0220143c6c0e, 0xe0c0220143c6c00, 0x84f35772fac2c232, 0xc041e55948fbfdce, 0x800000000000000e, 0}); +typedef Field<uint64_t, 64, 27, StatTable64, &SQR_TABLE_64, &SQR2_TABLE_64, &SQR4_TABLE_64, &SQR8_TABLE_64, &SQR16_TABLE_64, &QRT_TABLE_64, IdTrans, &ID_TRANS, &ID_TRANS> Field64; +#endif +} + +Sketch* ConstructClMul8Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_57 + case 57: return new SketchImpl<Field57>(implementation, 57); +#endif +#ifdef ENABLE_FIELD_INT_58 + case 58: return new SketchImpl<Field58>(implementation, 58); +#endif +#ifdef ENABLE_FIELD_INT_59 + case 59: return new SketchImpl<Field59>(implementation, 59); +#endif +#ifdef ENABLE_FIELD_INT_61 + case 61: return new SketchImpl<Field61>(implementation, 61); +#endif +#ifdef ENABLE_FIELD_INT_62 + case 62: return new SketchImpl<Field62>(implementation, 62); +#endif +#ifdef ENABLE_FIELD_INT_64 + case 64: return new SketchImpl<Field64>(implementation, 64); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri8Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_57 + case 57: return new SketchImpl<FieldTri57>(implementation, 57); +#endif +#ifdef ENABLE_FIELD_INT_58 + case 58: return new SketchImpl<FieldTri58>(implementation, 58); +#endif +#ifdef ENABLE_FIELD_INT_60 + case 60: return new SketchImpl<FieldTri60>(implementation, 60); +#endif +#ifdef ENABLE_FIELD_INT_62 + case 62: return new SketchImpl<FieldTri62>(implementation, 62); +#endif +#ifdef ENABLE_FIELD_INT_63 + case 63: return new SketchImpl<FieldTri63>(implementation, 63); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_common_impl.h b/src/minisketch/src/fields/clmul_common_impl.h new file mode 100644 index 0000000000..3d179a1081 --- /dev/null +++ b/src/minisketch/src/fields/clmul_common_impl.h @@ -0,0 +1,170 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_FIELDS_CLMUL_COMMON_IMPL_H_ +#define _MINISKETCH_FIELDS_CLMUL_COMMON_IMPL_H_ 1 + +#include <stdint.h> +#include <immintrin.h> + +#include "../int_utils.h" +#include "../lintrans.h" + +namespace { + +// The memory sanitizer in clang < 11 cannot reason through _mm_clmulepi64_si128 calls. +// Disable memory sanitization in the functions using them for those compilers. +#if defined(__clang__) && (__clang_major__ < 11) +# if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# define NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) +# endif +# endif +#endif +#ifndef NO_SANITIZE_MEMORY +# define NO_SANITIZE_MEMORY +#endif + +template<typename I, int BITS, I MOD> NO_SANITIZE_MEMORY I MulWithClMulReduce(I a, I b) +{ + static constexpr I MASK = Mask<BITS, I>(); + + const __m128i MOD128 = _mm_cvtsi64_si128(MOD); + __m128i product = _mm_clmulepi64_si128(_mm_cvtsi64_si128((uint64_t)a), _mm_cvtsi64_si128((uint64_t)b), 0x00); + if (BITS <= 32) { + __m128i high1 = _mm_srli_epi64(product, BITS); + __m128i red1 = _mm_clmulepi64_si128(high1, MOD128, 0x00); + __m128i high2 = _mm_srli_epi64(red1, BITS); + __m128i red2 = _mm_clmulepi64_si128(high2, MOD128, 0x00); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } else if (BITS == 64) { + __m128i red1 = _mm_clmulepi64_si128(product, MOD128, 0x01); + __m128i red2 = _mm_clmulepi64_si128(red1, MOD128, 0x01); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)); + } else if ((BITS % 8) == 0) { + __m128i high1 = _mm_srli_si128(product, BITS / 8); + __m128i red1 = _mm_clmulepi64_si128(high1, MOD128, 0x00); + __m128i high2 = _mm_srli_si128(red1, BITS / 8); + __m128i red2 = _mm_clmulepi64_si128(high2, MOD128, 0x00); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } else { + __m128i high1 = _mm_or_si128(_mm_srli_epi64(product, BITS), _mm_srli_si128(_mm_slli_epi64(product, 64 - BITS), 8)); + __m128i red1 = _mm_clmulepi64_si128(high1, MOD128, 0x00); + if ((uint64_t(MOD) >> (66 - BITS)) == 0) { + __m128i high2 = _mm_srli_epi64(red1, BITS); + __m128i red2 = _mm_clmulepi64_si128(high2, MOD128, 0x00); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } else { + __m128i high2 = _mm_or_si128(_mm_srli_epi64(red1, BITS), _mm_srli_si128(_mm_slli_epi64(red1, 64 - BITS), 8)); + __m128i red2 = _mm_clmulepi64_si128(high2, MOD128, 0x00); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } + } +} + +template<typename I, int BITS, int POS> NO_SANITIZE_MEMORY I MulTrinomial(I a, I b) +{ + static constexpr I MASK = Mask<BITS, I>(); + + __m128i product = _mm_clmulepi64_si128(_mm_cvtsi64_si128((uint64_t)a), _mm_cvtsi64_si128((uint64_t)b), 0x00); + if (BITS <= 32) { + __m128i high1 = _mm_srli_epi64(product, BITS); + __m128i red1 = _mm_xor_si128(high1, _mm_slli_epi64(high1, POS)); + if (POS == 1) { + return _mm_cvtsi128_si64(_mm_xor_si128(product, red1)) & MASK; + } else { + __m128i high2 = _mm_srli_epi64(red1, BITS); + __m128i red2 = _mm_xor_si128(high2, _mm_slli_epi64(high2, POS)); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } + } else { + __m128i high1 = _mm_or_si128(_mm_srli_epi64(product, BITS), _mm_srli_si128(_mm_slli_epi64(product, 64 - BITS), 8)); + if (BITS + POS <= 66) { + __m128i red1 = _mm_xor_si128(high1, _mm_slli_epi64(high1, POS)); + if (POS == 1) { + return _mm_cvtsi128_si64(_mm_xor_si128(product, red1)) & MASK; + } else if (BITS + POS <= 66) { + __m128i high2 = _mm_srli_epi64(red1, BITS); + __m128i red2 = _mm_xor_si128(high2, _mm_slli_epi64(high2, POS)); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } + } else { + const __m128i MOD128 = _mm_cvtsi64_si128(1 + (((uint64_t)1) << POS)); + __m128i red1 = _mm_clmulepi64_si128(high1, MOD128, 0x00); + __m128i high2 = _mm_or_si128(_mm_srli_epi64(red1, BITS), _mm_srli_si128(_mm_slli_epi64(red1, 64 - BITS), 8)); + __m128i red2 = _mm_xor_si128(high2, _mm_slli_epi64(high2, POS)); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } + } +} + +/** Implementation of fields that use the SSE clmul intrinsic for multiplication. */ +template<typename I, int B, I MOD, I (*MUL)(I, I), typename F, const F* SQR, const F* SQR2, const F* SQR4, const F* SQR8, const F* SQR16, const F* QRT, typename T, const T* LOAD, const T* SAVE> struct GenField +{ + typedef BitsInt<I, B> O; + typedef LFSR<O, MOD> L; + + static inline constexpr I Sqr1(I a) { return SQR->template Map<O>(a); } + static inline constexpr I Sqr2(I a) { return SQR2->template Map<O>(a); } + static inline constexpr I Sqr4(I a) { return SQR4->template Map<O>(a); } + static inline constexpr I Sqr8(I a) { return SQR8->template Map<O>(a); } + static inline constexpr I Sqr16(I a) { return SQR16->template Map<O>(a); } + +public: + typedef I Elem; + + inline constexpr int Bits() const { return B; } + + inline constexpr Elem Mul2(Elem val) const { return L::Call(val); } + + inline Elem Mul(Elem a, Elem b) const { return MUL(a, b); } + + class Multiplier + { + Elem m_val; + public: + inline constexpr explicit Multiplier(const GenField&, Elem a) : m_val(a) {} + constexpr Elem operator()(Elem a) const { return MUL(m_val, a); } + }; + + /** Compute the square of a. */ + inline constexpr Elem Sqr(Elem val) const { return SQR->template Map<O>(val); } + + /** Compute x such that x^2 + x = a (undefined result if no solution exists). */ + inline constexpr Elem Qrt(Elem val) const { return QRT->template Map<O>(val); } + + /** Compute the inverse of x1. */ + inline Elem Inv(Elem val) const { return InvLadder<I, O, B, MUL, Sqr1, Sqr2, Sqr4, Sqr8, Sqr16>(val); } + + /** Generate a random field element. */ + Elem FromSeed(uint64_t seed) const { + uint64_t k0 = 0x434c4d554c466c64ull; // "CLMULFld" + uint64_t k1 = seed; + uint64_t count = ((uint64_t)B) << 32; + I ret; + do { + ret = O::Mask(I(SipHash(k0, k1, count++))); + } while(ret == 0); + return LOAD->template Map<O>(ret); + } + + Elem Deserialize(BitReader& in) const { return LOAD->template Map<O>(in.Read<B, I>()); } + + void Serialize(BitWriter& out, Elem val) const { out.Write<B, I>(SAVE->template Map<O>(val)); } + + constexpr Elem FromUint64(uint64_t x) const { return LOAD->template Map<O>(O::Mask(I(x))); } + constexpr uint64_t ToUint64(Elem val) const { return uint64_t(SAVE->template Map<O>(val)); } +}; + +template<typename I, int B, I MOD, typename F, const F* SQR, const F* SQR2, const F* SQR4, const F* SQR8, const F* SQR16, const F* QRT, typename T, const T* LOAD, const T* SAVE> +using Field = GenField<I, B, MOD, MulWithClMulReduce<I, B, MOD>, F, SQR, SQR2, SQR4, SQR8, SQR16, QRT, T, LOAD, SAVE>; + +template<typename I, int B, int POS, typename F, const F* SQR, const F* SQR2, const F* SQR4, const F* SQR8, const F* SQR16, const F* QRT, typename T, const T* LOAD, const T* SAVE> +using FieldTri = GenField<I, B, I(1) + (I(1) << POS), MulTrinomial<I, B, POS>, F, SQR, SQR2, SQR4, SQR8, SQR16, QRT, T, LOAD, SAVE>; + +} + +#endif diff --git a/src/minisketch/src/fields/generic_1byte.cpp b/src/minisketch/src/fields/generic_1byte.cpp new file mode 100644 index 0000000000..5ce42dc5f7 --- /dev/null +++ b/src/minisketch/src/fields/generic_1byte.cpp @@ -0,0 +1,112 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_1) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_2 +// 2 bit field +typedef RecLinTrans<uint8_t, 2> StatTable2; +typedef RecLinTrans<uint8_t, 2> DynTable2; +constexpr StatTable2 SQR_TABLE_2({0x1, 0x3}); +constexpr StatTable2 QRT_TABLE_2({0x2, 0}); +typedef Field<uint8_t, 2, 3, StatTable2, DynTable2, &SQR_TABLE_2, &QRT_TABLE_2> Field2; +#endif + +#ifdef ENABLE_FIELD_INT_3 +// 3 bit field +typedef RecLinTrans<uint8_t, 3> StatTable3; +typedef RecLinTrans<uint8_t, 3> DynTable3; +constexpr StatTable3 SQR_TABLE_3({0x1, 0x4, 0x6}); +constexpr StatTable3 QRT_TABLE_3({0, 0x4, 0x6}); +typedef Field<uint8_t, 3, 3, StatTable3, DynTable3, &SQR_TABLE_3, &QRT_TABLE_3> Field3; +#endif + +#ifdef ENABLE_FIELD_INT_4 +// 4 bit field +typedef RecLinTrans<uint8_t, 4> StatTable4; +typedef RecLinTrans<uint8_t, 4> DynTable4; +constexpr StatTable4 SQR_TABLE_4({0x1, 0x4, 0x3, 0xc}); +constexpr StatTable4 QRT_TABLE_4({0x6, 0xa, 0x8, 0}); +typedef Field<uint8_t, 4, 3, StatTable4, DynTable4, &SQR_TABLE_4, &QRT_TABLE_4> Field4; +#endif + +#ifdef ENABLE_FIELD_INT_5 +// 5 bit field +typedef RecLinTrans<uint8_t, 5> StatTable5; +typedef RecLinTrans<uint8_t, 3, 2> DynTable5; +constexpr StatTable5 SQR_TABLE_5({0x1, 0x4, 0x10, 0xa, 0xd}); +constexpr StatTable5 QRT_TABLE_5({0x14, 0x8, 0xa, 0, 0xe}); +typedef Field<uint8_t, 5, 5, StatTable5, DynTable5, &SQR_TABLE_5, &QRT_TABLE_5> Field5; +#endif + +#ifdef ENABLE_FIELD_INT_6 +// 6 bit field +typedef RecLinTrans<uint8_t, 6> StatTable6; +typedef RecLinTrans<uint8_t, 3, 3> DynTable6; +constexpr StatTable6 SQR_TABLE_6({0x1, 0x4, 0x10, 0x3, 0xc, 0x30}); +constexpr StatTable6 QRT_TABLE_6({0x3a, 0x26, 0x24, 0x14, 0x20, 0}); +typedef Field<uint8_t, 6, 3, StatTable6, DynTable6, &SQR_TABLE_6, &QRT_TABLE_6> Field6; +#endif + +#ifdef ENABLE_FIELD_INT_7 +// 7 bit field +typedef RecLinTrans<uint8_t, 4, 3> StatTable7; +typedef RecLinTrans<uint8_t, 4, 3> DynTable7; +constexpr StatTable7 SQR_TABLE_7({0x1, 0x4, 0x10, 0x40, 0x6, 0x18, 0x60}); +constexpr StatTable7 QRT_TABLE_7({0, 0x14, 0x16, 0x72, 0x12, 0x40, 0x7a}); +typedef Field<uint8_t, 7, 3, StatTable7, DynTable7, &SQR_TABLE_7, &QRT_TABLE_7> Field7; +#endif + +#ifdef ENABLE_FIELD_INT_8 +// 8 bit field +typedef RecLinTrans<uint8_t, 4, 4> StatTable8; +typedef RecLinTrans<uint8_t, 4, 4> DynTable8; +constexpr StatTable8 SQR_TABLE_8({0x1, 0x4, 0x10, 0x40, 0x1b, 0x6c, 0xab, 0x9a}); +constexpr StatTable8 QRT_TABLE_8({0xbc, 0x2a, 0x28, 0x86, 0x2c, 0xde, 0x8e, 0}); +typedef Field<uint8_t, 8, 27, StatTable8, DynTable8, &SQR_TABLE_8, &QRT_TABLE_8> Field8; +#endif +} + +Sketch* ConstructGeneric1Byte(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_2 + case 2: return new SketchImpl<Field2>(implementation, 2); +#endif +#ifdef ENABLE_FIELD_INT_3 + case 3: return new SketchImpl<Field3>(implementation, 3); +#endif +#ifdef ENABLE_FIELD_INT_4 + case 4: return new SketchImpl<Field4>(implementation, 4); +#endif +#ifdef ENABLE_FIELD_INT_5 + case 5: return new SketchImpl<Field5>(implementation, 5); +#endif +#ifdef ENABLE_FIELD_INT_6 + case 6: return new SketchImpl<Field6>(implementation, 6); +#endif +#ifdef ENABLE_FIELD_INT_7 + case 7: return new SketchImpl<Field7>(implementation, 7); +#endif +#ifdef ENABLE_FIELD_INT_8 + case 8: return new SketchImpl<Field8>(implementation, 8); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_2bytes.cpp b/src/minisketch/src/fields/generic_2bytes.cpp new file mode 100644 index 0000000000..12bf3110a6 --- /dev/null +++ b/src/minisketch/src/fields/generic_2bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_2) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_9 +// 9 bit field +typedef RecLinTrans<uint16_t, 5, 4> StatTable9; +typedef RecLinTrans<uint16_t, 3, 3, 3> DynTable9; +constexpr StatTable9 SQR_TABLE_9({0x1, 0x4, 0x10, 0x40, 0x100, 0x6, 0x18, 0x60, 0x180}); +constexpr StatTable9 QRT_TABLE_9({0, 0x4e, 0x4c, 0x1aa, 0x48, 0x22, 0x1a2, 0x100, 0x58}); +typedef Field<uint16_t, 9, 3, StatTable9, DynTable9, &SQR_TABLE_9, &QRT_TABLE_9> Field9; +#endif + +#ifdef ENABLE_FIELD_INT_10 +// 10 bit field +typedef RecLinTrans<uint16_t, 5, 5> StatTable10; +typedef RecLinTrans<uint16_t, 4, 3, 3> DynTable10; +constexpr StatTable10 SQR_TABLE_10({0x1, 0x4, 0x10, 0x40, 0x100, 0x9, 0x24, 0x90, 0x240, 0x112}); +constexpr StatTable10 QRT_TABLE_10({0xec, 0x86, 0x84, 0x30e, 0x80, 0x3c2, 0x306, 0, 0x90, 0x296}); +typedef Field<uint16_t, 10, 9, StatTable10, DynTable10, &SQR_TABLE_10, &QRT_TABLE_10> Field10; +#endif + +#ifdef ENABLE_FIELD_INT_11 +// 11 bit field +typedef RecLinTrans<uint16_t, 6, 5> StatTable11; +typedef RecLinTrans<uint16_t, 4, 4, 3> DynTable11; +constexpr StatTable11 SQR_TABLE_11({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0xa, 0x28, 0xa0, 0x280, 0x205}); +constexpr StatTable11 QRT_TABLE_11({0x734, 0x48, 0x4a, 0x1de, 0x4e, 0x35e, 0x1d6, 0x200, 0x5e, 0, 0x37e}); +typedef Field<uint16_t, 11, 5, StatTable11, DynTable11, &SQR_TABLE_11, &QRT_TABLE_11> Field11; +#endif + +#ifdef ENABLE_FIELD_INT_12 +// 12 bit field +typedef RecLinTrans<uint16_t, 6, 6> StatTable12; +typedef RecLinTrans<uint16_t, 4, 4, 4> DynTable12; +constexpr StatTable12 SQR_TABLE_12({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x9, 0x24, 0x90, 0x240, 0x900, 0x412}); +constexpr StatTable12 QRT_TABLE_12({0x48, 0xc10, 0xc12, 0x208, 0xc16, 0xd82, 0x200, 0x110, 0xc06, 0, 0xda2, 0x5a4}); +typedef Field<uint16_t, 12, 9, StatTable12, DynTable12, &SQR_TABLE_12, &QRT_TABLE_12> Field12; +#endif + +#ifdef ENABLE_FIELD_INT_13 +// 13 bit field +typedef RecLinTrans<uint16_t, 5, 4, 4> StatTable13; +typedef RecLinTrans<uint16_t, 4, 3, 3, 3> DynTable13; +constexpr StatTable13 SQR_TABLE_13({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x36, 0xd8, 0x360, 0xd80, 0x161b, 0x185a}); +constexpr StatTable13 QRT_TABLE_13({0xcfc, 0x1500, 0x1502, 0x382, 0x1506, 0x149c, 0x38a, 0x118, 0x1516, 0, 0x14bc, 0x100e, 0x3ca}); +typedef Field<uint16_t, 13, 27, StatTable13, DynTable13, &SQR_TABLE_13, &QRT_TABLE_13> Field13; +#endif + +#ifdef ENABLE_FIELD_INT_14 +// 14 bit field +typedef RecLinTrans<uint16_t, 5, 5, 4> StatTable14; +typedef RecLinTrans<uint16_t, 4, 4, 3, 3> DynTable14; +constexpr StatTable14 SQR_TABLE_14({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x21, 0x84, 0x210, 0x840, 0x2100, 0x442, 0x1108}); +constexpr StatTable14 QRT_TABLE_14({0x13f2, 0x206, 0x204, 0x3e06, 0x200, 0x1266, 0x3e0e, 0x114, 0x210, 0, 0x1246, 0x2848, 0x3e4e, 0x2258}); +typedef Field<uint16_t, 14, 33, StatTable14, DynTable14, &SQR_TABLE_14, &QRT_TABLE_14> Field14; +#endif + +#ifdef ENABLE_FIELD_INT_15 +// 15 bit field +typedef RecLinTrans<uint16_t, 5, 5, 5> StatTable15; +typedef RecLinTrans<uint16_t, 4, 4, 4, 3> DynTable15; +constexpr StatTable15 SQR_TABLE_15({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x6, 0x18, 0x60, 0x180, 0x600, 0x1800, 0x6000}); +constexpr StatTable15 QRT_TABLE_15({0, 0x114, 0x116, 0x428, 0x112, 0x137a, 0x420, 0x6d62, 0x102, 0x73a, 0x135a, 0x6460, 0x460, 0x4000, 0x6de2}); +typedef Field<uint16_t, 15, 3, StatTable15, DynTable15, &SQR_TABLE_15, &QRT_TABLE_15> Field15; +#endif + +#ifdef ENABLE_FIELD_INT_16 +// 16 bit field +typedef RecLinTrans<uint16_t, 6, 5, 5> StatTable16; +typedef RecLinTrans<uint16_t, 4, 4, 4, 4> DynTable16; +constexpr StatTable16 SQR_TABLE_16({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x2b, 0xac, 0x2b0, 0xac0, 0x2b00, 0xac00, 0xb056, 0xc10e}); +constexpr StatTable16 QRT_TABLE_16({0x732, 0x72b8, 0x72ba, 0x7e96, 0x72be, 0x78b2, 0x7e9e, 0x8cba, 0x72ae, 0xfa24, 0x7892, 0x5892, 0x7ede, 0xbec6, 0x8c3a, 0}); +typedef Field<uint16_t, 16, 43, StatTable16, DynTable16, &SQR_TABLE_16, &QRT_TABLE_16> Field16; +#endif +} + +Sketch* ConstructGeneric2Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_9 + case 9: return new SketchImpl<Field9>(implementation, 9); +#endif +#ifdef ENABLE_FIELD_INT_10 + case 10: return new SketchImpl<Field10>(implementation, 10); +#endif +#ifdef ENABLE_FIELD_INT_11 + case 11: return new SketchImpl<Field11>(implementation, 11); +#endif +#ifdef ENABLE_FIELD_INT_12 + case 12: return new SketchImpl<Field12>(implementation, 12); +#endif +#ifdef ENABLE_FIELD_INT_13 + case 13: return new SketchImpl<Field13>(implementation, 13); +#endif +#ifdef ENABLE_FIELD_INT_14 + case 14: return new SketchImpl<Field14>(implementation, 14); +#endif +#ifdef ENABLE_FIELD_INT_15 + case 15: return new SketchImpl<Field15>(implementation, 15); +#endif +#ifdef ENABLE_FIELD_INT_16 + case 16: return new SketchImpl<Field16>(implementation, 16); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_3bytes.cpp b/src/minisketch/src/fields/generic_3bytes.cpp new file mode 100644 index 0000000000..13e85bd1a1 --- /dev/null +++ b/src/minisketch/src/fields/generic_3bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_3) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_17 +// 17 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5> StatTable17; +typedef RecLinTrans<uint32_t, 4, 4, 3, 3, 3> DynTable17; +constexpr StatTable17 SQR_TABLE_17({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x8012}); +constexpr StatTable17 QRT_TABLE_17({0, 0x4c3e, 0x4c3c, 0x1a248, 0x4c38, 0x428, 0x1a240, 0x1b608, 0x4c28, 0x206, 0x408, 0x4000, 0x1a200, 0x18006, 0x1b688, 0x14d2e, 0x4d28}); +typedef Field<uint32_t, 17, 9, StatTable17, DynTable17, &SQR_TABLE_17, &QRT_TABLE_17> Field17; +#endif + +#ifdef ENABLE_FIELD_INT_18 +// 18 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6> StatTable18; +typedef RecLinTrans<uint32_t, 4, 4, 4, 3, 3> DynTable18; +constexpr StatTable18 SQR_TABLE_18({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x10012}); +constexpr StatTable18 QRT_TABLE_18({0x9208, 0x422, 0x420, 0x8048, 0x424, 0x68b0, 0x8040, 0x30086, 0x434, 0x1040, 0x6890, 0x30ca2, 0x8000, 0x32896, 0x30006, 0, 0x534, 0x20532}); +typedef Field<uint32_t, 18, 9, StatTable18, DynTable18, &SQR_TABLE_18, &QRT_TABLE_18> Field18; +#endif + +#ifdef ENABLE_FIELD_INT_19 +// 19 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 4> StatTable19; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 3> DynTable19; +constexpr StatTable19 SQR_TABLE_19({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x3804e, 0x6011f}); +constexpr StatTable19 QRT_TABLE_19({0x5d6b0, 0x2f476, 0x2f474, 0x1d6a2, 0x2f470, 0x42a, 0x1d6aa, 0x1060, 0x2f460, 0x19e92, 0x40a, 0x1da98, 0x1d6ea, 0x28c78, 0x10e0, 0xf56a, 0x2f560, 0, 0x19c92}); +typedef Field<uint32_t, 19, 39, StatTable19, DynTable19, &SQR_TABLE_19, &QRT_TABLE_19> Field19; +#endif + +#ifdef ENABLE_FIELD_INT_20 +// 20 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 5> StatTable20; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4> DynTable20; +constexpr StatTable20 SQR_TABLE_20({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x90000, 0x40012}); +constexpr StatTable20 QRT_TABLE_20({0xc5dea, 0xc0110, 0xc0112, 0xe11de, 0xc0116, 0x24814, 0xe11d6, 0x20080, 0xc0106, 0xfe872, 0x24834, 0xe4106, 0xe1196, 0x1d9a4, 0x20000, 0x31190, 0xc0006, 0, 0xfea72, 0x7ea74}); +typedef Field<uint32_t, 20, 9, StatTable20, DynTable20, &SQR_TABLE_20, &QRT_TABLE_20> Field20; +#endif + +#ifdef ENABLE_FIELD_INT_21 +// 21 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5> StatTable21; +typedef RecLinTrans<uint32_t, 4, 4, 4, 3, 3, 3> DynTable21; +constexpr StatTable21 SQR_TABLE_21({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x80005}); +constexpr StatTable21 QRT_TABLE_21({0x1bd5fc, 0xbc196, 0xbc194, 0x74b96, 0xbc190, 0x1048, 0x74b9e, 0x672c8, 0xbc180, 0x4080, 0x1068, 0xc8200, 0x74bde, 0x64280, 0x67248, 0xc4280, 0xbc080, 0x80000, 0x4280, 0, 0x1468}); +typedef Field<uint32_t, 21, 5, StatTable21, DynTable21, &SQR_TABLE_21, &QRT_TABLE_21> Field21; +#endif + +#ifdef ENABLE_FIELD_INT_22 +// 22 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5> StatTable22; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 3, 3> DynTable22; +constexpr StatTable22 SQR_TABLE_22({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000}); +constexpr StatTable22 QRT_TABLE_22({0x210d16, 0x104a, 0x1048, 0x4088, 0x104c, 0x200420, 0x4080, 0x492dc, 0x105c, 0x1a67f0, 0x200400, 0x21155c, 0x40c0, 0x20346c, 0x4925c, 0x1af7ac, 0x115c, 0x2274ac, 0x1a65f0, 0x2a65f0, 0x200000, 0}); +typedef Field<uint32_t, 22, 3, StatTable22, DynTable22, &SQR_TABLE_22, &QRT_TABLE_22> Field22; +#endif + +#ifdef ENABLE_FIELD_INT_23 +// 23 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 5> StatTable23; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 3> DynTable23; +constexpr StatTable23 SQR_TABLE_23({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x42, 0x108, 0x420, 0x1080, 0x4200, 0x10800, 0x42000, 0x108000, 0x420000, 0x80042, 0x200108}); +constexpr StatTable23 QRT_TABLE_23({0, 0x1040, 0x1042, 0x43056, 0x1046, 0x121d76, 0x4305e, 0x40a0, 0x1056, 0x15176, 0x121d56, 0x7ee1f6, 0x4301e, 0x40000, 0x4020, 0x4f0be, 0x1156, 0x7cf0a0, 0x15376, 0x1ee9e8, 0x121956, 0x3ac9f6, 0x7ee9f6}); +typedef Field<uint32_t, 23, 33, StatTable23, DynTable23, &SQR_TABLE_23, &QRT_TABLE_23> Field23; +#endif + +#ifdef ENABLE_FIELD_INT_24 +// 24 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6> StatTable24; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4> DynTable24; +constexpr StatTable24 SQR_TABLE_24({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0xb0001b, 0xc0005a}); +constexpr StatTable24 QRT_TABLE_24({0x104e, 0xaf42a8, 0xaf42aa, 0xb78186, 0xaf42ae, 0x4090, 0xb7818e, 0x4a37c, 0xaf42be, 0x3688c0, 0x40b0, 0x80080e, 0xb781ce, 0xaf2232, 0x4a3fc, 0x856a82, 0xaf43be, 0x29c970, 0x368ac0, 0x968ace, 0x44b0, 0x77d570, 0x80000e, 0}); +typedef Field<uint32_t, 24, 27, StatTable24, DynTable24, &SQR_TABLE_24, &QRT_TABLE_24> Field24; +#endif +} + +Sketch* ConstructGeneric3Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_17 + case 17: return new SketchImpl<Field17>(implementation, 17); +#endif +#ifdef ENABLE_FIELD_INT_18 + case 18: return new SketchImpl<Field18>(implementation, 18); +#endif +#ifdef ENABLE_FIELD_INT_19 + case 19: return new SketchImpl<Field19>(implementation, 19); +#endif +#ifdef ENABLE_FIELD_INT_20 + case 20: return new SketchImpl<Field20>(implementation, 20); +#endif +#ifdef ENABLE_FIELD_INT_21 + case 21: return new SketchImpl<Field21>(implementation, 21); +#endif +#ifdef ENABLE_FIELD_INT_22 + case 22: return new SketchImpl<Field22>(implementation, 22); +#endif +#ifdef ENABLE_FIELD_INT_23 + case 23: return new SketchImpl<Field23>(implementation, 23); +#endif +#ifdef ENABLE_FIELD_INT_24 + case 24: return new SketchImpl<Field24>(implementation, 24); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_4bytes.cpp b/src/minisketch/src/fields/generic_4bytes.cpp new file mode 100644 index 0000000000..2a26b90521 --- /dev/null +++ b/src/minisketch/src/fields/generic_4bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_4) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_25 +// 25 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 5, 5> StatTable25; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 3, 3, 3> DynTable25; +constexpr StatTable25 SQR_TABLE_25({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x800012}); +constexpr StatTable25 QRT_TABLE_25({0, 0x482110, 0x482112, 0x1b3c3e6, 0x482116, 0x4960ae, 0x1b3c3ee, 0x4088, 0x482106, 0x58a726, 0x49608e, 0x5ce52e, 0x1b3c3ae, 0x2006, 0x4008, 0x1c1a8, 0x482006, 0x1e96488, 0x58a526, 0x400000, 0x49648e, 0x1800006, 0x5ced2e, 0xb3d3a8, 0x1b3d3ae}); +typedef Field<uint32_t, 25, 9, StatTable25, DynTable25, &SQR_TABLE_25, &QRT_TABLE_25> Field25; +#endif + +#ifdef ENABLE_FIELD_INT_26 +// 26 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5, 5> StatTable26; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 3, 3> DynTable26; +constexpr StatTable26 SQR_TABLE_26({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0x1b00000, 0x2c0001b, 0x300005a}); +constexpr StatTable26 QRT_TABLE_26({0x217b530, 0x2ae82a8, 0x2ae82aa, 0x2001046, 0x2ae82ae, 0x2de032e, 0x200104e, 0x70c10c, 0x2ae82be, 0x20151f2, 0x2de030e, 0xbc1400, 0x200100e, 0x178570, 0x70c18c, 0x2ae4232, 0x2ae83be, 0x211d742, 0x20153f2, 0x21f54f2, 0x2de070e, 0x5e0700, 0xbc1c00, 0x3abb97e, 0x200000e, 0}); +typedef Field<uint32_t, 26, 27, StatTable26, DynTable26, &SQR_TABLE_26, &QRT_TABLE_26> Field26; +#endif + +#ifdef ENABLE_FIELD_INT_27 +// 27 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5, 5> StatTable27; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 3> DynTable27; +constexpr StatTable27 SQR_TABLE_27({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x138000, 0x4e0000, 0x1380000, 0x4e00000, 0x380004e, 0x600011f}); +constexpr StatTable27 QRT_TABLE_27({0x6bf0530, 0x2be4496, 0x2be4494, 0x2bf0522, 0x2be4490, 0x1896cca, 0x2bf052a, 0x408a, 0x2be4480, 0x368ae72, 0x1896cea, 0x18d2ee0, 0x2bf056a, 0x1c76d6a, 0x400a, 0x336e9f8, 0x2be4580, 0x36baf12, 0x368ac72, 0x430360, 0x18968ea, 0x34a6b80, 0x18d26e0, 0xbf1560, 0x2bf156a, 0, 0x1c74d6a}); +typedef Field<uint32_t, 27, 39, StatTable27, DynTable27, &SQR_TABLE_27, &QRT_TABLE_27> Field27; +#endif + +#ifdef ENABLE_FIELD_INT_28 +// 28 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 5, 5> StatTable28; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 4> DynTable28; +constexpr StatTable28 SQR_TABLE_28({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000}); +constexpr StatTable28 QRT_TABLE_28({0x121d57a, 0x40216, 0x40214, 0x8112578, 0x40210, 0x10110, 0x8112570, 0x12597ec, 0x40200, 0x6983e00, 0x10130, 0x972b99c, 0x8112530, 0x8002000, 0x125976c, 0x815a76c, 0x40300, 0x936b29c, 0x6983c00, 0x97bb8ac, 0x10530, 0x9103000, 0x972b19c, 0xf6384ac, 0x8113530, 0x4113530, 0x8000000, 0}); +typedef Field<uint32_t, 28, 3, StatTable28, DynTable28, &SQR_TABLE_28, &QRT_TABLE_28> Field28; +#endif + +#ifdef ENABLE_FIELD_INT_29 +// 29 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6, 5> StatTable29; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 3, 3, 3> DynTable29; +constexpr StatTable29 SQR_TABLE_29({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x280000, 0xa00000, 0x2800000, 0xa000000, 0x8000005}); +constexpr StatTable29 QRT_TABLE_29({0x1b8351dc, 0xb87135e, 0xb87135c, 0xda7b35e, 0xb871358, 0x621a116, 0xda7b356, 0x40200, 0xb871348, 0xc9e2620, 0x621a136, 0x478b16, 0xda7b316, 0x6762e20, 0x40280, 0x6202000, 0xb871248, 0x627a316, 0xc9e2420, 0xcd1ad36, 0x621a536, 0x760e20, 0x478316, 0xa760e20, 0xda7a316, 0x8000000, 0x6760e20, 0, 0x44280}); +typedef Field<uint32_t, 29, 5, StatTable29, DynTable29, &SQR_TABLE_29, &QRT_TABLE_29> Field29; +#endif + +#ifdef ENABLE_FIELD_INT_30 +// 30 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6, 6> StatTable30; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 3, 3> DynTable30; +constexpr StatTable30 SQR_TABLE_30({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000}); +constexpr StatTable30 QRT_TABLE_30({0x2159df4a, 0x109134a, 0x1091348, 0x10114, 0x109134c, 0x3a203420, 0x1011c, 0x20004080, 0x109135c, 0x2005439c, 0x3a203400, 0x100400, 0x1015c, 0x3eb21930, 0x20004000, 0x20504c00, 0x109125c, 0x3b2b276c, 0x2005419c, 0x210450c0, 0x3a203000, 0x3e93186c, 0x100c00, 0x3aa23530, 0x1115c, 0x6b3286c, 0x3eb23930, 0xeb23930, 0x20000000, 0}); +typedef Field<uint32_t, 30, 3, StatTable30, DynTable30, &SQR_TABLE_30, &QRT_TABLE_30> Field30; +#endif + +#ifdef ENABLE_FIELD_INT_31 +// 31 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5, 5, 5> StatTable31; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 4, 3> DynTable31; +constexpr StatTable31 SQR_TABLE_31({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x4800000, 0x12000000, 0x48000000, 0x20000012}); +constexpr StatTable31 QRT_TABLE_31({0, 0x10110, 0x10112, 0x15076e, 0x10116, 0x117130e, 0x150766, 0x4743fa0, 0x10106, 0x1121008, 0x117132e, 0x176b248e, 0x150726, 0x172a2c88, 0x4743f20, 0x7eb81e86, 0x10006, 0x20008, 0x1121208, 0x56b2c8e, 0x117172e, 0x133f1bae, 0x176b2c8e, 0x7f2a0c8e, 0x151726, 0x10000000, 0x172a0c88, 0x60000006, 0x4747f20, 0x3eb89e80, 0x7eb89e86}); +typedef Field<uint32_t, 31, 9, StatTable31, DynTable31, &SQR_TABLE_31, &QRT_TABLE_31> Field31; +#endif + +#ifdef ENABLE_FIELD_INT_32 +// 32 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5, 5, 5> StatTable32; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 4, 4> DynTable32; +constexpr StatTable32 SQR_TABLE_32({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x8d, 0x234, 0x8d0, 0x2340, 0x8d00, 0x23400, 0x8d000, 0x234000, 0x8d0000, 0x2340000, 0x8d00000, 0x23400000, 0x8d000000, 0x3400011a, 0xd0000468, 0x40001037}); +constexpr StatTable32 QRT_TABLE_32({0x54fd1264, 0xc26fcd64, 0xc26fcd66, 0x238a7462, 0xc26fcd62, 0x973bccaa, 0x238a746a, 0x77766712, 0xc26fcd72, 0xc1bdd556, 0x973bcc8a, 0x572a094c, 0x238a742a, 0xb693be84, 0x77766792, 0x9555c03e, 0xc26fcc72, 0x568419f8, 0xc1bdd756, 0x96c3d2ca, 0x973bc88a, 0x54861fdc, 0x572a014c, 0xb79badc4, 0x238a642a, 0xb9b99fe0, 0xb6939e84, 0xc519fa86, 0x77762792, 0, 0x9555403e, 0x377627ba}); +typedef Field<uint32_t, 32, 141, StatTable32, DynTable32, &SQR_TABLE_32, &QRT_TABLE_32> Field32; +#endif +} + +Sketch* ConstructGeneric4Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_25 + case 25: return new SketchImpl<Field25>(implementation, 25); +#endif +#ifdef ENABLE_FIELD_INT_26 + case 26: return new SketchImpl<Field26>(implementation, 26); +#endif +#ifdef ENABLE_FIELD_INT_27 + case 27: return new SketchImpl<Field27>(implementation, 27); +#endif +#ifdef ENABLE_FIELD_INT_28 + case 28: return new SketchImpl<Field28>(implementation, 28); +#endif +#ifdef ENABLE_FIELD_INT_29 + case 29: return new SketchImpl<Field29>(implementation, 29); +#endif +#ifdef ENABLE_FIELD_INT_30 + case 30: return new SketchImpl<Field30>(implementation, 30); +#endif +#ifdef ENABLE_FIELD_INT_31 + case 31: return new SketchImpl<Field31>(implementation, 31); +#endif +#ifdef ENABLE_FIELD_INT_32 + case 32: return new SketchImpl<Field32>(implementation, 32); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_5bytes.cpp b/src/minisketch/src/fields/generic_5bytes.cpp new file mode 100644 index 0000000000..b06418184d --- /dev/null +++ b/src/minisketch/src/fields/generic_5bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_5) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_33 +// 33 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5> StatTable33; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable33; +constexpr StatTable33 SQR_TABLE_33({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x802, 0x2008, 0x8020, 0x20080, 0x80200, 0x200800, 0x802000, 0x2008000, 0x8020000, 0x20080000, 0x80200000, 0x800401, 0x2001004, 0x8004010, 0x20010040, 0x80040100}); +constexpr StatTable33 QRT_TABLE_33({0xba504dd4, 0x1e2798ef2, 0x1e2798ef0, 0x6698a4ec, 0x1e2798ef4, 0x1c7f1bef0, 0x6698a4e4, 0x16da1b384, 0x1e2798ee4, 0x661ca6ec, 0x1c7f1bed0, 0x1483b87a6, 0x6698a4a4, 0x800000, 0x16da1b304, 0x1a185101c, 0x1e2798fe4, 0xaa400954, 0x661ca4ec, 0x667caeec, 0x1c7f1bad0, 0x400800, 0x1483b8fa6, 0, 0x6698b4a4, 0x1c61da4b8, 0x802000, 0x16e5dadec, 0x16da1f304, 0x62fc8eec, 0x1a185901c, 0x1661da5ec, 0x1e2788fe4}); +typedef Field<uint64_t, 33, 1025, StatTable33, DynTable33, &SQR_TABLE_33, &QRT_TABLE_33> Field33; +#endif + +#ifdef ENABLE_FIELD_INT_34 +// 34 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5> StatTable34; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable34; +constexpr StatTable34 SQR_TABLE_34({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x81, 0x204, 0x810, 0x2040, 0x8100, 0x20400, 0x81000, 0x204000, 0x810000, 0x2040000, 0x8100000, 0x20400000, 0x81000000, 0x204000000, 0x10000102, 0x40000408, 0x100001020}); +constexpr StatTable34 QRT_TABLE_34({0x2f973a1f6, 0x40202, 0x40200, 0x348102060, 0x40204, 0x8000420, 0x348102068, 0x1092195c8, 0x40214, 0x3f6881b6e, 0x8000400, 0x3f810383e, 0x348102028, 0x340002068, 0x109219548, 0x24015a774, 0x40314, 0x3f050343e, 0x3f688196e, 0x3f81c3a3a, 0x8000000, 0x24031a560, 0x3f810303e, 0xb08c1a12, 0x348103028, 0xb2881906, 0x340000068, 0, 0x10921d548, 0x2e131e576, 0x240152774, 0x18921d55e, 0x50314, 0x14015271c}); +typedef Field<uint64_t, 34, 129, StatTable34, DynTable34, &SQR_TABLE_34, &QRT_TABLE_34> Field34; +#endif + +#ifdef ENABLE_FIELD_INT_35 +// 35 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5> StatTable35; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable35; +constexpr StatTable35 SQR_TABLE_35({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x280000, 0xa00000, 0x2800000, 0xa000000, 0x28000000, 0xa0000000, 0x280000000, 0x200000005}); +constexpr StatTable35 QRT_TABLE_35({0x5c2038114, 0x2bf547ee8, 0x2bf547eea, 0x2bf1074e8, 0x2bf547eee, 0x1883d0736, 0x2bf1074e0, 0x100420, 0x2bf547efe, 0x400800, 0x1883d0716, 0x5e90e4a0, 0x2bf1074a0, 0x4e70ac20, 0x1004a0, 0x2f060c880, 0x2bf547ffe, 0x37d55fffe, 0x400a00, 0x3372573de, 0x1883d0316, 0x700c20, 0x5e90eca0, 0x10604880, 0x2bf1064a0, 0x18f35377e, 0x4e708c20, 0x33f557ffe, 0x1044a0, 0x1bf557ffe, 0x2f0604880, 0x200000000, 0x2bf557ffe, 0, 0x37d57fffe}); +typedef Field<uint64_t, 35, 5, StatTable35, DynTable35, &SQR_TABLE_35, &QRT_TABLE_35> Field35; +#endif + +#ifdef ENABLE_FIELD_INT_36 +// 36 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6> StatTable36; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable36; +constexpr StatTable36 SQR_TABLE_36({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x201, 0x804, 0x2010, 0x8040, 0x20100, 0x80400, 0x201000, 0x804000, 0x2010000, 0x8040000, 0x20100000, 0x80400000, 0x201000000, 0x804000000, 0x10000402, 0x40001008, 0x100004020, 0x400010080}); +constexpr StatTable36 QRT_TABLE_36({0x40200, 0x8b0526186, 0x8b0526184, 0x240001000, 0x8b0526180, 0xcb6894d94, 0x240001008, 0xdb6880c22, 0x8b0526190, 0x8000200, 0xcb6894db4, 0x500424836, 0x240001048, 0x406cb2834, 0xdb6880ca2, 0x241200008, 0x8b0526090, 0xdb05021a6, 0x8000000, 0xdb01829b2, 0xcb68949b4, 0x1001000, 0x500424036, 0x106116406, 0x240000048, 0xcb29968a4, 0x406cb0834, 0, 0xdb6884ca2, 0x110010516, 0x241208008, 0x430434520, 0x8b0536090, 0x41208040, 0xdb05221a6, 0xb6884d14}); +typedef Field<uint64_t, 36, 513, StatTable36, DynTable36, &SQR_TABLE_36, &QRT_TABLE_36> Field36; +#endif + +#ifdef ENABLE_FIELD_INT_37 +// 37 bit field +typedef RecLinTrans<uint64_t, 6, 6, 5, 5, 5, 5, 5> StatTable37; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable37; +constexpr StatTable37 SQR_TABLE_37({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0xa6, 0x298, 0xa60, 0x2980, 0xa600, 0x29800, 0xa6000, 0x298000, 0xa60000, 0x2980000, 0xa600000, 0x29800000, 0xa6000000, 0x298000000, 0xa60000000, 0x980000053, 0x60000011f, 0x180000047c}); +constexpr StatTable37 QRT_TABLE_37({0xa3c62e7ba, 0xdc7a0c16a, 0xdc7a0c168, 0x12f7484546, 0xdc7a0c16c, 0xa9803a20, 0x12f748454e, 0xda07064a4, 0xdc7a0c17c, 0x123908de8e, 0xa9803a00, 0x122a888a8e, 0x12f748450e, 0x6790add8, 0xda0706424, 0x12e0a0384c, 0xdc7a0c07c, 0xcb28a2c2, 0x123908dc8e, 0xd09f85e86, 0xa9803e00, 0x124d682b6e, 0x122a88828e, 0x1738711a, 0x12f748550e, 0x73035b8, 0x67908dd8, 0xa0702438, 0xda0702424, 0xe0a0b860, 0x12e0a0b84c, 0x1c7a1c060, 0xdc7a1c07c, 0, 0xcb2aa2c2, 0x100000002c, 0x12390cdc8e}); +typedef Field<uint64_t, 37, 83, StatTable37, DynTable37, &SQR_TABLE_37, &QRT_TABLE_37> Field37; +#endif + +#ifdef ENABLE_FIELD_INT_38 +// 38 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5, 5> StatTable38; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable38; +constexpr StatTable38 SQR_TABLE_38({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x63, 0x18c, 0x630, 0x18c0, 0x6300, 0x18c00, 0x63000, 0x18c000, 0x630000, 0x18c0000, 0x6300000, 0x18c00000, 0x63000000, 0x18c000000, 0x630000000, 0x18c0000000, 0x2300000063, 0xc0000014a, 0x3000000528}); +constexpr StatTable38 QRT_TABLE_38({0x34b0ac6430, 0x2223262fa, 0x2223262f8, 0x35554405fe, 0x2223262fc, 0x355514098a, 0x35554405f6, 0x400840, 0x2223262ec, 0x1777726532, 0x35551409aa, 0x15c06fc0, 0x35554405b6, 0x1f5303fec, 0x4008c0, 0x236a21030, 0x2223263ec, 0x1a9008c00, 0x1777726732, 0x3692c60ab6, 0x3555140daa, 0x15556007ee, 0x15c067c0, 0x14a0b030f2, 0x35554415b6, 0x227c06d168, 0x1f5301fec, 0x16c3928fc2, 0x4048c0, 0x3a942c4c0, 0x236a29030, 0x1636a2902e, 0x2223363ec, 0x3a6e898276, 0x1a9028c00, 0x6de74eb2c, 0x1777766732, 0}); +typedef Field<uint64_t, 38, 99, StatTable38, DynTable38, &SQR_TABLE_38, &QRT_TABLE_38> Field38; +#endif + +#ifdef ENABLE_FIELD_INT_39 +// 39 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5> StatTable39; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable39; +constexpr StatTable39 SQR_TABLE_39({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x22, 0x88, 0x220, 0x880, 0x2200, 0x8800, 0x22000, 0x88000, 0x220000, 0x880000, 0x2200000, 0x8800000, 0x22000000, 0x88000000, 0x220000000, 0x880000000, 0x2200000000, 0x800000011, 0x2000000044}); +constexpr StatTable39 QRT_TABLE_39({0x66b02a408c, 0x100420, 0x100422, 0x14206080, 0x100426, 0x5dccefab1c, 0x14206088, 0x9fc11e5b6, 0x100436, 0x5466bea62a, 0x5dccefab3c, 0x9aa110536, 0x142060c8, 0x54739ed6e2, 0x9fc11e536, 0xe7a82c080, 0x100536, 0x4002000, 0x5466bea42a, 0x6a4022000, 0x5dccefaf3c, 0x9e8118536, 0x9aa110d36, 0x5680e080, 0x142070c8, 0x7d293c5b6, 0x54739ef6e2, 0x8d680e080, 0x9fc11a536, 0x6d282c080, 0xe7a824080, 0x800000000, 0x110536, 0x2d680e080, 0x4022000, 0, 0x5466baa42a, 0x46b03a44aa, 0x6a40a2000}); +typedef Field<uint64_t, 39, 17, StatTable39, DynTable39, &SQR_TABLE_39, &QRT_TABLE_39> Field39; +#endif + +#ifdef ENABLE_FIELD_INT_40 +// 40 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5> StatTable40; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable40; +constexpr StatTable40 SQR_TABLE_40({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x39, 0xe4, 0x390, 0xe40, 0x3900, 0xe400, 0x39000, 0xe4000, 0x390000, 0xe40000, 0x3900000, 0xe400000, 0x39000000, 0xe4000000, 0x390000000, 0xe40000000, 0x3900000000, 0xe400000000, 0x900000004b, 0x400000015e}); +constexpr StatTable40 QRT_TABLE_40({0x624b3cecc, 0xbc5c3f4c6, 0xbc5c3f4c4, 0xde1603e2c, 0xbc5c3f4c0, 0xaabec06cea, 0xde1603e24, 0x6cd9f724c2, 0xbc5c3f4d0, 0xcde1743818, 0xaabec06cca, 0xa138c314ca, 0xde1603e64, 0xaafc00f01a, 0x6cd9f72442, 0xcdca11bb4, 0xbc5c3f5d0, 0xa00002001a, 0xcde1743a18, 0xdf1407b90, 0xaabec068ca, 0xc043b482c8, 0xa138c31cca, 0xcb86977e3c, 0xde1602e64, 0x604596a326, 0xaafc00d01a, 0xcc1c165d0, 0x6cd9f76442, 0x673c94da26, 0xcdca19bb4, 0x67c0940a26, 0xbc5c2f5d0, 0xa4dca19bae, 0xa00000001a, 0x1bc5c2f5d0, 0xcde1703a18, 0, 0xdf1487b90, 0x8df1487b8a}); +typedef Field<uint64_t, 40, 57, StatTable40, DynTable40, &SQR_TABLE_40, &QRT_TABLE_40> Field40; +#endif +} + +Sketch* ConstructGeneric5Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_33 + case 33: return new SketchImpl<Field33>(implementation, 33); +#endif +#ifdef ENABLE_FIELD_INT_34 + case 34: return new SketchImpl<Field34>(implementation, 34); +#endif +#ifdef ENABLE_FIELD_INT_35 + case 35: return new SketchImpl<Field35>(implementation, 35); +#endif +#ifdef ENABLE_FIELD_INT_36 + case 36: return new SketchImpl<Field36>(implementation, 36); +#endif +#ifdef ENABLE_FIELD_INT_37 + case 37: return new SketchImpl<Field37>(implementation, 37); +#endif +#ifdef ENABLE_FIELD_INT_38 + case 38: return new SketchImpl<Field38>(implementation, 38); +#endif +#ifdef ENABLE_FIELD_INT_39 + case 39: return new SketchImpl<Field39>(implementation, 39); +#endif +#ifdef ENABLE_FIELD_INT_40 + case 40: return new SketchImpl<Field40>(implementation, 40); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_6bytes.cpp b/src/minisketch/src/fields/generic_6bytes.cpp new file mode 100644 index 0000000000..becb26e875 --- /dev/null +++ b/src/minisketch/src/fields/generic_6bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_6) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_41 +// 41 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5> StatTable41; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable41; +constexpr StatTable41 SQR_TABLE_41({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x4800000, 0x12000000, 0x48000000, 0x120000000, 0x480000000, 0x1200000000, 0x4800000000, 0x12000000000, 0x8000000012}); +constexpr StatTable41 QRT_TABLE_41({0, 0x1599a5e0b0, 0x1599a5e0b2, 0x105c119e0, 0x1599a5e0b6, 0x1a2030452a6, 0x105c119e8, 0x1a307c55b2e, 0x1599a5e0a6, 0x1ee3f47bc8e, 0x1a203045286, 0x400808, 0x105c119a8, 0x1a3038573a6, 0x1a307c55bae, 0x4d2882a520, 0x1599a5e1a6, 0x1ffbaa0b720, 0x1ee3f47be8e, 0x4d68c22528, 0x1a203045686, 0x200006, 0x400008, 0x1b79a21b200, 0x105c109a8, 0x1ef3886a526, 0x1a3038553a6, 0x1b692209200, 0x1a307c51bae, 0x5d99a4e1a6, 0x4d28822520, 0x185e109ae, 0x1599a4e1a6, 0x4e3f43be88, 0x1ffbaa2b720, 0x4000000000, 0x1ee3f43be8e, 0x18000000006, 0x4d68ca2528, 0xa203145680, 0x1a203145686}); +typedef Field<uint64_t, 41, 9, StatTable41, DynTable41, &SQR_TABLE_41, &QRT_TABLE_41> Field41; +#endif + +#ifdef ENABLE_FIELD_INT_42 +// 42 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6> StatTable42; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable42; +constexpr StatTable42 SQR_TABLE_42({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x81, 0x204, 0x810, 0x2040, 0x8100, 0x20400, 0x81000, 0x204000, 0x810000, 0x2040000, 0x8100000, 0x20400000, 0x81000000, 0x204000000, 0x810000000, 0x2040000000, 0x8100000000, 0x20400000000, 0x1000000102, 0x4000000408, 0x10000001020}); +constexpr StatTable42 QRT_TABLE_42({0x810200080, 0x120810806, 0x120810804, 0x1068c1a1000, 0x120810800, 0x34005023008, 0x1068c1a1008, 0x800004080, 0x120810810, 0x162818a10, 0x34005023028, 0x42408a14, 0x1068c1a1048, 0x1001040, 0x800004000, 0xb120808906, 0x120810910, 0x34000020068, 0x162818810, 0x68c021400, 0x34005023428, 0x10004000, 0x42408214, 0x162418214, 0x1068c1a0048, 0xb002018116, 0x1003040, 0x10008180448, 0x800000000, 0x62c08b04, 0xb120800906, 0x2408d1a3060, 0x120800910, 0x34401003028, 0x34000000068, 0, 0x162858810, 0xa042058116, 0x68c0a1400, 0x8162858806, 0x34005123428, 0x3068c0a1468}); +typedef Field<uint64_t, 42, 129, StatTable42, DynTable42, &SQR_TABLE_42, &QRT_TABLE_42> Field42; +#endif + +#ifdef ENABLE_FIELD_INT_43 +// 43 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5, 5, 5> StatTable43; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable43; +constexpr StatTable43 SQR_TABLE_43({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0xb2, 0x2c8, 0xb20, 0x2c80, 0xb200, 0x2c800, 0xb2000, 0x2c8000, 0xb20000, 0x2c80000, 0xb200000, 0x2c800000, 0xb2000000, 0x2c8000000, 0xb20000000, 0x2c80000000, 0xb200000000, 0x2c800000000, 0x32000000059, 0x4800000013d, 0x20000000446}); +constexpr StatTable43 QRT_TABLE_43({0x2bccc2d6f6c, 0x4bccc2d6f54, 0x4bccc2d6f56, 0x7cc7bc61df0, 0x4bccc2d6f52, 0x7d13b404b10, 0x7cc7bc61df8, 0x37456e9ac5a, 0x4bccc2d6f42, 0x4e042c6a6, 0x7d13b404b30, 0x4a56de9ef4c, 0x7cc7bc61db8, 0x14bc18d8e, 0x37456e9acda, 0x7c89f84fb1e, 0x4bccc2d6e42, 0x7ffae40d210, 0x4e042c4a6, 0x366f45dd06, 0x7d13b404f30, 0x496fcaf8cca, 0x4a56de9e74c, 0x370b62b6af4, 0x7cc7bc60db8, 0x1498185a8, 0x14bc1ad8e, 0x7e602c46a98, 0x37456e9ecda, 0x36ccc2c6e74, 0x7c89f847b1e, 0x7e27d06d516, 0x4bccc2c6e42, 0x7f93302c396, 0x7ffae42d210, 0x3dd3440706, 0x4e046c4a6, 0x78bbc09da36, 0x366f4ddd06, 0, 0x7d13b504f30, 0x8bbc09da00, 0x496fc8f8cca}); +typedef Field<uint64_t, 43, 89, StatTable43, DynTable43, &SQR_TABLE_43, &QRT_TABLE_43> Field43; +#endif + +#ifdef ENABLE_FIELD_INT_44 +// 44 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5> StatTable44; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable44; +constexpr StatTable44 SQR_TABLE_44({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x21, 0x84, 0x210, 0x840, 0x2100, 0x8400, 0x21000, 0x84000, 0x210000, 0x840000, 0x2100000, 0x8400000, 0x21000000, 0x84000000, 0x210000000, 0x840000000, 0x2100000000, 0x8400000000, 0x21000000000, 0x84000000000, 0x10000000042, 0x40000000108}); +constexpr StatTable44 QRT_TABLE_44({0xf05334f4f6e, 0x4002016, 0x4002014, 0xf04350e6246, 0x4002010, 0x4935b379a26, 0xf04350e624e, 0xf84250c228e, 0x4002000, 0xf04300e521e, 0x4935b379a06, 0xb966838dd48, 0xf04350e620e, 0xf7b8b80feda, 0xf84250c220e, 0xf972e097d5e, 0x4002100, 0x8000020000, 0xf04300e501e, 0x430025000, 0x4935b379e06, 0xf976a09dc5e, 0xb966838d548, 0xf84218c029a, 0xf04350e720e, 0x4925f36bf06, 0xf7b8b80deda, 0xb047d3ee758, 0xf84250c620e, 0xf80350e720e, 0xf972e09fd5e, 0x8091825284, 0x4012100, 0x9015063210, 0x8000000000, 0xff31a028c5e, 0xf04300a501e, 0x44340b7100, 0x4300a5000, 0, 0x4935b279e06, 0xa976b2dce18, 0xf976a29dc5e, 0x8935b279e18}); +typedef Field<uint64_t, 44, 33, StatTable44, DynTable44, &SQR_TABLE_44, &QRT_TABLE_44> Field44; +#endif + +#ifdef ENABLE_FIELD_INT_45 +// 45 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5> StatTable45; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable45; +constexpr StatTable45 SQR_TABLE_45({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x36, 0xd8, 0x360, 0xd80, 0x3600, 0xd800, 0x36000, 0xd8000, 0x360000, 0xd80000, 0x3600000, 0xd800000, 0x36000000, 0xd8000000, 0x360000000, 0xd80000000, 0x3600000000, 0xd800000000, 0x36000000000, 0xd8000000000, 0x16000000001b, 0x18000000005a}); +constexpr StatTable45 QRT_TABLE_45({0xede34e3e0fc, 0x1554148191aa, 0x1554148191a8, 0x1767be1dc4a6, 0x1554148191ac, 0x26bd4931492, 0x1767be1dc4ae, 0x233ab9c454a, 0x1554148191bc, 0x16939e8bb3dc, 0x26bd49314b2, 0x3c6ca8bac52, 0x1767be1dc4ee, 0x16caa5054c16, 0x233ab9c45ca, 0x14a1649628bc, 0x1554148190bc, 0x3c382881252, 0x16939e8bb1dc, 0x3c7ca0aa160, 0x26bd49310b2, 0x27f40158000, 0x3c6ca8ba452, 0x173fc092853c, 0x1767be1dd4ee, 0x16cbe284f25c, 0x16caa5056c16, 0x155559002f96, 0x233ab9c05ca, 0x26eb8908b32, 0x14a16496a8bc, 0x15440885333c, 0x1554148090bc, 0x17d60702e0, 0x3c3828a1252, 0x54548d10b2, 0x16939e8fb1dc, 0x3ac1e81b1d2, 0x3c7ca02a160, 0x166bd48310bc, 0x26bd48310b2, 0, 0x27f40358000, 0x10000000000e, 0x3c6cacba452}); +typedef Field<uint64_t, 45, 27, StatTable45, DynTable45, &SQR_TABLE_45, &QRT_TABLE_45> Field45; +#endif + +#ifdef ENABLE_FIELD_INT_46 +// 46 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5> StatTable46; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable46; +constexpr StatTable46 SQR_TABLE_46({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000, 0xc0000000, 0x300000000, 0xc00000000, 0x3000000000, 0xc000000000, 0x30000000000, 0xc0000000000, 0x300000000000}); +constexpr StatTable46 QRT_TABLE_46({0x211c4fd486ba, 0x100104a, 0x1001048, 0x104d0492d4, 0x100104c, 0x20005040c820, 0x104d0492dc, 0x40008080, 0x100105c, 0x24835068ce00, 0x20005040c800, 0x200000400800, 0x104d04929c, 0x100904325c, 0x40008000, 0x25da9e77daf0, 0x100115c, 0x1184e1696f0, 0x24835068cc00, 0x24825169dd5c, 0x20005040cc00, 0x3ea3241c60c0, 0x200000400000, 0x211c4e5496f0, 0x104d04829c, 0x20005340d86c, 0x100904125c, 0x24835968de5c, 0x4000c000, 0x6400a0c0, 0x25da9e775af0, 0x118cf1687ac, 0x101115c, 0x1ea1745cacc0, 0x1184e1496f0, 0x20181e445af0, 0x2483506ccc00, 0x20240060c0, 0x24825161dd5c, 0x1e21755dbd9c, 0x20005050cc00, 0x26a3746cacc0, 0x3ea3243c60c0, 0xea3243c60c0, 0x200000000000, 0}); +typedef Field<uint64_t, 46, 3, StatTable46, DynTable46, &SQR_TABLE_46, &QRT_TABLE_46> Field46; +#endif + +#ifdef ENABLE_FIELD_INT_47 +// 47 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5> StatTable47; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable47; +constexpr StatTable47 SQR_TABLE_47({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x42, 0x108, 0x420, 0x1080, 0x4200, 0x10800, 0x42000, 0x108000, 0x420000, 0x1080000, 0x4200000, 0x10800000, 0x42000000, 0x108000000, 0x420000000, 0x1080000000, 0x4200000000, 0x10800000000, 0x42000000000, 0x108000000000, 0x420000000000, 0x80000000042, 0x200000000108}); +constexpr StatTable47 QRT_TABLE_47({0, 0x1001040, 0x1001042, 0x1047043076, 0x1001046, 0x112471c241e, 0x104704307e, 0x4304e052168, 0x1001056, 0x10004000, 0x112471c243e, 0x172a09c949d6, 0x104704303e, 0x4002020, 0x4304e0521e8, 0x5400e220, 0x1001156, 0x172b08c85080, 0x10004200, 0x41200b0800, 0x112471c203e, 0x172f0cca50a0, 0x172a09c941d6, 0x7eb88a11c1d6, 0x104704203e, 0x1044042020, 0x4000020, 0x42001011156, 0x4304e0561e8, 0x172a28c95880, 0x54006220, 0x112931cc21e, 0x1011156, 0x53670f283e, 0x172b08ca5080, 0x7a80c414a03e, 0x10044200, 0x40000000000, 0x4120030800, 0x1928318801e, 0x112470c203e, 0x799283188000, 0x172f0cea50a0, 0x1eb88a91c1c8, 0x172a098941d6, 0x3ea8cc95e1f6, 0x7eb88a91c1d6}); +typedef Field<uint64_t, 47, 33, StatTable47, DynTable47, &SQR_TABLE_47, &QRT_TABLE_47> Field47; +#endif + +#ifdef ENABLE_FIELD_INT_48 +// 48 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6> StatTable48; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable48; +constexpr StatTable48 SQR_TABLE_48({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x2d, 0xb4, 0x2d0, 0xb40, 0x2d00, 0xb400, 0x2d000, 0xb4000, 0x2d0000, 0xb40000, 0x2d00000, 0xb400000, 0x2d000000, 0xb4000000, 0x2d0000000, 0xb40000000, 0x2d00000000, 0xb400000000, 0x2d000000000, 0xb4000000000, 0x2d0000000000, 0xb40000000000, 0xd0000000005a, 0x40000000011f}); +constexpr StatTable48 QRT_TABLE_48({0xc00442c284f0, 0xc16b7fda410a, 0xc16b7fda4108, 0xada3b5c79fbe, 0xc16b7fda410c, 0x16f3c18d5b0, 0xada3b5c79fb6, 0x7090a381f64, 0xc16b7fda411c, 0xcafc15d179f8, 0x16f3c18d590, 0x6630880e534e, 0xada3b5c79ff6, 0xa13dd1f49826, 0x7090a381fe4, 0xb87560f6a74, 0xc16b7fda401c, 0xaaaaffff0012, 0xcafc15d17bf8, 0xaafd15f07bf6, 0x16f3c18d190, 0x60000020000e, 0x6630880e5b4e, 0xcb977fcb401c, 0xada3b5c78ff6, 0x6663420cad0, 0xa13dd1f4b826, 0xc0045fc2f41c, 0x7090a385fe4, 0x6762e24b834, 0xb87560fea74, 0xc6351fed241c, 0xc16b7fdb401c, 0x60065622ea7a, 0xaaaafffd0012, 0xdf9562bea74, 0xcafc15d57bf8, 0x6657ea057bea, 0xaafd15f87bf6, 0xa79329ddaa66, 0x16f3c08d190, 0xa39229f0aa66, 0x60000000000e, 0x175fb4468ad0, 0x6630884e5b4e, 0, 0xcb977f4b401c, 0x2630884e5b40}); +typedef Field<uint64_t, 48, 45, StatTable48, DynTable48, &SQR_TABLE_48, &QRT_TABLE_48> Field48; +#endif +} + +Sketch* ConstructGeneric6Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_41 + case 41: return new SketchImpl<Field41>(implementation, 41); +#endif +#ifdef ENABLE_FIELD_INT_42 + case 42: return new SketchImpl<Field42>(implementation, 42); +#endif +#ifdef ENABLE_FIELD_INT_43 + case 43: return new SketchImpl<Field43>(implementation, 43); +#endif +#ifdef ENABLE_FIELD_INT_44 + case 44: return new SketchImpl<Field44>(implementation, 44); +#endif +#ifdef ENABLE_FIELD_INT_45 + case 45: return new SketchImpl<Field45>(implementation, 45); +#endif +#ifdef ENABLE_FIELD_INT_46 + case 46: return new SketchImpl<Field46>(implementation, 46); +#endif +#ifdef ENABLE_FIELD_INT_47 + case 47: return new SketchImpl<Field47>(implementation, 47); +#endif +#ifdef ENABLE_FIELD_INT_48 + case 48: return new SketchImpl<Field48>(implementation, 48); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_7bytes.cpp b/src/minisketch/src/fields/generic_7bytes.cpp new file mode 100644 index 0000000000..8222f37a64 --- /dev/null +++ b/src/minisketch/src/fields/generic_7bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_7) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_49 +// 49 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable49; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable49; +constexpr StatTable49 SQR_TABLE_49({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x402, 0x1008, 0x4020, 0x10080, 0x40200, 0x100800, 0x402000, 0x1008000, 0x4020000, 0x10080000, 0x40200000, 0x100800000, 0x402000000, 0x1008000000, 0x4020000000, 0x10080000000, 0x40200000000, 0x100800000000, 0x402000000000, 0x1008000000000, 0x20000000402, 0x80000001008, 0x200000004020, 0x800000010080}); +constexpr StatTable49 QRT_TABLE_49({0, 0x10004196, 0x10004194, 0x5099461f080, 0x10004190, 0x40840600c20, 0x5099461f088, 0x58a56349cfde, 0x10004180, 0x48641a0c03fe, 0x40840600c00, 0x10084002848, 0x5099461f0c8, 0x4002048, 0x58a56349cf5e, 0x5088460a048, 0x10004080, 0x4c2852624dde, 0x48641a0c01fe, 0x14893129c280, 0x40840600800, 0x1eb23c323ace8, 0x10084002048, 0x48740a09417e, 0x5099461e0c8, 0x40852604d96, 0x4000048, 0x5cad2b29c37e, 0x58a563498f5e, 0x20000200, 0x50884602048, 0x10000000000, 0x10014080, 0x4c2a56624d96, 0x4c2852604dde, 0x1ee2347438ca0, 0x48641a0801fe, 0x480000000048, 0x14893121c280, 0x14091121c080, 0x40840700800, 0x1a5099561e17e, 0x1eb23c303ace8, 0x8740a894136, 0x10084402048, 0x18101c501ace8, 0x48740a89417e, 0x15dace6286f96, 0x5099561e0c8}); +typedef Field<uint64_t, 49, 513, StatTable49, DynTable49, &SQR_TABLE_49, &QRT_TABLE_49> Field49; +#endif + +#ifdef ENABLE_FIELD_INT_50 +// 50 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable50; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable50; +constexpr StatTable50 SQR_TABLE_50({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x1d, 0x74, 0x1d0, 0x740, 0x1d00, 0x7400, 0x1d000, 0x74000, 0x1d0000, 0x740000, 0x1d00000, 0x7400000, 0x1d000000, 0x74000000, 0x1d0000000, 0x740000000, 0x1d00000000, 0x7400000000, 0x1d000000000, 0x74000000000, 0x1d0000000000, 0x740000000000, 0x1d00000000000, 0x340000000001d, 0x1000000000053}); +constexpr StatTable50 QRT_TABLE_50({0xfbdfa3ae9d4c, 0x38143245a4878, 0x38143245a487a, 0x38527487e7492, 0x38143245a487e, 0x3124c61f56d2a, 0x38527487e749a, 0xfa8c91b087c0, 0x38143245a486e, 0x3eca48c6196be, 0x3124c61f56d0a, 0x380000040080a, 0x38527487e74da, 0x976b2d8b39b4, 0xfa8c91b08740, 0xfa8cd5b02724, 0x38143245a496e, 0x316291dd013fe, 0x3eca48c6194be, 0x10344122064, 0x3124c61f5690a, 0x68c5f006ee40, 0x380000040000a, 0x852749fe64d0, 0x38527487e64da, 0x37ef8e9d0e9da, 0x976b2d8b19b4, 0x37fabd1cef34a, 0xfa8c91b0c740, 0x96282d9159b4, 0xfa8cd5b0a724, 0x464a8249dd0, 0x38143245b496e, 0x37eaa8ddc94be, 0x316291dd213fe, 0x392446035690a, 0x3eca48c6594be, 0x974b258b4964, 0x103441a2064, 0x385a7c87fb4da, 0x3124c61e5690a, 0xeb8ad5d9a724, 0x68c5f026ee40, 0x3724c61e5690a, 0x380000000000a, 0x3a8c5f026ee4a, 0x8527497e64d0, 0, 0x38527497e64da, 0x2fbdfa2ae8d0a}); +typedef Field<uint64_t, 50, 29, StatTable50, DynTable50, &SQR_TABLE_50, &QRT_TABLE_50> Field50; +#endif + +#ifdef ENABLE_FIELD_INT_51 +// 51 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable51; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable51; +constexpr StatTable51 SQR_TABLE_51({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x96, 0x258, 0x960, 0x2580, 0x9600, 0x25800, 0x96000, 0x258000, 0x960000, 0x2580000, 0x9600000, 0x25800000, 0x96000000, 0x258000000, 0x960000000, 0x2580000000, 0x9600000000, 0x25800000000, 0x96000000000, 0x258000000000, 0x960000000000, 0x2580000000000, 0x160000000004b, 0x580000000012c, 0x6000000000426}); +constexpr StatTable51 QRT_TABLE_51({0x778bf2703d152, 0x2aaaafbff2092, 0x2aaaafbff2090, 0x4d2119c7e7780, 0x2aaaafbff2094, 0x65de1df8ae194, 0x4d2119c7e7788, 0x67d63d7ba262c, 0x2aaaafbff2084, 0x28ff003f4167c, 0x65de1df8ae1b4, 0x658397fb1d034, 0x4d2119c7e77c8, 0x4d7c9284526ba, 0x67d63d7ba26ac, 0x6666333fc0cbe, 0x2aaaafbff2184, 0x295b807ab55ee, 0x28ff003f4147c, 0x2aaabfffe0016, 0x65de1df8ae5b4, 0x209210349d60, 0x658397fb1d834, 0x4d215dc7cf1c8, 0x4d2119c7e67c8, 0x662b2b3d7b4be, 0x4d7c9284506ba, 0x255af00b36e0, 0x67d63d7ba66ac, 0x65de1fb8ac1a6, 0x6666333fc8cbe, 0x662f3b3ded4be, 0x2aaaafbfe2184, 0x663a9dbc3a426, 0x295b807a955ee, 0x4cdc9ec128928, 0x28ff003f0147c, 0x28a0c93cd511c, 0x2aaabfff60016, 0x65d73cf8e78d4, 0x65de1df9ae5b4, 0x4d5eddc44f1c8, 0x209210149d60, 0x357fcc506c8a, 0x658397ff1d834, 0, 0x4d215dcfcf1c8, 0x63f536f5d4554, 0x4d2119d7e67c8, 0x4000000000022, 0x662b2b1d7b4be}); +typedef Field<uint64_t, 51, 75, StatTable51, DynTable51, &SQR_TABLE_51, &QRT_TABLE_51> Field51; +#endif + +#ifdef ENABLE_FIELD_INT_52 +// 52 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable52; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable52; +constexpr StatTable52 SQR_TABLE_52({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x90000, 0x240000, 0x900000, 0x2400000, 0x9000000, 0x24000000, 0x90000000, 0x240000000, 0x900000000, 0x2400000000, 0x9000000000, 0x24000000000, 0x90000000000, 0x240000000000, 0x900000000000, 0x2400000000000, 0x9000000000000, 0x4000000000012}); +constexpr StatTable52 QRT_TABLE_52({0xc108165459b0e, 0x10004086, 0x10004084, 0xc00000100104e, 0x10004080, 0x2041810a545b0, 0xc000001001046, 0x1181e055efc0, 0x10004090, 0x40810214390, 0x2041810a54590, 0xc000141019106, 0xc000001001006, 0x10816045ab40, 0x1181e055ef40, 0xc000111015196, 0x10004190, 0xe045c19af44a2, 0x40810214190, 0xe045809ad0532, 0x2041810a54190, 0xdb387a03fe646, 0xc000141019906, 0x2000000800000, 0xc000001000006, 0x2486548199c34, 0x108160458b40, 0x2041808a50534, 0x1181e055af40, 0xc0408312153d6, 0xc00011101d196, 0x21499f0e0eed0, 0x10014190, 0xe15dff9faabe2, 0xe045c19ad44a2, 0xdb787b01ea7d6, 0x40810254190, 0xe484409180532, 0xe045809a50532, 0xc14095164d896, 0x2041810b54190, 0x217dee8fb7a74, 0xdb387a01fe646, 0x441810b54190, 0xc000141419906, 0xc3386e15e7f46, 0x2000000000000, 0x1000141419900, 0xc000000000006, 0, 0x248654a199c34, 0xa48654a199c32}); +typedef Field<uint64_t, 52, 9, StatTable52, DynTable52, &SQR_TABLE_52, &QRT_TABLE_52> Field52; +#endif + +#ifdef ENABLE_FIELD_INT_53 +// 53 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5> StatTable53; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable53; +constexpr StatTable53 SQR_TABLE_53({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x8e, 0x238, 0x8e0, 0x2380, 0x8e00, 0x23800, 0x8e000, 0x238000, 0x8e0000, 0x2380000, 0x8e00000, 0x23800000, 0x8e000000, 0x238000000, 0x8e0000000, 0x2380000000, 0x8e00000000, 0x23800000000, 0x8e000000000, 0x238000000000, 0x8e0000000000, 0x2380000000000, 0x8e00000000000, 0x3800000000047, 0xe00000000011c, 0x18000000000437}); +constexpr StatTable53 QRT_TABLE_53({0xf940b90844076, 0x1f940b90844052, 0x1f940b90844050, 0x9d2a063b43e64, 0x1f940b90844054, 0x936f69323ec14, 0x9d2a063b43e6c, 0xe12270a88898, 0x1f940b90844044, 0x1f917f00bb5a3c, 0x936f69323ec34, 0x1f622df85b46ee, 0x9d2a063b43e2c, 0x9bc65ab040b66, 0xe12270a88818, 0x958330b931986, 0x1f940b90844144, 0x98e2a06e32e0, 0x1f917f00bb583c, 0x1f877970dc1024, 0x936f69323e834, 0x16cc3c9b1558c2, 0x1f622df85b4eee, 0x16de1c3351dae8, 0x9d2a063b42e2c, 0x1fecdc7855f8ee, 0x9bc65ab042b66, 0x933821b1cb6fe, 0xe12270a8c818, 0x1f675958641c0e, 0x958330b939986, 0x9d97e050e960, 0x1f940b90854144, 0x1f820fa0e38adc, 0x98e2a06c32e0, 0x1650f0e358a010, 0x1f917f00bf583c, 0x1643af4b037a3a, 0x1f877970d41024, 0x1ffe2c281d8c16, 0x936f69333e834, 0xf00d50ffccf8, 0x16cc3c9b3558c2, 0x16bc31cbca943a, 0x1f622df81b4eee, 0xa6cbd8007232, 0x16de1c33d1dae8, 0x15d2a062b42e10, 0x9d2a062b42e2c, 0x1aa77896586ca, 0x1fecdc7a55f8ee, 0, 0x9bc65af042b66}); +typedef Field<uint64_t, 53, 71, StatTable53, DynTable53, &SQR_TABLE_53, &QRT_TABLE_53> Field53; +#endif + +#ifdef ENABLE_FIELD_INT_54 +// 54 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6> StatTable54; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable54; +constexpr StatTable54 SQR_TABLE_54({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x201, 0x804, 0x2010, 0x8040, 0x20100, 0x80400, 0x201000, 0x804000, 0x2010000, 0x8040000, 0x20100000, 0x80400000, 0x201000000, 0x804000000, 0x2010000000, 0x8040000000, 0x20100000000, 0x80400000000, 0x201000000000, 0x804000000000, 0x2010000000000, 0x8040000000000, 0x20100000000000, 0x400000000402, 0x1000000001008, 0x4000000004020, 0x10000000010080}); +constexpr StatTable54 QRT_TABLE_54({0x201008000200, 0x26c10916494994, 0x26c10916494996, 0x40008008, 0x26c10916494992, 0x141a2434c12d12, 0x40008000, 0x36c00110594c22, 0x26c10916494982, 0x200000040200, 0x141a2434c12d32, 0x10010816104534, 0x40008040, 0x36da60b01308b2, 0x36c00110594ca2, 0x48200209000, 0x26c10916494882, 0x41b6da2d86106, 0x200000040000, 0x32db2c228965b0, 0x141a2434c12932, 0x9000000200048, 0x10010816104d34, 0x32db68b2832da4, 0x40009040, 0x40045928b4902, 0x36da60b01328b2, 0x1000040000, 0x36c00110590ca2, 0x101b69865a4120, 0x48200201000, 0x22da6434912884, 0x26c10916484882, 0x9000240208008, 0x41b6da2da6106, 0x22c14484c20180, 0x200000000000, 0x4016db29b6812, 0x32db2c228165b0, 0x9008200201048, 0x141a2434d12932, 0x32c36ca2c264b0, 0x9000000000048, 0x140a65b48a2c32, 0x10010816504d34, 0, 0x32db68b2032da4, 0x404490824814, 0x41009040, 0x14da60a4536126, 0x40045908b4902, 0x8000041009008, 0x36da60b41328b2, 0x6db68b2032c12}); +typedef Field<uint64_t, 54, 513, StatTable54, DynTable54, &SQR_TABLE_54, &QRT_TABLE_54> Field54; +#endif + +#ifdef ENABLE_FIELD_INT_55 +// 55 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable55; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable55; +constexpr StatTable55 SQR_TABLE_55({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x102, 0x408, 0x1020, 0x4080, 0x10200, 0x40800, 0x102000, 0x408000, 0x1020000, 0x4080000, 0x10200000, 0x40800000, 0x102000000, 0x408000000, 0x1020000000, 0x4080000000, 0x10200000000, 0x40800000000, 0x102000000000, 0x408000000000, 0x1020000000000, 0x4080000000000, 0x10200000000000, 0x40800000000000, 0x2000000000102, 0x8000000000408, 0x20000000001020}); +constexpr StatTable55 QRT_TABLE_55({0, 0x121d57b6623fde, 0x121d57b6623fdc, 0x68908340d10e00, 0x121d57b6623fd8, 0x100300510e20, 0x68908340d10e08, 0x10004096, 0x121d57b6623fc8, 0x100010000, 0x100300510e00, 0x7ea8c890a088e8, 0x68908340d10e48, 0x68809540871648, 0x10004016, 0x68808000808068, 0x121d57b6623ec8, 0x68909240d41c48, 0x100010200, 0x6884c170ad0216, 0x100300510a00, 0x68848160a50200, 0x7ea8c890a080e8, 0x7eecbca04ab4b6, 0x68908340d11e48, 0x120c54b62234c8, 0x68809540873648, 0x69929240d61c48, 0x10000016, 0x68808060800000, 0x68808000800068, 0x80000080, 0x121d57b6633ec8, 0x7ea8cb90a18ae8, 0x68909240d61c48, 0x16284090200080, 0x100050200, 0x474302a345e, 0x6884c170a50216, 0x166cbca0cab4de, 0x100300410a00, 0x1000000000000, 0x68848160850200, 0x688cc1f0a50296, 0x7ea8c890e080e8, 0x7e8cc1f0a50280, 0x7eecbca0cab4b6, 0x68000000000068, 0x68908341d11e48, 0x7880954487365e, 0x120c54b42234c8, 0x9929248d61c20, 0x68809544873648, 0x41121208561c20, 0x69929248d61c48}); +typedef Field<uint64_t, 55, 129, StatTable55, DynTable55, &SQR_TABLE_55, &QRT_TABLE_55> Field55; +#endif + +#ifdef ENABLE_FIELD_INT_56 +// 56 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable56; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable56; +constexpr StatTable56 SQR_TABLE_56({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x95, 0x254, 0x950, 0x2540, 0x9500, 0x25400, 0x95000, 0x254000, 0x950000, 0x2540000, 0x9500000, 0x25400000, 0x95000000, 0x254000000, 0x950000000, 0x2540000000, 0x9500000000, 0x25400000000, 0x95000000000, 0x254000000000, 0x950000000000, 0x2540000000000, 0x9500000000000, 0x25400000000000, 0x95000000000000, 0x5400000000012a, 0x5000000000043d, 0x40000000001061}); +constexpr StatTable56 QRT_TABLE_56({0x10004084, 0xd058f12fd5925e, 0xd058f12fd5925c, 0x41a60b5566d9f0, 0xd058f12fd59258, 0xbda60a142740ba, 0x41a60b5566d9f8, 0xd059f1afc5e688, 0xd058f12fd59248, 0xfc040841615a22, 0xbda60a1427409a, 0xbda60b5426c1ca, 0x41a60b5566d9b8, 0x1a60b4166b950, 0xd059f1afc5e608, 0xfc000041409822, 0xd058f12fd59348, 0xd1ee7a4ef4185c, 0xfc040841615822, 0x9049759b80b4a4, 0xbda60a1427449a, 0xd258e06f301e18, 0xbda60b5426c9ca, 0x6dfeeb3bf6d7d2, 0x41a60b5566c9b8, 0xbdef3ed4ae398a, 0x1a60b41669950, 0xd1ef3f8eeff04c, 0xd059f1afc5a608, 0xbda203340783de, 0xfc000041401822, 0x2dfefbaff2b27a, 0xd058f12fd49348, 0xfdb788a0706776, 0xd1ee7a4ef6185c, 0x2e5de0ae41337a, 0xfc040841655822, 0x41eb17d5ceecf8, 0x9049759b88b4a4, 0x40048874211afc, 0xbda60a1437449a, 0xd04a720f93400c, 0xd258e06f101e18, 0xbc559cf5ac7fce, 0xbda60b5466c9ca, 0x6dc9759b88b4d6, 0x6dfeeb3b76d7d2, 0x92feea7b275af0, 0x41a60b5466c9b8, 0, 0xbdef3ed6ae398a, 0x2811d5edd8ee2a, 0x1a60b45669950, 0xb1a60b5466c9ca, 0xd1ef3f86eff04c, 0xec493582c8f032}); +typedef Field<uint64_t, 56, 149, StatTable56, DynTable56, &SQR_TABLE_56, &QRT_TABLE_56> Field56; +#endif +} + +Sketch* ConstructGeneric7Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_49 + case 49: return new SketchImpl<Field49>(implementation, 49); +#endif +#ifdef ENABLE_FIELD_INT_50 + case 50: return new SketchImpl<Field50>(implementation, 50); +#endif +#ifdef ENABLE_FIELD_INT_51 + case 51: return new SketchImpl<Field51>(implementation, 51); +#endif +#ifdef ENABLE_FIELD_INT_52 + case 52: return new SketchImpl<Field52>(implementation, 52); +#endif +#ifdef ENABLE_FIELD_INT_53 + case 53: return new SketchImpl<Field53>(implementation, 53); +#endif +#ifdef ENABLE_FIELD_INT_54 + case 54: return new SketchImpl<Field54>(implementation, 54); +#endif +#ifdef ENABLE_FIELD_INT_55 + case 55: return new SketchImpl<Field55>(implementation, 55); +#endif +#ifdef ENABLE_FIELD_INT_56 + case 56: return new SketchImpl<Field56>(implementation, 56); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_8bytes.cpp b/src/minisketch/src/fields/generic_8bytes.cpp new file mode 100644 index 0000000000..8bb63e8d3e --- /dev/null +++ b/src/minisketch/src/fields/generic_8bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_8) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_57 +// 57 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable57; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable57; +constexpr StatTable57 SQR_TABLE_57({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x22, 0x88, 0x220, 0x880, 0x2200, 0x8800, 0x22000, 0x88000, 0x220000, 0x880000, 0x2200000, 0x8800000, 0x22000000, 0x88000000, 0x220000000, 0x880000000, 0x2200000000, 0x8800000000, 0x22000000000, 0x88000000000, 0x220000000000, 0x880000000000, 0x2200000000000, 0x8800000000000, 0x22000000000000, 0x88000000000000, 0x20000000000011, 0x80000000000044}); +constexpr StatTable57 QRT_TABLE_57({0xd0c3a82c902426, 0x232aa54103915e, 0x232aa54103915c, 0x1763e291e61699c, 0x232aa541039158, 0x1f424d678bb15e, 0x1763e291e616994, 0x26fd8122f10d36, 0x232aa541039148, 0x1e0a0206002000, 0x1f424d678bb17e, 0x5d72563f39d7e, 0x1763e291e6169d4, 0x1519beb9d597df4, 0x26fd8122f10db6, 0x150c3a87c90e4aa, 0x232aa541039048, 0x15514891f6179d4, 0x1e0a0206002200, 0x14ec9ba7a94c6aa, 0x1f424d678bb57e, 0x1e0f4286382420, 0x5d72563f3957e, 0x4000080000, 0x1763e291e6179d4, 0x1ac0e804882000, 0x1519beb9d595df4, 0x1f430d6793b57e, 0x26fd8122f14db6, 0x3c68e806882000, 0x150c3a87c9064aa, 0x1484fe18b915e, 0x232aa541029048, 0x14f91eb9b595df4, 0x15514891f6379d4, 0x48f6a82380420, 0x1e0a0206042200, 0x14b1beb99595df4, 0x14ec9ba7a9cc6aa, 0x4cf2a82b00420, 0x1f424d679bb57e, 0x26aa0002000000, 0x1e0f4286182420, 0x173f1039dd17df4, 0x5d72563b3957e, 0x4aa0002000000, 0x4000880000, 0x16d31eb9b595df4, 0x1763e291f6179d4, 0x20000000000000, 0x1ac0e806882000, 0x2caa0002000000, 0x1519beb99595df4, 0, 0x1f430d6f93b57e, 0x73e90d6d93b57e, 0x26fd8132f14db6}); +typedef Field<uint64_t, 57, 17, StatTable57, DynTable57, &SQR_TABLE_57, &QRT_TABLE_57> Field57; +#endif + +#ifdef ENABLE_FIELD_INT_58 +// 58 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable58; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable58; +constexpr StatTable58 SQR_TABLE_58({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x80001, 0x200004, 0x800010, 0x2000040, 0x8000100, 0x20000400, 0x80001000, 0x200004000, 0x800010000, 0x2000040000, 0x8000100000, 0x20000400000, 0x80001000000, 0x200004000000, 0x800010000000, 0x2000040000000, 0x8000100000000, 0x20000400000000, 0x80001000000000, 0x200004000000000, 0x10000100002, 0x40000400008, 0x100001000020, 0x400004000080, 0x1000010000200, 0x4000040000800, 0x10000100002000, 0x40000400008000, 0x100001000020000}); +constexpr StatTable58 QRT_TABLE_58({0x2450096792a5c5c, 0x610014271011c, 0x610014271011e, 0x1f0cb811314ea88, 0x610014271011a, 0x8000000420, 0x1f0cb811314ea80, 0x265407ad8a20bcc, 0x610014271010a, 0x3d18be98392ebd0, 0x8000000400, 0xc29b930e407056, 0x1f0cb811314eac0, 0x1fcef001154dee8, 0x265407ad8a20b4c, 0xc69b924c61f94a, 0x610014271000a, 0x211006895845190, 0x3d18be98392e9d0, 0x54007accac09cc, 0x8000000000, 0xc08b934e107854, 0xc29b930e407856, 0x275407adc220bcc, 0x1f0cb811314fac0, 0x1f6db815164ea8a, 0x1fcef001154fee8, 0x1b2db801945e396, 0x265407ad8a24b4c, 0x21100ec95865590, 0xc69b924c61794a, 0x273507b1e530ad6, 0x610014270000a, 0x1b4cb835b34e29c, 0x211006895865190, 0x3839bf20d47e016, 0x3d18be98396e9d0, 0x3858bd34f36e01c, 0x54007acca409cc, 0, 0x8000100000, 0xc29a130e507856, 0xc08b934e307854, 0x13253921d448296, 0xc29b930e007856, 0x13c60935f6486bc, 0x275407adca20bcc, 0x3571be8c5e6c9da, 0x1f0cb811214fac0, 0x410014261011c, 0x1f6db815364ea8a, 0x13a50921d1486b6, 0x1fcef001554fee8, 0x64001249245a5c, 0x1b2db801145e396, 0x8610014670200a, 0x265407ac8a24b4c, 0x1a5cbfbdeb0f30c}); +typedef Field<uint64_t, 58, 524289, StatTable58, DynTable58, &SQR_TABLE_58, &QRT_TABLE_58> Field58; +#endif + +#ifdef ENABLE_FIELD_INT_59 +// 59 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5> StatTable59; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable59; +constexpr StatTable59 SQR_TABLE_59({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x12a, 0x4a8, 0x12a0, 0x4a80, 0x12a00, 0x4a800, 0x12a000, 0x4a8000, 0x12a0000, 0x4a80000, 0x12a00000, 0x4a800000, 0x12a000000, 0x4a8000000, 0x12a0000000, 0x4a80000000, 0x12a00000000, 0x4a800000000, 0x12a000000000, 0x4a8000000000, 0x12a0000000000, 0x4a80000000000, 0x12a00000000000, 0x4a800000000000, 0x12a000000000000, 0x4a8000000000000, 0x2a000000000012a, 0x28000000000043d, 0x200000000001061}); +constexpr StatTable59 QRT_TABLE_59({0x38d905ab028567a, 0x789fa6ed3b44d72, 0x789fa6ed3b44d70, 0x74ec857e93d828c, 0x789fa6ed3b44d74, 0x116b3c1203c96, 0x74ec857e93d8284, 0xc25ebc3871e280, 0x789fa6ed3b44d64, 0x47a37c3d910b6, 0x116b3c1203cb6, 0xc7322d7a8f48de, 0x74ec857e93d82c4, 0xb509a0ea52e496, 0xc25ebc3871e200, 0x74fdee4681d3e0c, 0x789fa6ed3b44c64, 0x7ffbbd080b2f09a, 0x47a37c3d912b6, 0xd5c937bae506c8, 0x116b3c12038b6, 0xb173c76987625e, 0xc7322d7a8f40de, 0x7591ff36b3a682c, 0x74ec857e93d92c4, 0x72b253bfbfc90c4, 0xb509a0ea52c496, 0x79f2e7b10e6d452, 0xc25ebc3871a200, 0x78c86e951086aac, 0x74fdee4681dbe0c, 0x78c96eb514c602c, 0x789fa6ed3b54c64, 0xc34818b95658e8, 0x7ffbbd080b0f09a, 0x7399f563b1980f2, 0x47a37c3dd12b6, 0xa29e0e28c58880, 0xd5c937baed06c8, 0x788ac23520ac82c, 0x116b3c13038b6, 0xa2c857e83d92b6, 0xb173c769a7625e, 0x608da990122e48, 0xc7322d7acf40de, 0xa3a89269eebefe, 0x7591ff36bba682c, 0xa25ebc2871a200, 0x74ec857e83d92c4, 0x11f62e419f1cfe, 0x72b253bf9fc90c4, 0x7425ebc2871a272, 0xb509a0ee52c496, 0x4ed8555979c8de, 0x79f2e7b18e6d452, 0x6c3580d5915d4d2, 0xc25ebc2871a200, 0, 0x78c86e971086aac}); +typedef Field<uint64_t, 59, 149, StatTable59, DynTable59, &SQR_TABLE_59, &QRT_TABLE_59> Field59; +#endif + +#ifdef ENABLE_FIELD_INT_60 +// 60 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6> StatTable60; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable60; +constexpr StatTable60 SQR_TABLE_60({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000, 0xc0000000, 0x300000000, 0xc00000000, 0x3000000000, 0xc000000000, 0x30000000000, 0xc0000000000, 0x300000000000, 0xc00000000000, 0x3000000000000, 0xc000000000000, 0x30000000000000, 0xc0000000000000, 0x300000000000000, 0xc00000000000000}); +constexpr StatTable60 QRT_TABLE_60({0x6983c00fe00104a, 0x804570322e054e6, 0x804570322e054e4, 0x15673387e0a4e4, 0x804570322e054e0, 0x100010110, 0x15673387e0a4ec, 0x920d01f34442a70, 0x804570322e054f0, 0x7a8dc0f2e4058f0, 0x100010130, 0x120c01f140462f0, 0x15673387e0a4ac, 0x7bdbb2ca9a4fe5c, 0x920d01f34442af0, 0xe9c6b039ce0c4ac, 0x804570322e055f0, 0xfac8b080ca20c00, 0x7a8dc0f2e405af0, 0x7a8dc4b2e4a59f0, 0x100010530, 0x10000100000, 0x120c01f14046af0, 0x131a02d91c5db6c, 0x15673387e0b4ac, 0x15623387d0b4ac, 0x7bdbb2ca9a4de5c, 0x7ffbbbca0a8ee5c, 0x920d01f34446af0, 0x800000020000000, 0xe9c6b039ce044ac, 0x81130302500f000, 0x804570322e155f0, 0x935b72eb3a48e9c, 0xfac8b080ca00c00, 0x120c016140563c0, 0x7a8dc0f2e445af0, 0x7bcbb3ca8a4ee5c, 0x7a8dc4b2e4259f0, 0xc4000a0300, 0x100110530, 0x11623285c1b19c, 0x10000300000, 0x420890090c3000, 0x120c01f14446af0, 0x68d7b33b9e0b4ac, 0x131a02d9145db6c, 0xe8ccb1e18a56fc0, 0x15673386e0b4ac, 0x7aadc8f2e485af0, 0x15623385d0b4ac, 0x4a0990093c3000, 0x7bdbb2cada4de5c, 0xf9d6b3389e0b4ac, 0x7ffbbbca8a8ee5c, 0xdf6ba38cec84ac, 0x920d01f24446af0, 0x520d01f24446af0, 0x800000000000000, 0}); +typedef Field<uint64_t, 60, 3, StatTable60, DynTable60, &SQR_TABLE_60, &QRT_TABLE_60> Field60; +#endif + +#ifdef ENABLE_FIELD_INT_61 +// 61 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable61; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable61; +constexpr StatTable61 SQR_TABLE_61({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x138000, 0x4e0000, 0x1380000, 0x4e00000, 0x13800000, 0x4e000000, 0x138000000, 0x4e0000000, 0x1380000000, 0x4e00000000, 0x13800000000, 0x4e000000000, 0x138000000000, 0x4e0000000000, 0x1380000000000, 0x4e00000000000, 0x13800000000000, 0x4e000000000000, 0x138000000000000, 0x4e0000000000000, 0x1380000000000000, 0xe0000000000004e, 0x180000000000011f}); +constexpr StatTable61 QRT_TABLE_61({0x171d34fcdac955d0, 0x12cfc8c049e1c96, 0x12cfc8c049e1c94, 0x71d34fcdac955c2, 0x12cfc8c049e1c90, 0x631c871de564852, 0x71d34fcdac955ca, 0x129fa6407f27300, 0x12cfc8c049e1c80, 0x7094f6fdd0a3b12, 0x631c871de564872, 0xdb28cee59c8256a, 0x71d34fcdac9558a, 0xc8a0be15a915472, 0x129fa6407f27380, 0x12dfcb4058e0b80, 0x12cfc8c049e1d80, 0x117d7f04ad0118, 0x7094f6fdd0a3912, 0x621b576dbe35b6a, 0x631c871de564c72, 0x13c808a013a1ee0, 0xdb28cee59c82d6a, 0x113d79842a0272, 0x71d34fcdac9458a, 0x719776b580b6a98, 0xc8a0be15a917472, 0x6633498d6db760a, 0x129fa6407f23380, 0xbd4ae9e8c3e7560, 0x12dfcb4058e8b80, 0x8000000a, 0x12cfc8c049f1d80, 0x634ce9add3b26ea, 0x117d7f04af0118, 0xda3f19c5d66258a, 0x7094f6fdd0e3912, 0xb87427e85e71560, 0x621b576dbeb5b6a, 0xc8b0b085b8c4e0a, 0x631c871de464c72, 0x1538fc8649458a, 0x13c808a011a1ee0, 0xcddbca6d1cfe360, 0xdb28cee59882d6a, 0xae80f550d1ffff2, 0x113d7984aa0272, 0xda7770f5f195912, 0x71d34fcdbc9458a, 0x137c8a049a1ee0, 0x719776b5a0b6a98, 0xded39a9d236ba78, 0xc8a0be15e917472, 0x6732488ca7ce0a, 0x6633498dedb760a, 0xc0406d0527cb80a, 0x129fa6417f23380, 0x3d4ae9eac3e756a, 0xbd4ae9eac3e7560, 0, 0x12dfcb4458e8b80}); +typedef Field<uint64_t, 61, 39, StatTable61, DynTable61, &SQR_TABLE_61, &QRT_TABLE_61> Field61; +#endif + +#ifdef ENABLE_FIELD_INT_62 +// 62 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable62; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable62; +constexpr StatTable62 SQR_TABLE_62({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x20000001, 0x80000004, 0x200000010, 0x800000040, 0x2000000100, 0x8000000400, 0x20000001000, 0x80000004000, 0x200000010000, 0x800000040000, 0x2000000100000, 0x8000000400000, 0x20000001000000, 0x80000004000000, 0x200000010000000, 0x800000040000000, 0x2000000100000000, 0x440000002, 0x1100000008, 0x4400000020, 0x11000000080, 0x44000000200, 0x110000000800, 0x440000002000, 0x1100000008000, 0x4400000020000, 0x11000000080000, 0x44000000200000, 0x110000000800000, 0x440000002000000, 0x1100000008000000}); +constexpr StatTable62 QRT_TABLE_62({0x30268b6fba455d2c, 0x200000006, 0x200000004, 0x3d67cb6c1fe66c76, 0x200000000, 0x3fc4f1901abfa400, 0x3d67cb6c1fe66c7e, 0x35e79b6c0a66bcbe, 0x200000010, 0x1e9372bc57a9941e, 0x3fc4f1901abfa420, 0x21ec9d424957a5b0, 0x3d67cb6c1fe66c3e, 0x1cb35a6e52f5fb0e, 0x35e79b6c0a66bc3e, 0x215481024c13a730, 0x200000110, 0x1c324a6c52f75b08, 0x1e9372bc57a9961e, 0x3764a9d00f676820, 0x3fc4f1901abfa020, 0x355481020e132730, 0x21ec9d424957adb0, 0x3c43c32c0f34301e, 0x3d67cb6c1fe67c3e, 0x1496122c45259728, 0x1cb35a6e52f5db0e, 0x15e418405b72ec20, 0x35e79b6c0a66fc3e, 0x30268b6e3a445c38, 0x215481024c132730, 0x100010114, 0x200010110, 0, 0x1c324a6c52f55b08, 0x215581044d133776, 0x1e9372bc57ad961e, 0x2155810e4d133766, 0x3764a9d00f6f6820, 0x2157833c4d12323e, 0x3fc4f1901aafa020, 0x1c324a4252f55b58, 0x355481020e332730, 0x28332fc0509d41e, 0x21ec9d424917adb0, 0x215783be4d12332e, 0x3c43c32c0fb4301e, 0x2157822c4d06363e, 0x3d67cb6c1ee67c3e, 0x23f6b9d2484afb78, 0x1496122c47259728, 0x14b8184047648a80, 0x1cb35a6e56f5db0e, 0x3fe4f1901aefa820, 0x15e418405372ec20, 0x3d5fd72c1be276be, 0x35e79b6c1a66fc3e, 0x14b038d24774cf10, 0x30268b6e1a445c38, 0x1d17022e43a7172e, 0x215481020c132730, 0x2157022e4d07372e}); +typedef Field<uint64_t, 62, 536870913, StatTable62, DynTable62, &SQR_TABLE_62, &QRT_TABLE_62> Field62; +#endif + +#ifdef ENABLE_FIELD_INT_63 +// 63 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable63; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable63; +constexpr StatTable63 SQR_TABLE_63({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4000000000000000, 0x6, 0x18, 0x60, 0x180, 0x600, 0x1800, 0x6000, 0x18000, 0x60000, 0x180000, 0x600000, 0x1800000, 0x6000000, 0x18000000, 0x60000000, 0x180000000, 0x600000000, 0x1800000000, 0x6000000000, 0x18000000000, 0x60000000000, 0x180000000000, 0x600000000000, 0x1800000000000, 0x6000000000000, 0x18000000000000, 0x60000000000000, 0x180000000000000, 0x600000000000000, 0x1800000000000000, 0x6000000000000000}); +constexpr StatTable63 QRT_TABLE_63({0, 0x100010114, 0x100010116, 0x1001701051372, 0x100010112, 0x1000040220, 0x100170105137a, 0x5107703453bba, 0x100010102, 0x101130117155a, 0x1000040200, 0x40000200800, 0x100170105133a, 0x103151a137276d8, 0x5107703453b3a, 0x134e65fc7c222be0, 0x100010002, 0x100030103115a, 0x101130117175a, 0x106052d103f4de2, 0x1000040600, 0x15122707691d3a, 0x40000200000, 0x4530770bc57b3a, 0x100170105033a, 0x103011a131256d8, 0x103151a137256d8, 0x176f29eb55c7a8da, 0x5107703457b3a, 0x130b158b7767d0da, 0x134e65fc7c22abe0, 0x7bcaf59d2f62d3e2, 0x100000002, 0x1001401041260, 0x100030101115a, 0x5107e03443ab8, 0x101130113175a, 0x1043701251b3a, 0x106052d10374de2, 0x134e657d7c232be2, 0x1000140600, 0x106073d103b4be2, 0x15122707491d3a, 0x4438600ac07800, 0x40000600000, 0x176a199c5682d3e0, 0x4530770b457b3a, 0x7bca759c2f62d3e0, 0x100170005033a, 0x6116d02572de2, 0x103011a111256d8, 0x1346656d7c372de2, 0x103151a177256d8, 0x643c600aa07800, 0x176f29eb5dc7a8da, 0x7b4b758b2f67d0da, 0x5107713457b3a, 0x104570776b457b3a, 0x130b158b5767d0da, 0x734e65fc3c22abe0, 0x134e65fc3c22abe0, 0x4000000000000000, 0x7bcaf59daf62d3e2}); +typedef Field<uint64_t, 63, 3, StatTable63, DynTable63, &SQR_TABLE_63, &QRT_TABLE_63> Field63; +#endif + +#ifdef ENABLE_FIELD_INT_64 +// 64 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable64; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable64; +constexpr StatTable64 SQR_TABLE_64({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4000000000000000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0x1b00000, 0x6c00000, 0x1b000000, 0x6c000000, 0x1b0000000, 0x6c0000000, 0x1b00000000, 0x6c00000000, 0x1b000000000, 0x6c000000000, 0x1b0000000000, 0x6c0000000000, 0x1b00000000000, 0x6c00000000000, 0x1b000000000000, 0x6c000000000000, 0x1b0000000000000, 0x6c0000000000000, 0x1b00000000000000, 0x6c00000000000000, 0xb00000000000001b, 0xc00000000000005a}); +constexpr StatTable64 QRT_TABLE_64({0x19c9369f278adc02, 0x84b2b22ab2383ee4, 0x84b2b22ab2383ee6, 0x9d7b84b495b3e3f6, 0x84b2b22ab2383ee2, 0x37c470b49213f790, 0x9d7b84b495b3e3fe, 0x1000a0105137c, 0x84b2b22ab2383ef2, 0x368e964a8edce1fc, 0x37c470b49213f7b0, 0x19c9368e278fdf4c, 0x9d7b84b495b3e3be, 0x2e4da23cbc7d4570, 0x1000a010513fc, 0x84f35772bac24232, 0x84b2b22ab2383ff2, 0x37c570ba9314e4fc, 0x368e964a8edce3fc, 0xb377c390213cdb0e, 0x37c470b49213f3b0, 0x85ed5a3aa99c24f2, 0x19c9368e278fd74c, 0xaabff0000780000e, 0x9d7b84b495b3f3be, 0x84b6b3dab03038f2, 0x2e4da23cbc7d6570, 0x511ea03494ffc, 0x1000a010553fc, 0xae0c0220343c6c0e, 0x84f35772bac2c232, 0x800000008000000e, 0x84b2b22ab2393ff2, 0xb376c29c202bc97e, 0x37c570ba9316e4fc, 0x9c3062488879e6ce, 0x368e964a8ed8e3fc, 0x41e42c08e47e70, 0xb377c3902134db0e, 0x85b9b108a60f56ce, 0x37c470b49203f3b0, 0x19dd3b6e21f3cb4c, 0x85ed5a3aa9bc24f2, 0x198ddf682c428ac0, 0x19c9368e27cfd74c, 0x4b7c68431ca84b0, 0xaabff0000700000e, 0x8040655489ffefbe, 0x9d7b84b494b3f3be, 0x18c1354e32bfa74c, 0x84b6b3dab23038f2, 0xaaf613cc0f74627e, 0x2e4da23cb87d6570, 0x3248b3d6b3342a8c, 0x511ea0b494ffc, 0xb60813c00e70700e, 0x1000a110553fc, 0x1e0d022a05393ffc, 0xae0c0220143c6c0e, 0xe0c0220143c6c00, 0x84f35772fac2c232, 0xc041e55948fbfdce, 0x800000000000000e, 0}); +typedef Field<uint64_t, 64, 27, StatTable64, DynTable64, &SQR_TABLE_64, &QRT_TABLE_64> Field64; +#endif +} + +Sketch* ConstructGeneric8Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_57 + case 57: return new SketchImpl<Field57>(implementation, 57); +#endif +#ifdef ENABLE_FIELD_INT_58 + case 58: return new SketchImpl<Field58>(implementation, 58); +#endif +#ifdef ENABLE_FIELD_INT_59 + case 59: return new SketchImpl<Field59>(implementation, 59); +#endif +#ifdef ENABLE_FIELD_INT_60 + case 60: return new SketchImpl<Field60>(implementation, 60); +#endif +#ifdef ENABLE_FIELD_INT_61 + case 61: return new SketchImpl<Field61>(implementation, 61); +#endif +#ifdef ENABLE_FIELD_INT_62 + case 62: return new SketchImpl<Field62>(implementation, 62); +#endif +#ifdef ENABLE_FIELD_INT_63 + case 63: return new SketchImpl<Field63>(implementation, 63); +#endif +#ifdef ENABLE_FIELD_INT_64 + case 64: return new SketchImpl<Field64>(implementation, 64); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_common_impl.h b/src/minisketch/src/fields/generic_common_impl.h new file mode 100644 index 0000000000..1d1d030b5f --- /dev/null +++ b/src/minisketch/src/fields/generic_common_impl.h @@ -0,0 +1,70 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_FIELDS_GENERIC_COMMON_IMPL_H_ +#define _MINISKETCH_FIELDS_GENERIC_COMMON_IMPL_H_ 1 + +#include <stdint.h> + +#include "../int_utils.h" +#include "../lintrans.h" + +namespace { + +/** Generic implementation for fields whose elements can be represented by an integer type. */ +template<typename I, int B, uint32_t MOD, typename F, typename T, const F* SQR, const F* QRT> class Field +{ + typedef BitsInt<I, B> O; + typedef LFSR<O, MOD> L; + +public: + typedef I Elem; + constexpr int Bits() const { return B; } + + constexpr inline Elem Mul2(Elem val) const { return L::Call(val); } + + class Multiplier + { + T table; + public: + explicit Multiplier(const Field&, Elem a) { table.template Build<L::Call>(a); } + constexpr inline Elem operator()(Elem a) const { return table.template Map<O>(a); } + }; + + Elem Mul(Elem a, Elem b) const { return GFMul<I, B, L, O>(a, b); } + + /** Compute the square of a. */ + inline constexpr Elem Sqr(Elem a) const { return SQR->template Map<O>(a); } + + /** Compute x such that x^2 + x = a (undefined result if no solution exists). */ + inline constexpr Elem Qrt(Elem a) const { return QRT->template Map<O>(a); } + + /** Compute the inverse of x1. */ + Elem Inv(Elem a) const { return InvExtGCD<I, O, B, MOD>(a); } + + /** Generate a random field element. */ + Elem FromSeed(uint64_t seed) const { + uint64_t k0 = 0x496e744669656c64ull; // "IntField" + uint64_t k1 = seed; + uint64_t count = ((uint64_t)B) << 32; + Elem ret; + do { + ret = O::Mask(I(SipHash(k0, k1, count++))); + } while(ret == 0); + return ret; + } + + Elem Deserialize(BitReader& in) const { return in.template Read<B, I>(); } + + void Serialize(BitWriter& out, Elem val) const { out.template Write<B, I>(val); } + + constexpr Elem FromUint64(uint64_t x) const { return O::Mask(I(x)); } + constexpr uint64_t ToUint64(Elem val) const { return uint64_t(val); } +}; + +} + +#endif diff --git a/src/minisketch/src/int_utils.h b/src/minisketch/src/int_utils.h new file mode 100644 index 0000000000..62b2c38a29 --- /dev/null +++ b/src/minisketch/src/int_utils.h @@ -0,0 +1,290 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_INT_UTILS_H_ +#define _MINISKETCH_INT_UTILS_H_ + +#include <stdlib.h> + +#include <limits> +#include <algorithm> +#include <type_traits> + +#ifdef _MSC_VER +# include <intrin.h> +#endif + +template<int bits> +static constexpr inline uint64_t Rot(uint64_t x) { return (x << bits) | (x >> (64 - bits)); } + +static inline void SipHashRound(uint64_t& v0, uint64_t& v1, uint64_t& v2, uint64_t& v3) { + v0 += v1; v1 = Rot<13>(v1); v1 ^= v0; + v0 = Rot<32>(v0); + v2 += v3; v3 = Rot<16>(v3); v3 ^= v2; + v0 += v3; v3 = Rot<21>(v3); v3 ^= v0; + v2 += v1; v1 = Rot<17>(v1); v1 ^= v2; + v2 = Rot<32>(v2); +} + +inline uint64_t SipHash(uint64_t k0, uint64_t k1, uint64_t data) { + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ data; + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + v0 ^= data; + v3 ^= 0x800000000000000ULL; + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + v0 ^= 0x800000000000000ULL; + v2 ^= 0xFF; + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + return v0 ^ v1 ^ v2 ^ v3; +} + +class BitWriter { + unsigned char state = 0; + int offset = 0; + unsigned char* out; + +public: + BitWriter(unsigned char* output) : out(output) {} + + template<int BITS, typename I> + inline void Write(I val) { + int bits = BITS; + if (bits + offset >= 8) { + state |= ((val & ((I(1) << (8 - offset)) - 1)) << offset); + *(out++) = state; + val >>= (8 - offset); + bits -= 8 - offset; + offset = 0; + state = 0; + } + while (bits >= 8) { + *(out++) = val & 255; + val >>= 8; + bits -= 8; + } + state |= ((val & ((I(1) << bits) - 1)) << offset); + offset += bits; + } + + inline void Flush() { + if (offset) { + *(out++) = state; + state = 0; + offset = 0; + } + } +}; + +class BitReader { + unsigned char state = 0; + int offset = 0; + const unsigned char* in; + +public: + BitReader(const unsigned char* input) : in(input) {} + + template<int BITS, typename I> + inline I Read() { + int bits = BITS; + if (offset >= bits) { + I ret = state & ((1 << bits) - 1); + state >>= bits; + offset -= bits; + return ret; + } + I val = state; + int out = offset; + while (out + 8 <= bits) { + val |= ((I(*(in++))) << out); + out += 8; + } + if (out < bits) { + unsigned char c = *(in++); + val |= (c & ((I(1) << (bits - out)) - 1)) << out; + state = c >> (bits - out); + offset = 8 - (bits - out); + } else { + state = 0; + offset = 0; + } + return val; + } +}; + +/** Return a value of type I with its `bits` lowest bits set (bits must be > 0). */ +template<int BITS, typename I> +constexpr inline I Mask() { return ((I((I(-1)) << (std::numeric_limits<I>::digits - BITS))) >> (std::numeric_limits<I>::digits - BITS)); } + +/** Compute the smallest power of two that is larger than val. */ +template<typename I> +static inline int CountBits(I val, int max) { +#ifdef HAVE_CLZ + (void)max; + if (val == 0) return 0; + if (std::numeric_limits<unsigned>::digits >= std::numeric_limits<I>::digits) { + return std::numeric_limits<unsigned>::digits - __builtin_clz(val); + } else if (std::numeric_limits<unsigned long>::digits >= std::numeric_limits<I>::digits) { + return std::numeric_limits<unsigned long>::digits - __builtin_clzl(val); + } else { + return std::numeric_limits<unsigned long long>::digits - __builtin_clzll(val); + } +#elif _MSC_VER + (void)max; + unsigned long index; + unsigned char ret; + if (std::numeric_limits<I>::digits <= 32) { + ret = _BitScanReverse(&index, val); + } else { + ret = _BitScanReverse64(&index, val); + } + if (!ret) return 0; + return index; +#else + while (max && (val >> (max - 1) == 0)) --max; + return max; +#endif +} + +template<typename I, int BITS> +class BitsInt { +private: + static_assert(std::is_integral<I>::value && std::is_unsigned<I>::value, "BitsInt requires an unsigned integer type"); + static_assert(BITS > 0 && BITS <= std::numeric_limits<I>::digits, "BitsInt requires 1 <= Bits <= representation type size"); + + static constexpr I MASK = Mask<BITS, I>(); + +public: + + typedef I Repr; + + static constexpr int SIZE = BITS; + + static void inline Swap(I& a, I& b) { + std::swap(a, b); + } + + static constexpr inline bool IsZero(I a) { return a == 0; } + static constexpr inline I Mask(I val) { return val & MASK; } + static constexpr inline I Shift(I val, int bits) { return ((val << bits) & MASK); } + static constexpr inline I UnsafeShift(I val, int bits) { return (val << bits); } + + template<int Offset, int Count> + static constexpr inline int MidBits(I val) { + static_assert(Count > 0, "BITSInt::MidBits needs Count > 0"); + static_assert(Count + Offset <= BITS, "BitsInt::MidBits overflow of Count+Offset"); + return (val >> Offset) & ((I(1) << Count) - 1); + } + + template<int Count> + static constexpr inline int TopBits(I val) { + static_assert(Count > 0, "BitsInt::TopBits needs Count > 0"); + static_assert(Count <= BITS, "BitsInt::TopBits needs Offset <= BITS"); + return val >> (BITS - Count); + } + + static inline constexpr I CondXorWith(I val, bool cond, I v) { + return val ^ (-I(cond) & v); + } + + template<I MOD> + static inline constexpr I CondXorWith(I val, bool cond) { + return val ^ (-I(cond) & MOD); + } + + static inline int Bits(I val, int max) { return CountBits<I>(val, max); } +}; + +/** Class which implements a stateless LFSR for generic moduli. */ +template<typename F, uint32_t MOD> +struct LFSR { + typedef typename F::Repr I; + /** Shift a value `a` up once, treating it as an `N`-bit LFSR, with pattern `MOD`. */ + static inline constexpr I Call(const I& a) { + return F::template CondXorWith<MOD>(F::Shift(a, 1), F::template TopBits<1>(a)); + } +}; + +/** Helper class for carryless multiplications. */ +template<typename I, int N, typename L, typename F, int K> struct GFMulHelper; +template<typename I, int N, typename L, typename F> struct GFMulHelper<I, N, L, F, 0> +{ + static inline constexpr I Run(const I& a, const I& b) { return I(0); } +}; +template<typename I, int N, typename L, typename F, int K> struct GFMulHelper +{ + static inline constexpr I Run(const I& a, const I& b) { return F::CondXorWith(GFMulHelper<I, N, L, F, K - 1>::Run(L::Call(a), b), F::template MidBits<N - K, 1>(b), a); } +}; + +/** Compute the carry-less multiplication of a and b, with N bits, using L as LFSR type. */ +template<typename I, int N, typename L, typename F> inline constexpr I GFMul(const I& a, const I& b) { return GFMulHelper<I, N, L, F, N>::Run(a, b); } + +/** Compute the inverse of x using an extgcd algorithm. */ +template<typename I, typename F, int BITS, uint32_t MOD> +inline I InvExtGCD(I x) +{ + if (F::IsZero(x)) return x; + I t(0), newt(1); + I r(MOD), newr = x; + int rlen = BITS + 1, newrlen = F::Bits(newr, BITS); + while (newr) { + int q = rlen - newrlen; + r ^= F::Shift(newr, q); + t ^= F::UnsafeShift(newt, q); + rlen = F::Bits(r, rlen - 1); + if (r < newr) { + F::Swap(t, newt); + F::Swap(r, newr); + std::swap(rlen, newrlen); + } + } + return t; +} + +/** Compute the inverse of x1 using an exponentiation ladder. + * + * The `MUL` argument is a multiplication function, `SQR` is a squaring function, and the `SQRi` arguments + * compute x**(2**i). + */ +template<typename I, typename F, int BITS, I (*MUL)(I, I), I (*SQR)(I), I (*SQR2)(I), I(*SQR4)(I), I(*SQR8)(I), I(*SQR16)(I)> +inline I InvLadder(I x1) +{ + static constexpr int INV_EXP = BITS - 1; + I x2 = (INV_EXP >= 2) ? MUL(SQR(x1), x1) : I(); + I x4 = (INV_EXP >= 4) ? MUL(SQR2(x2), x2) : I(); + I x8 = (INV_EXP >= 8) ? MUL(SQR4(x4), x4) : I(); + I x16 = (INV_EXP >= 16) ? MUL(SQR8(x8), x8) : I(); + I x32 = (INV_EXP >= 32) ? MUL(SQR16(x16), x16) : I(); + I r; + if (INV_EXP >= 32) { + r = x32; + } else if (INV_EXP >= 16) { + r = x16; + } else if (INV_EXP >= 8) { + r = x8; + } else if (INV_EXP >= 4) { + r = x4; + } else if (INV_EXP >= 2) { + r = x2; + } else { + r = x1; + } + if (INV_EXP >= 32 && (INV_EXP & 16)) r = MUL(SQR16(r), x16); + if (INV_EXP >= 16 && (INV_EXP & 8)) r = MUL(SQR8(r), x8); + if (INV_EXP >= 8 && (INV_EXP & 4)) r = MUL(SQR4(r), x4); + if (INV_EXP >= 4 && (INV_EXP & 2)) r = MUL(SQR2(r), x2); + if (INV_EXP >= 2 && (INV_EXP & 1)) r = MUL(SQR(r), x1); + return SQR(r); +} + +#endif diff --git a/src/minisketch/src/lintrans.h b/src/minisketch/src/lintrans.h new file mode 100644 index 0000000000..b9d8ea8e1d --- /dev/null +++ b/src/minisketch/src/lintrans.h @@ -0,0 +1,150 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_LINTRANS_H_ +#define _MINISKETCH_LINTRANS_H_ + +#include "int_utils.h" + +/** A type to represent integers in the type system. */ +template<int N> struct Num {}; + +/** A Linear N-bit transformation over the field I. */ +template<typename I, int N> class LinTrans { +private: + I table[1 << N]; +public: + LinTrans() = default; + + /* Construct a transformation over 3 to 8 bits, using the images of each bit. */ + constexpr LinTrans(I a, I b) : table{I(0), I(a), I(b), I(a ^ b)} {} + constexpr LinTrans(I a, I b, I c) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c)} {} + constexpr LinTrans(I a, I b, I c, I d) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d)} {} + constexpr LinTrans(I a, I b, I c, I d, I e) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d), I(e), I(a ^ e), I(b ^ e), I(a ^ b ^ e), I(c ^ e), I(a ^ c ^ e), I(b ^ c ^ e), I(a ^ b ^ c ^ e), I(d ^ e), I(a ^ d ^ e), I(b ^ d ^ e), I(a ^ b ^ d ^ e), I(c ^ d ^ e), I(a ^ c ^ d ^ e), I(b ^ c ^ d ^ e), I(a ^ b ^ c ^ d ^ e)} {} + constexpr LinTrans(I a, I b, I c, I d, I e, I f) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d), I(e), I(a ^ e), I(b ^ e), I(a ^ b ^ e), I(c ^ e), I(a ^ c ^ e), I(b ^ c ^ e), I(a ^ b ^ c ^ e), I(d ^ e), I(a ^ d ^ e), I(b ^ d ^ e), I(a ^ b ^ d ^ e), I(c ^ d ^ e), I(a ^ c ^ d ^ e), I(b ^ c ^ d ^ e), I(a ^ b ^ c ^ d ^ e), I(f), I(a ^ f), I(b^ f), I(a ^ b ^ f), I(c^ f), I(a ^ c ^ f), I(b ^ c ^ f), I(a ^ b ^ c ^ f), I(d ^ f), I(a ^ d ^ f), I(b ^ d ^ f), I(a ^ b ^ d ^ f), I(c ^ d ^ f), I(a ^ c ^ d ^ f), I(b ^ c ^ d ^ f), I(a ^ b ^ c ^ d ^ f), I(e ^ f), I(a ^ e ^ f), I(b ^ e ^ f), I(a ^ b ^ e ^ f), I(c ^ e ^ f), I(a ^ c ^ e ^ f), I(b ^ c ^ e ^ f), I(a ^ b ^ c ^ e ^ f), I(d ^ e ^ f), I(a ^ d ^ e ^ f), I(b ^ d ^ e ^ f), I(a ^ b ^ d ^ e ^ f), I(c ^ d ^ e ^ f), I(a ^ c ^ d ^ e ^ f), I(b ^ c ^ d ^ e ^ f), I(a ^ b ^ c ^ d ^ e ^ f)} {} + constexpr LinTrans(I a, I b, I c, I d, I e, I f, I g) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d), I(e), I(a ^ e), I(b ^ e), I(a ^ b ^ e), I(c ^ e), I(a ^ c ^ e), I(b ^ c ^ e), I(a ^ b ^ c ^ e), I(d ^ e), I(a ^ d ^ e), I(b ^ d ^ e), I(a ^ b ^ d ^ e), I(c ^ d ^ e), I(a ^ c ^ d ^ e), I(b ^ c ^ d ^ e), I(a ^ b ^ c ^ d ^ e), I(f), I(a ^ f), I(b^ f), I(a ^ b ^ f), I(c^ f), I(a ^ c ^ f), I(b ^ c ^ f), I(a ^ b ^ c ^ f), I(d ^ f), I(a ^ d ^ f), I(b ^ d ^ f), I(a ^ b ^ d ^ f), I(c ^ d ^ f), I(a ^ c ^ d ^ f), I(b ^ c ^ d ^ f), I(a ^ b ^ c ^ d ^ f), I(e ^ f), I(a ^ e ^ f), I(b ^ e ^ f), I(a ^ b ^ e ^ f), I(c ^ e ^ f), I(a ^ c ^ e ^ f), I(b ^ c ^ e ^ f), I(a ^ b ^ c ^ e ^ f), I(d ^ e ^ f), I(a ^ d ^ e ^ f), I(b ^ d ^ e ^ f), I(a ^ b ^ d ^ e ^ f), I(c ^ d ^ e ^ f), I(a ^ c ^ d ^ e ^ f), I(b ^ c ^ d ^ e ^ f), I(a ^ b ^ c ^ d ^ e ^ f), I(g), I(a ^ g), I(b ^ g), I(a ^ b ^ g), I(c ^ g), I(a ^ c ^ g), I(b ^ c ^ g), I(a ^ b ^ c ^ g), I(d ^ g), I(a ^ d ^ g), I(b ^ d ^ g), I(a ^ b ^ d ^ g), I(c ^ d ^ g), I(a ^ c ^ d ^ g), I(b ^ c ^ d ^ g), I(a ^ b ^ c ^ d ^ g), I(e ^ g), I(a ^ e ^ g), I(b ^ e ^ g), I(a ^ b ^ e ^ g), I(c ^ e ^ g), I(a ^ c ^ e ^ g), I(b ^ c ^ e ^ g), I(a ^ b ^ c ^ e ^ g), I(d ^ e ^ g), I(a ^ d ^ e ^ g), I(b ^ d ^ e ^ g), I(a ^ b ^ d ^ e ^ g), I(c ^ d ^ e ^ g), I(a ^ c ^ d ^ e ^ g), I(b ^ c ^ d ^ e ^ g), I(a ^ b ^ c ^ d ^ e ^ g), I(f ^ g), I(a ^ f ^ g), I(b^ f ^ g), I(a ^ b ^ f ^ g), I(c^ f ^ g), I(a ^ c ^ f ^ g), I(b ^ c ^ f ^ g), I(a ^ b ^ c ^ f ^ g), I(d ^ f ^ g), I(a ^ d ^ f ^ g), I(b ^ d ^ f ^ g), I(a ^ b ^ d ^ f ^ g), I(c ^ d ^ f ^ g), I(a ^ c ^ d ^ f ^ g), I(b ^ c ^ d ^ f ^ g), I(a ^ b ^ c ^ d ^ f ^ g), I(e ^ f ^ g), I(a ^ e ^ f ^ g), I(b ^ e ^ f ^ g), I(a ^ b ^ e ^ f ^ g), I(c ^ e ^ f ^ g), I(a ^ c ^ e ^ f ^ g), I(b ^ c ^ e ^ f ^ g), I(a ^ b ^ c ^ e ^ f ^ g), I(d ^ e ^ f ^ g), I(a ^ d ^ e ^ f ^ g), I(b ^ d ^ e ^ f ^ g), I(a ^ b ^ d ^ e ^ f ^ g), I(c ^ d ^ e ^ f ^ g), I(a ^ c ^ d ^ e ^ f ^ g), I(b ^ c ^ d ^ e ^ f ^ g), I(a ^ b ^ c ^ d ^ e ^ f ^ g)} {} + constexpr LinTrans(I a, I b, I c, I d, I e, I f, I g, I h) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d), I(e), I(a ^ e), I(b ^ e), I(a ^ b ^ e), I(c ^ e), I(a ^ c ^ e), I(b ^ c ^ e), I(a ^ b ^ c ^ e), I(d ^ e), I(a ^ d ^ e), I(b ^ d ^ e), I(a ^ b ^ d ^ e), I(c ^ d ^ e), I(a ^ c ^ d ^ e), I(b ^ c ^ d ^ e), I(a ^ b ^ c ^ d ^ e), I(f), I(a ^ f), I(b^ f), I(a ^ b ^ f), I(c^ f), I(a ^ c ^ f), I(b ^ c ^ f), I(a ^ b ^ c ^ f), I(d ^ f), I(a ^ d ^ f), I(b ^ d ^ f), I(a ^ b ^ d ^ f), I(c ^ d ^ f), I(a ^ c ^ d ^ f), I(b ^ c ^ d ^ f), I(a ^ b ^ c ^ d ^ f), I(e ^ f), I(a ^ e ^ f), I(b ^ e ^ f), I(a ^ b ^ e ^ f), I(c ^ e ^ f), I(a ^ c ^ e ^ f), I(b ^ c ^ e ^ f), I(a ^ b ^ c ^ e ^ f), I(d ^ e ^ f), I(a ^ d ^ e ^ f), I(b ^ d ^ e ^ f), I(a ^ b ^ d ^ e ^ f), I(c ^ d ^ e ^ f), I(a ^ c ^ d ^ e ^ f), I(b ^ c ^ d ^ e ^ f), I(a ^ b ^ c ^ d ^ e ^ f), I(g), I(a ^ g), I(b ^ g), I(a ^ b ^ g), I(c ^ g), I(a ^ c ^ g), I(b ^ c ^ g), I(a ^ b ^ c ^ g), I(d ^ g), I(a ^ d ^ g), I(b ^ d ^ g), I(a ^ b ^ d ^ g), I(c ^ d ^ g), I(a ^ c ^ d ^ g), I(b ^ c ^ d ^ g), I(a ^ b ^ c ^ d ^ g), I(e ^ g), I(a ^ e ^ g), I(b ^ e ^ g), I(a ^ b ^ e ^ g), I(c ^ e ^ g), I(a ^ c ^ e ^ g), I(b ^ c ^ e ^ g), I(a ^ b ^ c ^ e ^ g), I(d ^ e ^ g), I(a ^ d ^ e ^ g), I(b ^ d ^ e ^ g), I(a ^ b ^ d ^ e ^ g), I(c ^ d ^ e ^ g), I(a ^ c ^ d ^ e ^ g), I(b ^ c ^ d ^ e ^ g), I(a ^ b ^ c ^ d ^ e ^ g), I(f ^ g), I(a ^ f ^ g), I(b^ f ^ g), I(a ^ b ^ f ^ g), I(c^ f ^ g), I(a ^ c ^ f ^ g), I(b ^ c ^ f ^ g), I(a ^ b ^ c ^ f ^ g), I(d ^ f ^ g), I(a ^ d ^ f ^ g), I(b ^ d ^ f ^ g), I(a ^ b ^ d ^ f ^ g), I(c ^ d ^ f ^ g), I(a ^ c ^ d ^ f ^ g), I(b ^ c ^ d ^ f ^ g), I(a ^ b ^ c ^ d ^ f ^ g), I(e ^ f ^ g), I(a ^ e ^ f ^ g), I(b ^ e ^ f ^ g), I(a ^ b ^ e ^ f ^ g), I(c ^ e ^ f ^ g), I(a ^ c ^ e ^ f ^ g), I(b ^ c ^ e ^ f ^ g), I(a ^ b ^ c ^ e ^ f ^ g), I(d ^ e ^ f ^ g), I(a ^ d ^ e ^ f ^ g), I(b ^ d ^ e ^ f ^ g), I(a ^ b ^ d ^ e ^ f ^ g), I(c ^ d ^ e ^ f ^ g), I(a ^ c ^ d ^ e ^ f ^ g), I(b ^ c ^ d ^ e ^ f ^ g), I(a ^ b ^ c ^ d ^ e ^ f ^ g), I(h), I(a ^ h), I(b ^ h), I(a ^ b ^ h), I(c ^ h), I(a ^ c ^ h), I(b ^ c ^ h), I(a ^ b ^ c ^ h), I(d ^ h), I(a ^ d ^ h), I(b ^ d ^ h), I(a ^ b ^ d ^ h), I(c ^ d ^ h), I(a ^ c ^ d ^ h), I(b ^ c ^ d ^ h), I(a ^ b ^ c ^ d ^ h), I(e ^ h), I(a ^ e ^ h), I(b ^ e ^ h), I(a ^ b ^ e ^ h), I(c ^ e ^ h), I(a ^ c ^ e ^ h), I(b ^ c ^ e ^ h), I(a ^ b ^ c ^ e ^ h), I(d ^ e ^ h), I(a ^ d ^ e ^ h), I(b ^ d ^ e ^ h), I(a ^ b ^ d ^ e ^ h), I(c ^ d ^ e ^ h), I(a ^ c ^ d ^ e ^ h), I(b ^ c ^ d ^ e ^ h), I(a ^ b ^ c ^ d ^ e ^ h), I(f ^ h), I(a ^ f ^ h), I(b^ f ^ h), I(a ^ b ^ f ^ h), I(c^ f ^ h), I(a ^ c ^ f ^ h), I(b ^ c ^ f ^ h), I(a ^ b ^ c ^ f ^ h), I(d ^ f ^ h), I(a ^ d ^ f ^ h), I(b ^ d ^ f ^ h), I(a ^ b ^ d ^ f ^ h), I(c ^ d ^ f ^ h), I(a ^ c ^ d ^ f ^ h), I(b ^ c ^ d ^ f ^ h), I(a ^ b ^ c ^ d ^ f ^ h), I(e ^ f ^ h), I(a ^ e ^ f ^ h), I(b ^ e ^ f ^ h), I(a ^ b ^ e ^ f ^ h), I(c ^ e ^ f ^ h), I(a ^ c ^ e ^ f ^ h), I(b ^ c ^ e ^ f ^ h), I(a ^ b ^ c ^ e ^ f ^ h), I(d ^ e ^ f ^ h), I(a ^ d ^ e ^ f ^ h), I(b ^ d ^ e ^ f ^ h), I(a ^ b ^ d ^ e ^ f ^ h), I(c ^ d ^ e ^ f ^ h), I(a ^ c ^ d ^ e ^ f ^ h), I(b ^ c ^ d ^ e ^ f ^ h), I(a ^ b ^ c ^ d ^ e ^ f ^ h), I(g ^ h), I(a ^ g ^ h), I(b ^ g ^ h), I(a ^ b ^ g ^ h), I(c ^ g ^ h), I(a ^ c ^ g ^ h), I(b ^ c ^ g ^ h), I(a ^ b ^ c ^ g ^ h), I(d ^ g ^ h), I(a ^ d ^ g ^ h), I(b ^ d ^ g ^ h), I(a ^ b ^ d ^ g ^ h), I(c ^ d ^ g ^ h), I(a ^ c ^ d ^ g ^ h), I(b ^ c ^ d ^ g ^ h), I(a ^ b ^ c ^ d ^ g ^ h), I(e ^ g ^ h), I(a ^ e ^ g ^ h), I(b ^ e ^ g ^ h), I(a ^ b ^ e ^ g ^ h), I(c ^ e ^ g ^ h), I(a ^ c ^ e ^ g ^ h), I(b ^ c ^ e ^ g ^ h), I(a ^ b ^ c ^ e ^ g ^ h), I(d ^ e ^ g ^ h), I(a ^ d ^ e ^ g ^ h), I(b ^ d ^ e ^ g ^ h), I(a ^ b ^ d ^ e ^ g ^ h), I(c ^ d ^ e ^ g ^ h), I(a ^ c ^ d ^ e ^ g ^ h), I(b ^ c ^ d ^ e ^ g ^ h), I(a ^ b ^ c ^ d ^ e ^ g ^ h), I(f ^ g ^ h), I(a ^ f ^ g ^ h), I(b^ f ^ g ^ h), I(a ^ b ^ f ^ g ^ h), I(c^ f ^ g ^ h), I(a ^ c ^ f ^ g ^ h), I(b ^ c ^ f ^ g ^ h), I(a ^ b ^ c ^ f ^ g ^ h), I(d ^ f ^ g ^ h), I(a ^ d ^ f ^ g ^ h), I(b ^ d ^ f ^ g ^ h), I(a ^ b ^ d ^ f ^ g ^ h), I(c ^ d ^ f ^ g ^ h), I(a ^ c ^ d ^ f ^ g ^ h), I(b ^ c ^ d ^ f ^ g ^ h), I(a ^ b ^ c ^ d ^ f ^ g ^ h), I(e ^ f ^ g ^ h), I(a ^ e ^ f ^ g ^ h), I(b ^ e ^ f ^ g ^ h), I(a ^ b ^ e ^ f ^ g ^ h), I(c ^ e ^ f ^ g ^ h), I(a ^ c ^ e ^ f ^ g ^ h), I(b ^ c ^ e ^ f ^ g ^ h), I(a ^ b ^ c ^ e ^ f ^ g ^ h), I(d ^ e ^ f ^ g ^ h), I(a ^ d ^ e ^ f ^ g ^ h), I(b ^ d ^ e ^ f ^ g ^ h), I(a ^ b ^ d ^ e ^ f ^ g ^ h), I(c ^ d ^ e ^ f ^ g ^ h), I(a ^ c ^ d ^ e ^ f ^ g ^ h), I(b ^ c ^ d ^ e ^ f ^ g ^ h), I(a ^ b ^ c ^ d ^ e ^ f ^ g ^ h)} {} + + /* Construct a transformation over 3 to 8 bits, using a pointer to the bit's images. */ + constexpr LinTrans(const I* p, Num<2>) : LinTrans(I(p[0]), I(p[1])) {} + constexpr LinTrans(const I* p, Num<3>) : LinTrans(I(p[0]), I(p[1]), I(p[2])) {} + constexpr LinTrans(const I* p, Num<4>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3])) {} + constexpr LinTrans(const I* p, Num<5>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3]), I(p[4])) {} + constexpr LinTrans(const I* p, Num<6>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3]), I(p[4]), I(p[5])) {} + constexpr LinTrans(const I* p, Num<7>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3]), I(p[4]), I(p[5]), I(p[6])) {} + constexpr LinTrans(const I* p, Num<8>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3]), I(p[4]), I(p[5]), I(p[6]), I(p[7])) {} + + template<I (*F)(const I&)> + inline I Build(Num<1>, I a) + { + table[0] = I(); table[1] = a; + return a; + } + + template<I (*F)(const I&)> + inline I Build(Num<2>, I a) + { + I b = F(a); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; + return b; + } + + template<I (*F)(const I&)> + inline I Build(Num<3>, I a) + { + I b = F(a), c = F(b); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; table[4] = c; table[5] = a ^ c; table[6] = b ^ c; table[7] = a ^ b ^ c; + return c; + } + + template<I (*F)(const I&)> + inline I Build(Num<4>, I a) + { + I b = F(a), c = F(b), d = F(c); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; table[4] = c; table[5] = a ^ c; table[6] = b ^ c; table[7] = a ^ b ^ c; + table[8] = d; table[9] = a ^ d; table[10] = b ^ d; table[11] = a ^ b ^ d; table[12] = c ^ d; table[13] = a ^ c ^ d; table[14] = b ^ c ^ d; table[15] = a ^ b ^ c ^ d; + return d; + } + + template<I (*F)(const I&)> + inline I Build(Num<5>, I a) + { + I b = F(a), c = F(b), d = F(c), e = F(d); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; table[4] = c; table[5] = a ^ c; table[6] = b ^ c; table[7] = a ^ b ^ c; + table[8] = d; table[9] = a ^ d; table[10] = b ^ d; table[11] = a ^ b ^ d; table[12] = c ^ d; table[13] = a ^ c ^ d; table[14] = b ^ c ^ d; table[15] = a ^ b ^ c ^ d; + table[16] = e; table[17] = a ^ e; table[18] = b ^ e; table[19] = a ^ b ^ e; table[20] = c ^ e; table[21] = a ^ c ^ e; table[22] = b ^ c ^ e; table[23] = a ^ b ^ c ^ e; + table[24] = d ^ e; table[25] = a ^ d ^ e; table[26] = b ^ d ^ e; table[27] = a ^ b ^ d ^ e; table[28] = c ^ d ^ e; table[29] = a ^ c ^ d ^ e; table[30] = b ^ c ^ d ^ e; table[31] = a ^ b ^ c ^ d ^ e; + return e; + } + + template<I (*F)(const I&)> + inline I Build(Num<6>, I a) + { + I b = F(a), c = F(b), d = F(c), e = F(d), f = F(e); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; table[4] = c; table[5] = a ^ c; table[6] = b ^ c; table[7] = a ^ b ^ c; + table[8] = d; table[9] = a ^ d; table[10] = b ^ d; table[11] = a ^ b ^ d; table[12] = c ^ d; table[13] = a ^ c ^ d; table[14] = b ^ c ^ d; table[15] = a ^ b ^ c ^ d; + table[16] = e; table[17] = a ^ e; table[18] = b ^ e; table[19] = a ^ b ^ e; table[20] = c ^ e; table[21] = a ^ c ^ e; table[22] = b ^ c ^ e; table[23] = a ^ b ^ c ^ e; + table[24] = d ^ e; table[25] = a ^ d ^ e; table[26] = b ^ d ^ e; table[27] = a ^ b ^ d ^ e; table[28] = c ^ d ^ e; table[29] = a ^ c ^ d ^ e; table[30] = b ^ c ^ d ^ e; table[31] = a ^ b ^ c ^ d ^ e; + table[32] = f; table[33] = a ^ f; table[34] = b ^ f; table[35] = a ^ b ^ f; table[36] = c ^ f; table[37] = a ^ c ^ f; table[38] = b ^ c ^ f; table[39] = a ^ b ^ c ^ f; + table[40] = d ^ f; table[41] = a ^ d ^ f; table[42] = b ^ d ^ f; table[43] = a ^ b ^ d ^ f; table[44] = c ^ d ^ f; table[45] = a ^ c ^ d ^ f; table[46] = b ^ c ^ d ^ f; table[47] = a ^ b ^ c ^ d ^ f; + table[48] = e ^ f; table[49] = a ^ e ^ f; table[50] = b ^ e ^ f; table[51] = a ^ b ^ e ^ f; table[52] = c ^ e ^ f; table[53] = a ^ c ^ e ^ f; table[54] = b ^ c ^ e ^ f; table[55] = a ^ b ^ c ^ e ^ f; + table[56] = d ^ e ^ f; table[57] = a ^ d ^ e ^ f; table[58] = b ^ d ^ e ^ f; table[59] = a ^ b ^ d ^ e ^ f; table[60] = c ^ d ^ e ^ f; table[61] = a ^ c ^ d ^ e ^ f; table[62] = b ^ c ^ d ^ e ^ f; table[63] = a ^ b ^ c ^ d ^ e ^ f; + return f; + } + + template<typename O, int P> + inline I constexpr Map(I a) const { return table[O::template MidBits<P, N>(a)]; } + + template<typename O, int P> + inline I constexpr TopMap(I a) const { static_assert(P + N == O::SIZE, "TopMap inconsistency"); return table[O::template TopBits<N>(a)]; } +}; + + +/** A linear transformation constructed using LinTrans tables for sections of bits. */ +template<typename I, int... N> class RecLinTrans; + +template<typename I, int N> class RecLinTrans<I, N> { + LinTrans<I, N> trans; +public: + static constexpr int BITS = N; + constexpr RecLinTrans(const I* p, Num<BITS>) : trans(p, Num<N>()) {} + constexpr RecLinTrans() = default; + constexpr RecLinTrans(const I (&init)[BITS]) : RecLinTrans(init, Num<BITS>()) {} + + template<typename O, int P = 0> + inline I constexpr Map(I a) const { return trans.template TopMap<O, P>(a); } + + template<I (*F)(const I&)> + inline void Build(I a) { trans.template Build<F>(Num<N>(), a); } +}; + +template<typename I, int N, int... X> class RecLinTrans<I, N, X...> { + LinTrans<I, N> trans; + RecLinTrans<I, X...> rec; +public: + static constexpr int BITS = RecLinTrans<I, X...>::BITS + N; + constexpr RecLinTrans(const I* p, Num<BITS>) : trans(p, Num<N>()), rec(p + N, Num<BITS - N>()) {} + constexpr RecLinTrans() = default; + constexpr RecLinTrans(const I (&init)[BITS]) : RecLinTrans(init, Num<BITS>()) {} + + template<typename O, int P = 0> + inline I constexpr Map(I a) const { return trans.template Map<O, P>(a) ^ rec.template Map<O, P + N>(a); } + + template<I (*F)(const I&)> + inline void Build(I a) { I n = trans.template Build<F>(Num<N>(), a); rec.template Build<F>(F(n)); } +}; + +/** The identity transformation. */ +class IdTrans { +public: + template<typename O, typename I> + inline I constexpr Map(I a) const { return a; } +}; + +/** A singleton for the identity transformation. */ +constexpr IdTrans ID_TRANS{}; + +#endif diff --git a/src/minisketch/src/minisketch.cpp b/src/minisketch/src/minisketch.cpp new file mode 100644 index 0000000000..e9a322f139 --- /dev/null +++ b/src/minisketch/src/minisketch.cpp @@ -0,0 +1,490 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + + +#include <new> + +#define MINISKETCH_BUILD +#ifdef _MINISKETCH_H_ +# error "minisketch.h cannot be included before minisketch.cpp" +#endif +#include "../include/minisketch.h" + +#include "false_positives.h" +#include "fielddefines.h" +#include "sketch.h" + +#ifdef HAVE_CLMUL +# ifdef _MSC_VER +# include <intrin.h> +# else +# include <cpuid.h> +# endif +#endif + +Sketch* ConstructGeneric1Byte(int bits, int implementation); +Sketch* ConstructGeneric2Bytes(int bits, int implementation); +Sketch* ConstructGeneric3Bytes(int bits, int implementation); +Sketch* ConstructGeneric4Bytes(int bits, int implementation); +Sketch* ConstructGeneric5Bytes(int bits, int implementation); +Sketch* ConstructGeneric6Bytes(int bits, int implementation); +Sketch* ConstructGeneric7Bytes(int bits, int implementation); +Sketch* ConstructGeneric8Bytes(int bits, int implementation); + +#ifdef HAVE_CLMUL +Sketch* ConstructClMul1Byte(int bits, int implementation); +Sketch* ConstructClMul2Bytes(int bits, int implementation); +Sketch* ConstructClMul3Bytes(int bits, int implementation); +Sketch* ConstructClMul4Bytes(int bits, int implementation); +Sketch* ConstructClMul5Bytes(int bits, int implementation); +Sketch* ConstructClMul6Bytes(int bits, int implementation); +Sketch* ConstructClMul7Bytes(int bits, int implementation); +Sketch* ConstructClMul8Bytes(int bits, int implementation); +Sketch* ConstructClMulTri1Byte(int bits, int implementation); +Sketch* ConstructClMulTri2Bytes(int bits, int implementation); +Sketch* ConstructClMulTri3Bytes(int bits, int implementation); +Sketch* ConstructClMulTri4Bytes(int bits, int implementation); +Sketch* ConstructClMulTri5Bytes(int bits, int implementation); +Sketch* ConstructClMulTri6Bytes(int bits, int implementation); +Sketch* ConstructClMulTri7Bytes(int bits, int implementation); +Sketch* ConstructClMulTri8Bytes(int bits, int implementation); +#endif + +namespace { + +enum class FieldImpl { + GENERIC = 0, +#ifdef HAVE_CLMUL + CLMUL, + CLMUL_TRI, +#endif +}; + +static inline bool EnableClmul() +{ +#ifdef HAVE_CLMUL +#ifdef _MSC_VER + int regs[4]; + __cpuid(regs, 1); + return (regs[2] & 0x2); +#else + uint32_t eax, ebx, ecx, edx; + return (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & 0x2)); +#endif +#else + return false; +#endif +} + +Sketch* Construct(int bits, int impl) +{ + switch (FieldImpl(impl)) { + case FieldImpl::GENERIC: + switch ((bits + 7) / 8) { + case 1: + return ConstructGeneric1Byte(bits, impl); + case 2: + return ConstructGeneric2Bytes(bits, impl); + case 3: + return ConstructGeneric3Bytes(bits, impl); + case 4: + return ConstructGeneric4Bytes(bits, impl); + case 5: + return ConstructGeneric5Bytes(bits, impl); + case 6: + return ConstructGeneric6Bytes(bits, impl); + case 7: + return ConstructGeneric7Bytes(bits, impl); + case 8: + return ConstructGeneric8Bytes(bits, impl); + default: + return nullptr; + } + break; +#ifdef HAVE_CLMUL + case FieldImpl::CLMUL: + if (EnableClmul()) { + switch ((bits + 7) / 8) { + case 1: + return ConstructClMul1Byte(bits, impl); + case 2: + return ConstructClMul2Bytes(bits, impl); + case 3: + return ConstructClMul3Bytes(bits, impl); + case 4: + return ConstructClMul4Bytes(bits, impl); + case 5: + return ConstructClMul5Bytes(bits, impl); + case 6: + return ConstructClMul6Bytes(bits, impl); + case 7: + return ConstructClMul7Bytes(bits, impl); + case 8: + return ConstructClMul8Bytes(bits, impl); + default: + return nullptr; + } + } + break; + case FieldImpl::CLMUL_TRI: + if (EnableClmul()) { + switch ((bits + 7) / 8) { + case 1: + return ConstructClMulTri1Byte(bits, impl); + case 2: + return ConstructClMulTri2Bytes(bits, impl); + case 3: + return ConstructClMulTri3Bytes(bits, impl); + case 4: + return ConstructClMulTri4Bytes(bits, impl); + case 5: + return ConstructClMulTri5Bytes(bits, impl); + case 6: + return ConstructClMulTri6Bytes(bits, impl); + case 7: + return ConstructClMulTri7Bytes(bits, impl); + case 8: + return ConstructClMulTri8Bytes(bits, impl); + default: + return nullptr; + } + } + break; +#endif + } + return nullptr; +} + +} + +extern "C" { + +int minisketch_bits_supported(uint32_t bits) { +#ifdef ENABLE_FIELD_INT_2 + if (bits == 2) return true; +#endif +#ifdef ENABLE_FIELD_INT_3 + if (bits == 3) return true; +#endif +#ifdef ENABLE_FIELD_INT_4 + if (bits == 4) return true; +#endif +#ifdef ENABLE_FIELD_INT_5 + if (bits == 5) return true; +#endif +#ifdef ENABLE_FIELD_INT_6 + if (bits == 6) return true; +#endif +#ifdef ENABLE_FIELD_INT_7 + if (bits == 7) return true; +#endif +#ifdef ENABLE_FIELD_INT_8 + if (bits == 8) return true; +#endif +#ifdef ENABLE_FIELD_INT_9 + if (bits == 9) return true; +#endif +#ifdef ENABLE_FIELD_INT_10 + if (bits == 10) return true; +#endif +#ifdef ENABLE_FIELD_INT_11 + if (bits == 11) return true; +#endif +#ifdef ENABLE_FIELD_INT_12 + if (bits == 12) return true; +#endif +#ifdef ENABLE_FIELD_INT_13 + if (bits == 13) return true; +#endif +#ifdef ENABLE_FIELD_INT_14 + if (bits == 14) return true; +#endif +#ifdef ENABLE_FIELD_INT_15 + if (bits == 15) return true; +#endif +#ifdef ENABLE_FIELD_INT_16 + if (bits == 16) return true; +#endif +#ifdef ENABLE_FIELD_INT_17 + if (bits == 17) return true; +#endif +#ifdef ENABLE_FIELD_INT_18 + if (bits == 18) return true; +#endif +#ifdef ENABLE_FIELD_INT_19 + if (bits == 19) return true; +#endif +#ifdef ENABLE_FIELD_INT_20 + if (bits == 20) return true; +#endif +#ifdef ENABLE_FIELD_INT_21 + if (bits == 21) return true; +#endif +#ifdef ENABLE_FIELD_INT_22 + if (bits == 22) return true; +#endif +#ifdef ENABLE_FIELD_INT_23 + if (bits == 23) return true; +#endif +#ifdef ENABLE_FIELD_INT_24 + if (bits == 24) return true; +#endif +#ifdef ENABLE_FIELD_INT_25 + if (bits == 25) return true; +#endif +#ifdef ENABLE_FIELD_INT_26 + if (bits == 26) return true; +#endif +#ifdef ENABLE_FIELD_INT_27 + if (bits == 27) return true; +#endif +#ifdef ENABLE_FIELD_INT_28 + if (bits == 28) return true; +#endif +#ifdef ENABLE_FIELD_INT_29 + if (bits == 29) return true; +#endif +#ifdef ENABLE_FIELD_INT_30 + if (bits == 30) return true; +#endif +#ifdef ENABLE_FIELD_INT_31 + if (bits == 31) return true; +#endif +#ifdef ENABLE_FIELD_INT_32 + if (bits == 32) return true; +#endif +#ifdef ENABLE_FIELD_INT_33 + if (bits == 33) return true; +#endif +#ifdef ENABLE_FIELD_INT_34 + if (bits == 34) return true; +#endif +#ifdef ENABLE_FIELD_INT_35 + if (bits == 35) return true; +#endif +#ifdef ENABLE_FIELD_INT_36 + if (bits == 36) return true; +#endif +#ifdef ENABLE_FIELD_INT_37 + if (bits == 37) return true; +#endif +#ifdef ENABLE_FIELD_INT_38 + if (bits == 38) return true; +#endif +#ifdef ENABLE_FIELD_INT_39 + if (bits == 39) return true; +#endif +#ifdef ENABLE_FIELD_INT_40 + if (bits == 40) return true; +#endif +#ifdef ENABLE_FIELD_INT_41 + if (bits == 41) return true; +#endif +#ifdef ENABLE_FIELD_INT_42 + if (bits == 42) return true; +#endif +#ifdef ENABLE_FIELD_INT_43 + if (bits == 43) return true; +#endif +#ifdef ENABLE_FIELD_INT_44 + if (bits == 44) return true; +#endif +#ifdef ENABLE_FIELD_INT_45 + if (bits == 45) return true; +#endif +#ifdef ENABLE_FIELD_INT_46 + if (bits == 46) return true; +#endif +#ifdef ENABLE_FIELD_INT_47 + if (bits == 47) return true; +#endif +#ifdef ENABLE_FIELD_INT_48 + if (bits == 48) return true; +#endif +#ifdef ENABLE_FIELD_INT_49 + if (bits == 49) return true; +#endif +#ifdef ENABLE_FIELD_INT_50 + if (bits == 50) return true; +#endif +#ifdef ENABLE_FIELD_INT_51 + if (bits == 51) return true; +#endif +#ifdef ENABLE_FIELD_INT_52 + if (bits == 52) return true; +#endif +#ifdef ENABLE_FIELD_INT_53 + if (bits == 53) return true; +#endif +#ifdef ENABLE_FIELD_INT_54 + if (bits == 54) return true; +#endif +#ifdef ENABLE_FIELD_INT_55 + if (bits == 55) return true; +#endif +#ifdef ENABLE_FIELD_INT_56 + if (bits == 56) return true; +#endif +#ifdef ENABLE_FIELD_INT_57 + if (bits == 57) return true; +#endif +#ifdef ENABLE_FIELD_INT_58 + if (bits == 58) return true; +#endif +#ifdef ENABLE_FIELD_INT_59 + if (bits == 59) return true; +#endif +#ifdef ENABLE_FIELD_INT_60 + if (bits == 60) return true; +#endif +#ifdef ENABLE_FIELD_INT_61 + if (bits == 61) return true; +#endif +#ifdef ENABLE_FIELD_INT_62 + if (bits == 62) return true; +#endif +#ifdef ENABLE_FIELD_INT_63 + if (bits == 63) return true; +#endif +#ifdef ENABLE_FIELD_INT_64 + if (bits == 64) return true; +#endif + return false; +} + +uint32_t minisketch_implementation_max() { + uint32_t ret = 0; +#ifdef HAVE_CLMUL + ret += 2; +#endif + return ret; +} + +int minisketch_implementation_supported(uint32_t bits, uint32_t implementation) { + if (!minisketch_bits_supported(bits) || implementation > minisketch_implementation_max()) { + return 0; + } + try { + Sketch* sketch = Construct(bits, implementation); + if (sketch) { + delete sketch; + return 1; + } + } catch (const std::bad_alloc&) {} + return 0; +} + +minisketch* minisketch_create(uint32_t bits, uint32_t implementation, size_t capacity) { + try { + Sketch* sketch = Construct(bits, implementation); + if (sketch) { + try { + sketch->Init(capacity); + } catch (const std::bad_alloc&) { + delete sketch; + throw; + } + sketch->Ready(); + } + return (minisketch*)sketch; + } catch (const std::bad_alloc&) { + return nullptr; + } +} + +uint32_t minisketch_bits(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + return s->Bits(); +} + +size_t minisketch_capacity(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + return s->Syndromes(); +} + +uint32_t minisketch_implementation(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + return s->Implementation(); +} + +minisketch* minisketch_clone(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + Sketch* r = (Sketch*) minisketch_create(s->Bits(), s->Implementation(), s->Syndromes()); + if (r) { + r->Merge(s); + } + return (minisketch*) r; +} + +void minisketch_destroy(minisketch* sketch) { + if (sketch) { + Sketch* s = (Sketch*)sketch; + s->UnReady(); + delete s; + } +} + +size_t minisketch_serialized_size(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + size_t bits = s->Bits(); + size_t syndromes = s->Syndromes(); + return (bits * syndromes + 7) / 8; +} + +void minisketch_serialize(const minisketch* sketch, unsigned char* output) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + s->Serialize(output); +} + +void minisketch_deserialize(minisketch* sketch, const unsigned char* input) { + Sketch* s = (Sketch*)sketch; + s->Check(); + s->Deserialize(input); +} + +void minisketch_add_uint64(minisketch* sketch, uint64_t element) { + Sketch* s = (Sketch*)sketch; + s->Check(); + s->Add(element); +} + +size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch) { + Sketch* s1 = (Sketch*)sketch; + const Sketch* s2 = (const Sketch*)other_sketch; + s1->Check(); + s2->Check(); + if (s1->Bits() != s2->Bits()) return 0; + if (s1->Implementation() != s2->Implementation()) return 0; + return s1->Merge(s2); +} + +ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_t* output) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + return s->Decode(max_elements, output); +} + +void minisketch_set_seed(minisketch* sketch, uint64_t seed) { + Sketch* s = (Sketch*)sketch; + s->Check(); + s->SetSeed(seed); +} + +size_t minisketch_compute_capacity(uint32_t bits, size_t max_elements, uint32_t fpbits) { + return ComputeCapacity(bits, max_elements, fpbits); +} + +size_t minisketch_compute_max_elements(uint32_t bits, size_t capacity, uint32_t fpbits) { + return ComputeMaxElements(bits, capacity, fpbits); +} + +} diff --git a/src/minisketch/src/sketch.h b/src/minisketch/src/sketch.h new file mode 100644 index 0000000000..3e9bad793d --- /dev/null +++ b/src/minisketch/src/sketch.h @@ -0,0 +1,42 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_STATE_H_ +#define _MINISKETCH_STATE_H_ + +#include <stdint.h> +#include <stdlib.h> + +/** Abstract class for internal representation of a minisketch object. */ +class Sketch +{ + uint64_t m_canary; + const int m_implementation; + const int m_bits; + +public: + Sketch(int implementation, int bits) : m_implementation(implementation), m_bits(bits) {} + + void Ready() { m_canary = 0x6d496e536b65LU; } + void Check() const { if (m_canary != 0x6d496e536b65LU) abort(); } + void UnReady() { m_canary = 1; } + int Implementation() const { return m_implementation; } + int Bits() const { return m_bits; } + + virtual ~Sketch() {} + virtual size_t Syndromes() const = 0; + + virtual void Init(int syndromes) = 0; + virtual void Add(uint64_t element) = 0; + virtual void Serialize(unsigned char*) const = 0; + virtual void Deserialize(const unsigned char*) = 0; + virtual size_t Merge(const Sketch* other_sketch) = 0; + virtual void SetSeed(uint64_t seed) = 0; + + virtual int Decode(int max_count, uint64_t* roots) const = 0; +}; + +#endif diff --git a/src/minisketch/src/sketch_impl.h b/src/minisketch/src/sketch_impl.h new file mode 100644 index 0000000000..4547b742f2 --- /dev/null +++ b/src/minisketch/src/sketch_impl.h @@ -0,0 +1,433 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_SKETCH_IMPL_H_ +#define _MINISKETCH_SKETCH_IMPL_H_ + +#include <random> + +#include "util.h" +#include "sketch.h" +#include "int_utils.h" + +/** Compute the remainder of a polynomial division of val by mod, putting the result in mod. */ +template<typename F> +void PolyMod(const std::vector<typename F::Elem>& mod, std::vector<typename F::Elem>& val, const F& field) { + size_t modsize = mod.size(); + CHECK_SAFE(modsize > 0 && mod.back() == 1); + if (val.size() < modsize) return; + CHECK_SAFE(val.back() != 0); + while (val.size() >= modsize) { + auto term = val.back(); + val.pop_back(); + if (term != 0) { + typename F::Multiplier mul(field, term); + for (size_t x = 0; x < mod.size() - 1; ++x) { + val[val.size() - modsize + 1 + x] ^= mul(mod[x]); + } + } + } + while (val.size() > 0 && val.back() == 0) val.pop_back(); +} + +/** Compute the quotient of a polynomial division of val by mod, putting the quotient in div and the remainder in val. */ +template<typename F> +void DivMod(const std::vector<typename F::Elem>& mod, std::vector<typename F::Elem>& val, std::vector<typename F::Elem>& div, const F& field) { + size_t modsize = mod.size(); + CHECK_SAFE(mod.size() > 0 && mod.back() == 1); + if (val.size() < mod.size()) { + div.clear(); + return; + } + CHECK_SAFE(val.back() != 0); + div.resize(val.size() - mod.size() + 1); + while (val.size() >= modsize) { + auto term = val.back(); + div[val.size() - modsize] = term; + val.pop_back(); + if (term != 0) { + typename F::Multiplier mul(field, term); + for (size_t x = 0; x < mod.size() - 1; ++x) { + val[val.size() - modsize + 1 + x] ^= mul(mod[x]); + } + } + } +} + +/** Make a polynomial monic. */ +template<typename F> +typename F::Elem MakeMonic(std::vector<typename F::Elem>& a, const F& field) { + CHECK_SAFE(a.back() != 0); + if (a.back() == 1) return 0; + auto inv = field.Inv(a.back()); + typename F::Multiplier mul(field, inv); + a.back() = 1; + for (size_t i = 0; i < a.size() - 1; ++i) { + a[i] = mul(a[i]); + } + return inv; +} + +/** Compute the GCD of two polynomials, putting the result in a. b will be cleared. */ +template<typename F> +void GCD(std::vector<typename F::Elem>& a, std::vector<typename F::Elem>& b, const F& field) { + if (a.size() < b.size()) std::swap(a, b); + while (b.size() > 0) { + if (b.size() == 1) { + a.resize(1); + a[0] = 1; + return; + } + MakeMonic(b, field); + PolyMod(b, a, field); + std::swap(a, b); + } +} + +/** Square a polynomial. */ +template<typename F> +void Sqr(std::vector<typename F::Elem>& poly, const F& field) { + if (poly.size() == 0) return; + poly.resize(poly.size() * 2 - 1); + for (int x = poly.size() - 1; x >= 0; --x) { + poly[x] = (x & 1) ? 0 : field.Sqr(poly[x / 2]); + } +} + +/** Compute the trace map of (param*x) modulo mod, putting the result in out. */ +template<typename F> +void TraceMod(const std::vector<typename F::Elem>& mod, std::vector<typename F::Elem>& out, const typename F::Elem& param, const F& field) { + out.reserve(mod.size() * 2); + out.resize(2); + out[0] = 0; + out[1] = param; + + for (int i = 0; i < field.Bits() - 1; ++i) { + Sqr(out, field); + if (out.size() < 2) out.resize(2); + out[1] = param; + PolyMod(mod, out, field); + } +} + +/** One step of the root finding algorithm; finds roots of stack[pos] and adds them to roots. Stack elements >= pos are destroyed. + * + * It operates on a stack of polynomials. The polynomial operated on is `stack[pos]`, where elements of `stack` with index higher + * than `pos` are used as scratch space. + * + * `stack[pos]` is assumed to be square-free polynomial. If `fully_factorizable` is true, it is also assumed to have no irreducible + * factors of degree higher than 1. + + * This implements the Berlekamp trace algorithm, plus an efficient test to fail fast in + * case the polynomial cannot be fully factored. + */ +template<typename F> +bool RecFindRoots(std::vector<std::vector<typename F::Elem>>& stack, size_t pos, std::vector<typename F::Elem>& roots, bool fully_factorizable, int depth, typename F::Elem randv, const F& field) { + auto& ppoly = stack[pos]; + // We assert ppoly.size() > 1 (instead of just ppoly.size() > 0) to additionally exclude + // constants polynomials because + // - ppoly is not constant initially (this is ensured by FindRoots()), and + // - we never recurse on a constant polynomial. + CHECK_SAFE(ppoly.size() > 1 && ppoly.back() == 1); + /* 1st degree input: constant term is the root. */ + if (ppoly.size() == 2) { + roots.push_back(ppoly[0]); + return true; + } + /* 2nd degree input: use direct quadratic solver. */ + if (ppoly.size() == 3) { + CHECK_RETURN(ppoly[1] != 0, false); // Equations of the form (x^2 + a) have two identical solutions; contradicts square-free assumption. */ + auto input = field.Mul(ppoly[0], field.Sqr(field.Inv(ppoly[1]))); + auto root = field.Qrt(input); + if ((field.Sqr(root) ^ root) != input) { + CHECK_SAFE(!fully_factorizable); + return false; // No root found. + } + auto sol = field.Mul(root, ppoly[1]); + roots.push_back(sol); + roots.push_back(sol ^ ppoly[1]); + return true; + } + /* 3rd degree input and more: recurse further. */ + if (pos + 3 > stack.size()) { + // Allocate memory if necessary. + stack.resize((pos + 3) * 2); + } + auto& poly = stack[pos]; + auto& tmp = stack[pos + 1]; + auto& trace = stack[pos + 2]; + trace.clear(); + tmp.clear(); + for (int iter = 0;; ++iter) { + // Compute the polynomial (trace(x*randv) mod poly(x)) symbolically, + // and put the result in `trace`. + TraceMod(poly, trace, randv, field); + + if (iter >= 1 && !fully_factorizable) { + // If the polynomial cannot be factorized completely (it has an + // irreducible factor of degree higher than 1), we want to avoid + // the case where this is only detected after trying all BITS + // independent split attempts fail (see the assert below). + // + // Observe that if we call y = randv*x, it is true that: + // + // trace = y + y^2 + y^4 + y^8 + ... y^(FIELDSIZE/2) mod poly + // + // Due to the Frobenius endomorphism, this means: + // + // trace^2 = y^2 + y^4 + y^8 + ... + y^FIELDSIZE mod poly + // + // Or, adding them up: + // + // trace + trace^2 = y + y^FIELDSIZE mod poly. + // = randv*x + randv^FIELDSIZE*x^FIELDSIZE + // = randv*x + randv*x^FIELDSIZE + // = randv*(x + x^FIELDSIZE). + // (all mod poly) + // + // x + x^FIELDSIZE is the polynomial which has every field element + // as root once. Whenever x + x^FIELDSIZE is multiple of poly, + // this means it only has unique first degree factors. The same + // holds for its constant multiple randv*(x + x^FIELDSIZE) = + // trace + trace^2. + // + // We use this test to quickly verify whether the polynomial is + // fully factorizable after already having computed a trace. + // We don't invoke it immediately; only when splitting has failed + // at least once, which avoids it for most polynomials that are + // fully factorizable (or at least pushes the test down the + // recursion to factors which are smaller and thus faster). + tmp = trace; + Sqr(tmp, field); + for (size_t i = 0; i < trace.size(); ++i) { + tmp[i] ^= trace[i]; + } + while (tmp.size() && tmp.back() == 0) tmp.pop_back(); + PolyMod(poly, tmp, field); + + // Whenever the test fails, we can immediately abort the root + // finding. Whenever it succeeds, we can remember and pass down + // the information that it is in fact fully factorizable, avoiding + // the need to run the test again. + if (tmp.size() != 0) return false; + fully_factorizable = true; + } + + if (fully_factorizable) { + // Every succesful iteration of this algorithm splits the input + // polynomial further into buckets, each corresponding to a subset + // of 2^(BITS-depth) roots. If after depth splits the degree of + // the polynomial is >= 2^(BITS-depth), something is wrong. + CHECK_RETURN(field.Bits() - depth >= std::numeric_limits<decltype(poly.size())>::digits || + (poly.size() - 2) >> (field.Bits() - depth) == 0, false); + } + + depth++; + // In every iteration we multiply randv by 2. As a result, the set + // of randv values forms a GF(2)-linearly independent basis of splits. + randv = field.Mul2(randv); + tmp = poly; + GCD(trace, tmp, field); + if (trace.size() != poly.size() && trace.size() > 1) break; + } + MakeMonic(trace, field); + DivMod(trace, poly, tmp, field); + // At this point, the stack looks like [... (poly) tmp trace], and we want to recursively + // find roots of trace and tmp (= poly/trace). As we don't care about poly anymore, move + // trace into its position first. + std::swap(poly, trace); + // Now the stack is [... (trace) tmp ...]. First we factor tmp (at pos = pos+1), and then + // we factor trace (at pos = pos). + if (!RecFindRoots(stack, pos + 1, roots, fully_factorizable, depth, randv, field)) return false; + // The stack position pos contains trace, the polynomial with all of poly's roots which (after + // multiplication with randv) have trace 0. This is never the case for irreducible factors + // (which always end up in tmp), so we can set fully_factorizable to true when recursing. + bool ret = RecFindRoots(stack, pos, roots, true, depth, randv, field); + // Because of the above, recursion can never fail here. + CHECK_SAFE(ret); + return ret; +} + +/** Returns the roots of a fully factorizable polynomial + * + * This function assumes that the input polynomial is square-free + * and not the zero polynomial (represented by an empty vector). + * + * In case the square-free polynomial is not fully factorizable, i.e., it + * has fewer roots than its degree, the empty vector is returned. + */ +template<typename F> +std::vector<typename F::Elem> FindRoots(const std::vector<typename F::Elem>& poly, typename F::Elem basis, const F& field) { + std::vector<typename F::Elem> roots; + CHECK_RETURN(poly.size() != 0, {}); + CHECK_RETURN(basis != 0, {}); + if (poly.size() == 1) return roots; // No roots when the polynomial is a constant. + roots.reserve(poly.size() - 1); + std::vector<std::vector<typename F::Elem>> stack = {poly}; + + // Invoke the recursive factorization algorithm. + if (!RecFindRoots(stack, 0, roots, false, 0, basis, field)) { + // Not fully factorizable. + return {}; + } + CHECK_RETURN(poly.size() - 1 == roots.size(), {}); + return roots; +} + +template<typename F> +std::vector<typename F::Elem> BerlekampMassey(const std::vector<typename F::Elem>& syndromes, size_t max_degree, const F& field) { + std::vector<typename F::Multiplier> table; + std::vector<typename F::Elem> current, prev, tmp; + current.reserve(syndromes.size() / 2 + 1); + prev.reserve(syndromes.size() / 2 + 1); + tmp.reserve(syndromes.size() / 2 + 1); + current.resize(1); + current[0] = 1; + prev.resize(1); + prev[0] = 1; + typename F::Elem b = 1, b_inv = 1; + bool b_have_inv = true; + table.reserve(syndromes.size()); + + for (size_t n = 0; n != syndromes.size(); ++n) { + table.emplace_back(field, syndromes[n]); + auto discrepancy = syndromes[n]; + for (size_t i = 1; i < current.size(); ++i) discrepancy ^= table[n - i](current[i]); + if (discrepancy != 0) { + int x = n + 1 - (current.size() - 1) - (prev.size() - 1); + if (!b_have_inv) { + b_inv = field.Inv(b); + b_have_inv = true; + } + bool swap = 2 * (current.size() - 1) <= n; + if (swap) { + if (prev.size() + x - 1 > max_degree) return {}; // We'd exceed maximum degree + tmp = current; + current.resize(prev.size() + x); + } + typename F::Multiplier mul(field, field.Mul(discrepancy, b_inv)); + for (size_t i = 0; i < prev.size(); ++i) current[i + x] ^= mul(prev[i]); + if (swap) { + std::swap(prev, tmp); + b = discrepancy; + b_have_inv = false; + } + } + } + CHECK_RETURN(current.size() && current.back() != 0, {}); + return current; +} + +template<typename F> +std::vector<typename F::Elem> ReconstructAllSyndromes(const std::vector<typename F::Elem>& odd_syndromes, const F& field) { + std::vector<typename F::Elem> all_syndromes; + all_syndromes.resize(odd_syndromes.size() * 2); + for (size_t i = 0; i < odd_syndromes.size(); ++i) { + all_syndromes[i * 2] = odd_syndromes[i]; + all_syndromes[i * 2 + 1] = field.Sqr(all_syndromes[i]); + } + return all_syndromes; +} + +template<typename F> +void AddToOddSyndromes(std::vector<typename F::Elem>& osyndromes, typename F::Elem data, const F& field) { + auto sqr = field.Sqr(data); + typename F::Multiplier mul(field, sqr); + for (auto& osyndrome : osyndromes) { + osyndrome ^= data; + data = mul(data); + } +} + +template<typename F> +std::vector<typename F::Elem> FullDecode(const std::vector<typename F::Elem>& osyndromes, const F& field) { + auto asyndromes = ReconstructAllSyndromes<typename F::Elem>(osyndromes, field); + auto poly = BerlekampMassey(asyndromes, field); + std::reverse(poly.begin(), poly.end()); + return FindRoots(poly, field); +} + +template<typename F> +class SketchImpl final : public Sketch +{ + const F m_field; + std::vector<typename F::Elem> m_syndromes; + typename F::Elem m_basis; + +public: + template<typename... Args> + SketchImpl(int implementation, int bits, const Args&... args) : Sketch(implementation, bits), m_field(args...) { + std::random_device rng; + std::uniform_int_distribution<uint64_t> dist; + m_basis = m_field.FromSeed(dist(rng)); + } + + size_t Syndromes() const override { return m_syndromes.size(); } + void Init(int count) override { m_syndromes.assign(count, 0); } + + void Add(uint64_t val) override + { + auto elem = m_field.FromUint64(val); + AddToOddSyndromes(m_syndromes, elem, m_field); + } + + void Serialize(unsigned char* ptr) const override + { + BitWriter writer(ptr); + for (const auto& val : m_syndromes) { + m_field.Serialize(writer, val); + } + writer.Flush(); + } + + void Deserialize(const unsigned char* ptr) override + { + BitReader reader(ptr); + for (auto& val : m_syndromes) { + val = m_field.Deserialize(reader); + } + } + + int Decode(int max_count, uint64_t* out) const override + { + auto all_syndromes = ReconstructAllSyndromes(m_syndromes, m_field); + auto poly = BerlekampMassey(all_syndromes, max_count, m_field); + if (poly.size() == 0) return -1; + if (poly.size() == 1) return 0; + if ((int)poly.size() > 1 + max_count) return -1; + std::reverse(poly.begin(), poly.end()); + auto roots = FindRoots(poly, m_basis, m_field); + if (roots.size() == 0) return -1; + + for (const auto& root : roots) { + *(out++) = m_field.ToUint64(root); + } + return roots.size(); + } + + size_t Merge(const Sketch* other_sketch) override + { + // Sad cast. This is safe only because the caller code in minisketch.cpp checks + // that implementation and field size match. + const SketchImpl* other = static_cast<const SketchImpl*>(other_sketch); + m_syndromes.resize(std::min(m_syndromes.size(), other->m_syndromes.size())); + for (size_t i = 0; i < m_syndromes.size(); ++i) { + m_syndromes[i] ^= other->m_syndromes[i]; + } + return m_syndromes.size(); + } + + void SetSeed(uint64_t seed) override + { + if (seed == (uint64_t)-1) { + m_basis = 1; + } else { + m_basis = m_field.FromSeed(seed); + } + } +}; + +#endif diff --git a/src/minisketch/src/test.cpp b/src/minisketch/src/test.cpp new file mode 100644 index 0000000000..417937ea5f --- /dev/null +++ b/src/minisketch/src/test.cpp @@ -0,0 +1,316 @@ +/********************************************************************** + * Copyright (c) 2018,2021 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include <algorithm> +#include <cstdio> +#include <limits> +#include <random> +#include <stdexcept> +#include <vector> + +#include "../include/minisketch.h" +#include "util.h" + +namespace { + +uint64_t Combination(uint64_t n, uint64_t k) { + if (n - k < k) k = n - k; + uint64_t ret = 1; + for (uint64_t i = 1; i <= k; ++i) { + ret = (ret * n) / i; + --n; + } + return ret; +} + +/** Create a vector with Minisketch objects, one for each implementation. */ +std::vector<Minisketch> CreateSketches(uint32_t bits, size_t capacity) { + if (!Minisketch::BitsSupported(bits)) return {}; + std::vector<Minisketch> ret; + for (uint32_t impl = 0; impl <= Minisketch::MaxImplementation(); ++impl) { + if (Minisketch::ImplementationSupported(bits, impl)) { + CHECK(Minisketch::BitsSupported(bits)); + ret.push_back(Minisketch(bits, impl, capacity)); + CHECK((bool)ret.back()); + } else { + // implementation 0 must always work unless field size is disabled + CHECK(impl != 0 || !Minisketch::BitsSupported(bits)); + } + } + return ret; +} + +/** Test properties by exhaustively decoding all 2**(bits*capacity) sketches + * with specified capacity and bits. */ +void TestExhaustive(uint32_t bits, size_t capacity) { + auto sketches = CreateSketches(bits, capacity); + if (sketches.empty()) return; + auto sketches_rebuild = CreateSketches(bits, capacity); + + std::vector<unsigned char> serialized; + std::vector<unsigned char> serialized_empty; + std::vector<uint64_t> counts; //!< counts[i] = number of results with i elements + std::vector<uint64_t> elements_0; //!< Result vector for elements for impl=0 + std::vector<uint64_t> elements_other; //!< Result vector for elements for other impls + std::vector<uint64_t> elements_too_small; //!< Result vector that's too small + + counts.resize(capacity + 1); + serialized.resize(sketches[0].GetSerializedSize()); + serialized_empty.resize(sketches[0].GetSerializedSize()); + + // Iterate over all (bits)-bit sketches with (capacity) syndromes. + for (uint64_t x = 0; (x >> (bits * capacity)) == 0; ++x) { + // Construct the serialization. + for (size_t i = 0; i < serialized.size(); ++i) { + serialized[i] = (x >> (i * 8)) & 0xFF; + } + + // Compute all the solutions + sketches[0].Deserialize(serialized); + elements_0.resize(64); + bool decodable_0 = sketches[0].Decode(elements_0); + std::sort(elements_0.begin(), elements_0.end()); + + // Verify that decoding with other implementations agrees. + for (size_t impl = 1; impl < sketches.size(); ++impl) { + sketches[impl].Deserialize(serialized); + elements_other.resize(64); + bool decodable_other = sketches[impl].Decode(elements_other); + CHECK(decodable_other == decodable_0); + std::sort(elements_other.begin(), elements_other.end()); + CHECK(elements_other == elements_0); + } + + // If there are solutions: + if (decodable_0) { + if (!elements_0.empty()) { + // Decoding with limit one less than the number of elements should fail. + elements_too_small.resize(elements_0.size() - 1); + for (size_t impl = 0; impl < sketches.size(); ++impl) { + CHECK(!sketches[impl].Decode(elements_too_small)); + } + } + + // Reconstruct the sketch from the solutions. + for (size_t impl = 0; impl < sketches.size(); ++impl) { + // Clear the sketch. + sketches_rebuild[impl].Deserialize(serialized_empty); + // Load all decoded elements into it. + for (uint64_t elem : elements_0) { + CHECK(elem != 0); + CHECK(elem >> bits == 0); + sketches_rebuild[impl].Add(elem); + } + // Reserialize the result + auto serialized_rebuild = sketches_rebuild[impl].Serialize(); + // Compare + CHECK(serialized == serialized_rebuild); + // Count it + if (impl == 0 && elements_0.size() <= capacity) ++counts[elements_0.size()]; + } + } + } + + // Verify that the number of decodable sketches with given elements is expected. + uint64_t mask = bits == 64 ? UINT64_MAX : (uint64_t{1} << bits) - 1; + for (uint64_t i = 0; i <= capacity && (i & mask) == i; ++i) { + CHECK(counts[i] == Combination(mask, i)); + } +} + +/** Test properties of sketches with random elements put in. */ +void TestRandomized(uint32_t bits, size_t max_capacity, size_t iter) { + std::random_device rnd; + std::uniform_int_distribution<uint64_t> capacity_dist(0, std::min<uint64_t>(std::numeric_limits<uint64_t>::max() >> (64 - bits), max_capacity)); + std::uniform_int_distribution<uint64_t> element_dist(1, std::numeric_limits<uint64_t>::max() >> (64 - bits)); + std::uniform_int_distribution<uint64_t> rand64(0, std::numeric_limits<uint64_t>::max()); + std::uniform_int_distribution<int64_t> size_offset_dist(-3, 3); + + std::vector<uint64_t> decode_0; + std::vector<uint64_t> decode_other; + std::vector<uint64_t> decode_temp; + std::vector<uint64_t> elements; + + for (size_t i = 0; i < iter; ++i) { + // Determine capacity, and construct Minisketch objects for all implementations. + uint64_t capacity = capacity_dist(rnd); + auto sketches = CreateSketches(bits, capacity); + // Sanity checks + if (sketches.empty()) return; + for (size_t impl = 0; impl < sketches.size(); ++impl) { + CHECK(sketches[impl].GetBits() == bits); + CHECK(sketches[impl].GetCapacity() == capacity); + CHECK(sketches[impl].GetSerializedSize() == sketches[0].GetSerializedSize()); + } + // Determine the number of elements, and create a vector to store them in. + size_t element_count = std::max<int64_t>(0, std::max<int64_t>(0, capacity + size_offset_dist(rnd))); + elements.resize(element_count); + // Add the elements to all sketches + for (size_t j = 0; j < element_count; ++j) { + uint64_t elem = element_dist(rnd); + CHECK(elem != 0); + elements[j] = elem; + for (auto& sketch : sketches) sketch.Add(elem); + } + // Remove pairs of duplicates in elements, as they cancel out. + std::sort(elements.begin(), elements.end()); + size_t real_element_count = element_count; + for (size_t pos = 0; pos + 1 < elements.size(); ++pos) { + if (elements[pos] == elements[pos + 1]) { + real_element_count -= 2; + // Set both elements to 0; afterwards we will move these to the end. + elements[pos] = 0; + elements[pos + 1] = 0; + ++pos; + } + } + if (real_element_count < element_count) { + // Move all introduced zeroes (masking duplicates) to the end. + std::sort(elements.begin(), elements.end(), [](uint64_t a, uint64_t b) { return a != b && (b == 0 || (a != 0 && a < b)); }); + CHECK(elements[real_element_count] == 0); + elements.resize(real_element_count); + } + // Create and compare serializations + auto serialized_0 = sketches[0].Serialize(); + for (size_t impl = 1; impl < sketches.size(); ++impl) { + auto serialized_other = sketches[impl].Serialize(); + CHECK(serialized_other == serialized_0); + } + // Deserialize and reserialize them + for (size_t impl = 0; impl < sketches.size(); ++impl) { + sketches[impl].Deserialize(serialized_0); + auto reserialized = sketches[impl].Serialize(); + CHECK(reserialized == serialized_0); + } + // Decode with limit set to the capacity, and compare results + decode_0.resize(capacity); + bool decodable_0 = sketches[0].Decode(decode_0); + std::sort(decode_0.begin(), decode_0.end()); + for (size_t impl = 1; impl < sketches.size(); ++impl) { + decode_other.resize(capacity); + bool decodable_other = sketches[impl].Decode(decode_other); + CHECK(decodable_other == decodable_0); + std::sort(decode_other.begin(), decode_other.end()); + CHECK(decode_other == decode_0); + } + // If the result is decodable, it should also be decodable with limit + // set to the actual number of elements, and not with one less. + if (decodable_0) { + for (auto& sketch : sketches) { + decode_temp.resize(decode_0.size()); + bool decodable = sketch.Decode(decode_temp); + CHECK(decodable); + std::sort(decode_temp.begin(), decode_temp.end()); + CHECK(decode_temp == decode_0); + if (!decode_0.empty()) { + decode_temp.resize(decode_0.size() - 1); + decodable = sketch.Decode(decode_temp); + CHECK(!decodable); + } + } + } + // If the actual number of elements is not higher than the capacity, the + // result should be decodable, and the result should match what we put in. + if (real_element_count <= capacity) { + CHECK(decodable_0); + CHECK(decode_0 == elements); + } + } +} + +void TestComputeFunctions() { + for (uint32_t bits = 0; bits <= 256; ++bits) { + for (uint32_t fpbits = 0; fpbits <= 512; ++fpbits) { + std::vector<size_t> table_max_elements(1025); + for (size_t capacity = 0; capacity <= 1024; ++capacity) { + table_max_elements[capacity] = minisketch_compute_max_elements(bits, capacity, fpbits); + // Exception for bits==0 + if (bits == 0) CHECK(table_max_elements[capacity] == 0); + // A sketch with capacity N cannot guarantee decoding more than N elements. + CHECK(table_max_elements[capacity] <= capacity); + // When asking for N bits of false positive protection, either no solution exists, or no more than ceil(N / bits) excess capacity should be needed. + if (bits > 0) CHECK(table_max_elements[capacity] == 0 || capacity - table_max_elements[capacity] <= (fpbits + bits - 1) / bits); + // Increasing capacity by one, if there is a solution, should always increment the max_elements by at least one as well. + if (capacity > 0) CHECK(table_max_elements[capacity] == 0 || table_max_elements[capacity] > table_max_elements[capacity - 1]); + } + + std::vector<size_t> table_capacity(513); + for (size_t max_elements = 0; max_elements <= 512; ++max_elements) { + table_capacity[max_elements] = minisketch_compute_capacity(bits, max_elements, fpbits); + // Exception for bits==0 + if (bits == 0) CHECK(table_capacity[max_elements] == 0); + // To be able to decode N elements, capacity needs to be at least N. + if (bits > 0) CHECK(table_capacity[max_elements] >= max_elements); + // A sketch of N bits in total cannot have more than N bits of false positive protection; + if (bits > 0) CHECK(bits * table_capacity[max_elements] >= fpbits); + // When asking for N bits of false positive protection, no more than ceil(N / bits) excess capacity should be needed. + if (bits > 0) CHECK(table_capacity[max_elements] - max_elements <= (fpbits + bits - 1) / bits); + // Increasing max_elements by one can only increment the capacity by 0 or 1. + if (max_elements > 0 && fpbits < 256) CHECK(table_capacity[max_elements] == table_capacity[max_elements - 1] || table_capacity[max_elements] == table_capacity[max_elements - 1] + 1); + // Check round-tripping max_elements->capacity->max_elements (only a lower bound) + CHECK(table_capacity[max_elements] <= 1024); + CHECK(table_max_elements[table_capacity[max_elements]] == 0 || table_max_elements[table_capacity[max_elements]] >= max_elements); + } + + for (size_t capacity = 0; capacity <= 512; ++capacity) { + // Check round-tripping capacity->max_elements->capacity (exact, if it exists) + CHECK(table_max_elements[capacity] <= 512); + CHECK(table_max_elements[capacity] == 0 || table_capacity[table_max_elements[capacity]] == capacity); + } + } + } +} + +} // namespace + +int main(int argc, char** argv) { + uint64_t test_complexity = 4; + if (argc > 1) { + size_t len = 0; + std::string arg{argv[1]}; + try { + test_complexity = 0; + long long complexity = std::stoll(arg, &len); + if (complexity >= 1 && len == arg.size() && ((uint64_t)complexity <= std::numeric_limits<uint64_t>::max() >> 10)) { + test_complexity = complexity; + } + } catch (const std::logic_error&) {} + if (test_complexity == 0) { + fprintf(stderr, "Invalid complexity specified: '%s'\n", arg.c_str()); + return 1; + } + } + +#ifdef MINISKETCH_VERIFY + const char* mode = " in verify mode"; +#else + const char* mode = ""; +#endif + printf("Running libminisketch tests%s with complexity=%llu\n", mode, (unsigned long long)test_complexity); + + TestComputeFunctions(); + + for (unsigned j = 2; j <= 64; ++j) { + TestRandomized(j, 8, (test_complexity << 10) / j); + TestRandomized(j, 128, (test_complexity << 7) / j); + TestRandomized(j, 4096, test_complexity / j); + } + + // Test capacity==0 together with all field sizes, and then + // all combinations of bits and capacity up to a certain bits*capacity, + // depending on test_complexity. + for (int weight = 0; weight <= 40; ++weight) { + for (int bits = 2; weight == 0 ? bits <= 64 : (bits <= 32 && bits <= weight); ++bits) { + int capacity = weight / bits; + if (capacity * bits != weight) continue; + TestExhaustive(bits, capacity); + } + if (weight >= 16 && test_complexity >> (weight - 16) == 0) break; + } + + printf("All tests successful.\n"); + return 0; +} diff --git a/src/minisketch/src/util.h b/src/minisketch/src/util.h new file mode 100644 index 0000000000..fdb3f3a231 --- /dev/null +++ b/src/minisketch/src/util.h @@ -0,0 +1,74 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_UTIL_H_ +#define _MINISKETCH_UTIL_H_ + +#ifdef MINISKETCH_VERIFY +#include <stdio.h> +#endif + +#if !defined(__GNUC_PREREQ) +# if defined(__GNUC__)&&defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(_maj,_min) \ + ((__GNUC__<<16)+__GNUC_MINOR__>=((_maj)<<16)+(_min)) +# else +# define __GNUC_PREREQ(_maj,_min) 0 +# endif +#endif + +#if __GNUC_PREREQ(3, 0) +#define EXPECT(x,c) __builtin_expect((x),(c)) +#else +#define EXPECT(x,c) (x) +#endif + +/* Assertion macros */ + +/** + * Unconditional failure on condition failure. + * Primarily used in testing harnesses. + */ +#define CHECK(cond) do { \ + if (EXPECT(!(cond), 0)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, "Check condition failed: " #cond); \ + abort(); \ + } \ +} while(0) + +/** + * Check macro that does nothing in normal non-verify builds but crashes in verify builds. + * This is used to test conditions at runtime that should always be true, but are either + * expensive to test or in locations where returning on failure would be messy. + */ +#ifdef MINISKETCH_VERIFY +#define CHECK_SAFE(cond) CHECK(cond) +#else +#define CHECK_SAFE(cond) +#endif + +/** + * Check a condition and return on failure in non-verify builds, crash in verify builds. + * Used for inexpensive conditions which believed to be always true in locations where + * a graceful exit is possible. + */ +#ifdef MINISKETCH_VERIFY +#define CHECK_RETURN(cond, rvar) do { \ + if (EXPECT(!(cond), 0)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, "Check condition failed: " #cond); \ + abort(); \ + return rvar; /* Does nothing, but causes compile to warn on incorrect return types. */ \ + } \ +} while(0) +#else +#define CHECK_RETURN(cond, rvar) do { \ + if (EXPECT(!(cond), 0)) { \ + return rvar; \ + } \ +} while(0) +#endif + +#endif diff --git a/src/minisketch/tests/pyminisketch.py b/src/minisketch/tests/pyminisketch.py new file mode 100755 index 0000000000..7a9ea9c1f1 --- /dev/null +++ b/src/minisketch/tests/pyminisketch.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +"""Native Python (slow) reimplementation of libminisketch' algorithms.""" + +import random +import unittest + +# Irreducible polynomials over GF(2) to use (represented as integers). +# +# Most fields can be defined by multiple such polynomials. Minisketch uses the one with the minimal +# number of nonzero coefficients, and tie-breaking by picking the lexicographically first among +# those. +# +# All polynomials for degrees 2 through 64 (inclusive) are given. +GF2_MODULI = [ + None, None, + 2**2 + 2**1 + 1, + 2**3 + 2**1 + 1, + 2**4 + 2**1 + 1, + 2**5 + 2**2 + 1, + 2**6 + 2**1 + 1, + 2**7 + 2**1 + 1, + 2**8 + 2**4 + 2**3 + 2**1 + 1, + 2**9 + 2**1 + 1, + 2**10 + 2**3 + 1, + 2**11 + 2**2 + 1, + 2**12 + 2**3 + 1, + 2**13 + 2**4 + 2**3 + 2**1 + 1, + 2**14 + 2**5 + 1, + 2**15 + 2**1 + 1, + 2**16 + 2**5 + 2**3 + 2**1 + 1, + 2**17 + 2**3 + 1, + 2**18 + 2**3 + 1, + 2**19 + 2**5 + 2**2 + 2**1 + 1, + 2**20 + 2**3 + 1, + 2**21 + 2**2 + 1, + 2**22 + 2**1 + 1, + 2**23 + 2**5 + 1, + 2**24 + 2**4 + 2**3 + 2**1 + 1, + 2**25 + 2**3 + 1, + 2**26 + 2**4 + 2**3 + 2**1 + 1, + 2**27 + 2**5 + 2**2 + 2**1 + 1, + 2**28 + 2**1 + 1, + 2**29 + 2**2 + 1, + 2**30 + 2**1 + 1, + 2**31 + 2**3 + 1, + 2**32 + 2**7 + 2**3 + 2**2 + 1, + 2**33 + 2**10 + 1, + 2**34 + 2**7 + 1, + 2**35 + 2**2 + 1, + 2**36 + 2**9 + 1, + 2**37 + 2**6 + 2**4 + 2**1 + 1, + 2**38 + 2**6 + 2**5 + 2**1 + 1, + 2**39 + 2**4 + 1, + 2**40 + 2**5 + 2**4 + 2**3 + 1, + 2**41 + 2**3 + 1, + 2**42 + 2**7 + 1, + 2**43 + 2**6 + 2**4 + 2**3 + 1, + 2**44 + 2**5 + 1, + 2**45 + 2**4 + 2**3 + 2**1 + 1, + 2**46 + 2**1 + 1, + 2**47 + 2**5 + 1, + 2**48 + 2**5 + 2**3 + 2**2 + 1, + 2**49 + 2**9 + 1, + 2**50 + 2**4 + 2**3 + 2**2 + 1, + 2**51 + 2**6 + 2**3 + 2**1 + 1, + 2**52 + 2**3 + 1, + 2**53 + 2**6 + 2**2 + 2**1 + 1, + 2**54 + 2**9 + 1, + 2**55 + 2**7 + 1, + 2**56 + 2**7 + 2**4 + 2**2 + 1, + 2**57 + 2**4 + 1, + 2**58 + 2**19 + 1, + 2**59 + 2**7 + 2**4 + 2**2 + 1, + 2**60 + 2**1 + 1, + 2**61 + 2**5 + 2**2 + 2**1 + 1, + 2**62 + 2**29 + 1, + 2**63 + 2**1 + 1, + 2**64 + 2**4 + 2**3 + 2**1 + 1 +] + +class GF2Ops: + """Class to perform GF(2^field_size) operations on elements represented as integers. + + Given that elements are represented as integers, addition is simply xor, and not + exposed here. + """ + + def __init__(self, field_size): + """Construct a GF2Ops object for the specified field size.""" + self.field_size = field_size + self._modulus = GF2_MODULI[field_size] + assert self._modulus is not None + + def mul2(self, x): + """Multiply x by 2 in GF(2^field_size).""" + x <<= 1 + if x >> self.field_size: + x ^= self._modulus + return x + + def mul(self, x, y): + """Multiply x by y in GF(2^field_size).""" + ret = 0 + while y: + if y & 1: + ret ^= x + y >>= 1 + x = self.mul2(x) + return ret + + def sqr(self, x): + """Square x in GF(2^field_size).""" + return self.mul(x, x) + + def inv(self, x): + """Compute the inverse of x in GF(2^field_size).""" + assert x != 0 + # Use the extended polynomial Euclidean GCD algorithm on (modulus, x), over GF(2). + # See https://en.wikipedia.org/wiki/Polynomial_greatest_common_divisor. + t1, t2 = 0, 1 + r1, r2 = self._modulus, x + r1l, r2l = self.field_size + 1, r2.bit_length() + while r2: + q = r1l - r2l + r1 ^= r2 << q + t1 ^= t2 << q + r1l = r1.bit_length() + if r1 < r2: + t1, t2 = t2, t1 + r1, r2 = r2, r1 + r1l, r2l = r2l, r1l + assert r1 == 1 + return t1 + +class TestGF2Ops(unittest.TestCase): + """Test class for basic arithmetic properties of GF2Ops.""" + + def field_size_test(self, field_size): + """Test operations for given field_size.""" + + gf = GF2Ops(field_size) + for i in range(100): + x = random.randrange(1 << field_size) + y = random.randrange(1 << field_size) + x2 = gf.mul2(x) + xy = gf.mul(x, y) + self.assertEqual(x2, gf.mul(x, 2)) # mul2(x) == x*2 + self.assertEqual(x2, gf.mul(2, x)) # mul2(x) == 2*x + self.assertEqual(xy == 0, x == 0 or y == 0) + self.assertEqual(xy == x, y == 1 or x == 0) + self.assertEqual(xy == y, x == 1 or y == 0) + self.assertEqual(xy, gf.mul(y, x)) # x*y == y*x + if i < 10: + xp = x + for _ in range(field_size): + xp = gf.sqr(xp) + self.assertEqual(xp, x) # x^(2^field_size) == x + if y != 0: + yi = gf.inv(y) + self.assertEqual(y == yi, y == 1) # y==1/x iff y==1 + self.assertEqual(gf.mul(y, yi), 1) # y*(1/y) == 1 + yii = gf.inv(yi) + self.assertEqual(y, yii) # 1/(1/y) == y + if x != 0: + xi = gf.inv(x) + xyi = gf.inv(xy) + self.assertEqual(xyi, gf.mul(xi, yi)) # (1/x)*(1/y) == 1/(x*y) + + def test(self): + """Run tests.""" + for field_size in range(2, 65): + self.field_size_test(field_size) + +# The operations below operate on polynomials over GF(2^field_size), represented as lists of +# integers: +# +# [a, b, c, ...] = a + b*x + c*x^2 + ... +# +# As an invariant, there are never any trailing zeroes in the list representation. +# +# Examples: +# * [] = 0 +# * [3] = 3 +# * [0, 1] = x +# * [2, 0, 5] = 5*x^2 + 2 + +def poly_monic(poly, gf): + """Return a monic version of the polynomial poly.""" + # Multiply every coefficient with the inverse of the top coefficient. + inv = gf.inv(poly[-1]) + return [gf.mul(inv, v) for v in poly] + +def poly_divmod(poly, mod, gf): + """Return the polynomial (quotient, remainder) of poly divided by mod.""" + assert len(mod) > 0 and mod[-1] == 1 # Require monic mod. + if len(poly) < len(mod): + return ([], poly) + val = list(poly) + div = [0 for _ in range(len(val) - len(mod) + 1)] + while len(val) >= len(mod): + term = val[-1] + div[len(val) - len(mod)] = term + # If the highest coefficient in val is nonzero, subtract a multiple of mod from it. + val.pop() + if term != 0: + for x in range(len(mod) - 1): + val[1 + x - len(mod)] ^= gf.mul(term, mod[x]) + # Prune trailing zero coefficients. + while len(val) > 0 and val[-1] == 0: + val.pop() + return div, val + +def poly_gcd(a, b, gf): + """Return the polynomial GCD of a and b.""" + if len(a) < len(b): + a, b = b, a + # Use Euclid's algorithm to find the GCD of a and b. + # see https://en.wikipedia.org/wiki/Polynomial_greatest_common_divisor#Euclid's_algorithm. + while len(b) > 0: + b = poly_monic(b, gf) + (_, b), a = poly_divmod(a, b, gf), b + return a + +def poly_sqr(poly, gf): + """Return the square of polynomial poly.""" + if len(poly) == 0: + return [] + # In characteristic-2 fields, thanks to Frobenius' endomorphism ((a + b)^2 = a^2 + b^2), + # squaring a polynomial is easy: square all the coefficients and interleave with zeroes. + # E.g., (3 + 5*x + 17*x^2)^2 = 3^2 + (5*x)^2 + (17*x^2)^2. + # See https://en.wikipedia.org/wiki/Frobenius_endomorphism. + return [0 if i & 1 else gf.sqr(poly[i // 2]) for i in range(2 * len(poly) - 1)] + +def poly_tracemod(poly, param, gf): + """Compute y + y^2 + y^4 + ... + y^(2^(field_size-1)) mod poly, where y = param*x.""" + out = [0, param] + for _ in range(gf.field_size - 1): + # In each loop iteration i, we start with out = y + y^2 + ... + y^(2^i). By squaring that we + # transform it into out = y^2 + y^4 + ... + y^(2^(i+1)). + out = poly_sqr(out, gf) + # Thus, we just need to add y again to it to get out = y + ... + y^(2^(i+1)). + while len(out) < 2: + out.append(0) + out[1] = param + # Finally take a modulus to keep the intermediary polynomials small. + _, out = poly_divmod(out, poly, gf) + return out + +def poly_frobeniusmod(poly, gf): + """Compute x^(2^field_size) mod poly.""" + out = [0, 1] + for _ in range(gf.field_size): + _, out = poly_divmod(poly_sqr(out, gf), poly, gf) + return out + +def poly_find_roots(poly, gf): + """Find the roots of poly if fully factorizable with unique roots, [] otherwise.""" + assert len(poly) > 0 + # If the polynomial is constant (and nonzero), it has no roots. + if len(poly) == 1: + return [] + # Make the polynomial monic (which doesn't change its roots). + poly = poly_monic(poly, gf) + # If the polynomial is of the form x+a, return a. + if len(poly) == 2: + return [poly[0]] + # Otherwise, first test that poly can be completely factored into unique roots. The polynomial + # x^(2^fieldsize)-x has every field element once as root. Thus we want to know that that is a + # multiple of poly. Compute x^(field_size) mod poly, which needs to equal x if that is the case + # (unless poly has degree <= 1, but that case is handled above). + if poly_frobeniusmod(poly, gf) != [0, 1]: + return [] + + def rec_split(poly, randv): + """Recursively split poly using the Berlekamp trace algorithm.""" + # See https://hal.archives-ouvertes.fr/hal-00626997/document. + assert len(poly) > 1 and poly[-1] == 1 # Require a monic poly. + # If poly is of the form x+a, its root is a. + if len(poly) == 2: + return [poly[0]] + # Try consecutive randomization factors randv, until one is found that factors poly. + while True: + # Compute the trace of (randv*x) mod poly. This is a polynomial that maps half of the + # domain to 0, and the other half to 1. Which half that is is controlled by randv. + # By taking it modulo poly, we only add a multiple of poly. Thus the result has at least + # the shared roots of the trace polynomial and poly still, but may have others. + trace = poly_tracemod(poly, randv, gf) + # Using the set {2^i*a for i=0..fieldsize-1} gives optimally independent randv values + # (no more than fieldsize are ever needed). + randv = gf.mul2(randv) + # Now take the GCD of this trace polynomial with poly. The result is a polynomial + # that only has the shared roots of the trace polynomial and poly as roots. + gcd = poly_gcd(trace, poly, gf) + # If the result has a degree higher than 1, and lower than that of poly, we found a + # useful factorization. + if len(gcd) != len(poly) and len(gcd) > 1: + break + # Otherwise, continue with another randv. + # Find the actual factors: the monic version of the GCD above, and poly divided by it. + factor1 = poly_monic(gcd, gf) + factor2, _ = poly_divmod(poly, gcd, gf) + # Recurse. + return rec_split(factor1, randv) + rec_split(factor2, randv) + + # Invoke the recursive splitting with a random initial factor, and sort the results. + return sorted(rec_split(poly, random.randrange(1, 1 << gf.field_size))) + +class TestPolyFindRoots(unittest.TestCase): + """Test class for poly_find_roots.""" + + def field_size_test(self, field_size): + """Run tests for given field_size.""" + gf = GF2Ops(field_size) + for test_size in [0, 1, 2, 3, 10]: + roots = [random.randrange(1 << field_size) for _ in range(test_size)] + roots_set = set(roots) + # Construct a polynomial with all elements of roots as roots (with multiplicity). + poly = [1] + for root in roots: + new_poly = [0] + poly + for n, c in enumerate(poly): + new_poly[n] ^= gf.mul(c, root) + poly = new_poly + # Invoke the root finding algorithm. + found_roots = poly_find_roots(poly, gf) + # The result must match the input, unless any roots were repeated. + if len(roots) == len(roots_set): + self.assertEqual(found_roots, sorted(roots)) + else: + self.assertEqual(found_roots, []) + + def test(self): + """Run tests.""" + for field_size in range(2, 65): + self.field_size_test(field_size) + +def berlekamp_massey(syndromes, gf): + """Implement the Berlekamp-Massey algorithm. + + Takes as input a sequence of GF(2^field_size) elements, and returns the shortest LSFR + that generates it, represented as a polynomial. + """ + # See https://en.wikipedia.org/wiki/Berlekamp%E2%80%93Massey_algorithm. + current = [1] + prev = [1] + b_inv = 1 + for n, discrepancy in enumerate(syndromes): + # Compute discrepancy + for i in range(1, len(current)): + discrepancy ^= gf.mul(syndromes[n - i], current[i]) + + # Correct if discrepancy is nonzero. + if discrepancy: + x = n + 1 - (len(current) - 1) - (len(prev) - 1) + if 2 * (len(current) - 1) <= n: + tmp = list(current) + current.extend(0 for _ in range(len(prev) + x - len(current))) + mul = gf.mul(discrepancy, b_inv) + for i, v in enumerate(prev): + current[i + x] ^= gf.mul(mul, v) + prev = tmp + b_inv = gf.inv(discrepancy) + else: + mul = gf.mul(discrepancy, b_inv) + for i, v in enumerate(prev): + current[i + x] ^= gf.mul(mul, v) + return current + +class Minisketch: + """A Minisketch sketch. + + This represents a sketch of a certain capacity, with elements of a certain bit size. + """ + + def __init__(self, field_size, capacity): + """Initialize an empty sketch with the specified field_size size and capacity.""" + self.field_size = field_size + self.capacity = capacity + self.odd_syndromes = [0] * capacity + self.gf = GF2Ops(field_size) + + def add(self, element): + """Add an element to this sketch. 1 <= element < 2**field_size.""" + sqr = self.gf.sqr(element) + for pos in range(self.capacity): + self.odd_syndromes[pos] ^= element + element = self.gf.mul(sqr, element) + + def serialized_size(self): + """Compute how many bytes a serialization of this sketch will be in size.""" + return (self.capacity * self.field_size + 7) // 8 + + def serialize(self): + """Serialize this sketch to bytes.""" + val = 0 + for i in range(self.capacity): + val |= self.odd_syndromes[i] << (self.field_size * i) + return val.to_bytes(self.serialized_size(), 'little') + + def deserialize(self, byte_data): + """Deserialize a byte array into this sketch, overwriting its contents.""" + assert len(byte_data) == self.serialized_size() + val = int.from_bytes(byte_data, 'little') + for i in range(self.capacity): + self.odd_syndromes[i] = (val >> (self.field_size * i)) & ((1 << self.field_size) - 1) + + def clone(self): + """Return a clone of this sketch.""" + ret = Minisketch(self.field_size, self.capacity) + ret.odd_syndromes = list(self.odd_syndromes) + ret.gf = self.gf + return ret + + def merge(self, other): + """Merge a sketch with another sketch. Corresponds to XOR'ing their serializations.""" + assert self.capacity == other.capacity + assert self.field_size == other.field_size + for i in range(self.capacity): + self.odd_syndromes[i] ^= other.odd_syndromes[i] + + def decode(self, max_count=None): + """Decode the contents of this sketch. + + Returns either a list of elements or None if undecodable. + """ + # We know the odd syndromes s1=x+y+..., s3=x^3+y^3+..., s5=..., and reconstruct the even + # syndromes from this: + # * s2 = x^2+y^2+.... = (x+y+...)^2 = s1^2 + # * s4 = x^4+y^4+.... = (x^2+y^2+...)^2 = s2^2 + # * s6 = x^6+y^6+.... = (x^3+y^3+...)^2 = s3^2 + all_syndromes = [0 for _ in range(2 * len(self.odd_syndromes))] + for i in range(len(self.odd_syndromes)): + all_syndromes[i * 2] = self.odd_syndromes[i] + all_syndromes[i * 2 + 1] = self.gf.sqr(all_syndromes[i]) + # Given the syndromes, find the polynomial that generates them. + poly = berlekamp_massey(all_syndromes, self.gf) + # Deal with failure and trivial cases. + if len(poly) == 0: + return None + if len(poly) == 1: + return [] + if max_count is not None and len(poly) > 1 + max_count: + return None + # If the polynomial can be factored into (1-m1*x)*(1-m2*x)*...*(1-mn*x), then {m1,m2,...,mn} + # is our set. As each factor (1-m*x) has 1/m as root, we're really just looking for the + # inverses of the roots. We find these by reversing the order of the coefficients, and + # finding the roots. + roots = poly_find_roots(list(reversed(poly)), self.gf) + if len(roots) == 0: + return None + return roots + +class TestMinisketch(unittest.TestCase): + """Test class for Minisketch.""" + + @classmethod + def construct_data(cls, field_size, num_a_only, num_b_only, num_both): + """Construct two random lists of elements in [1..2**field_size-1]. + + Each list will have unique elements that don't appear in the other (num_a_only in the first + and num_b_only in the second), and num_both elements will appear in both.""" + sample = [] + # Simulate random.sample here (which doesn't work with ranges over 2**63). + for _ in range(num_a_only + num_b_only + num_both): + while True: + r = random.randrange(1, 1 << field_size) + if r not in sample: + sample.append(r) + break + full_a = sample[:num_a_only + num_both] + full_b = sample[num_a_only:] + random.shuffle(full_a) + random.shuffle(full_b) + return full_a, full_b + + def field_size_capacity_test(self, field_size, capacity): + """Test Minisketch methods for a specific field and capacity.""" + used_capacity = random.randrange(capacity + 1) + num_a = random.randrange(used_capacity + 1) + num_both = random.randrange(min(2 * capacity, (1 << field_size) - 1 - used_capacity) + 1) + full_a, full_b = self.construct_data(field_size, num_a, used_capacity - num_a, num_both) + sketch_a = Minisketch(field_size, capacity) + sketch_b = Minisketch(field_size, capacity) + for v in full_a: + sketch_a.add(v) + for v in full_b: + sketch_b.add(v) + sketch_combined = sketch_a.clone() + sketch_b_ser = sketch_b.serialize() + sketch_b_received = Minisketch(field_size, capacity) + sketch_b_received.deserialize(sketch_b_ser) + sketch_combined.merge(sketch_b_received) + decode = sketch_combined.decode() + self.assertEqual(decode, sorted(set(full_a) ^ set(full_b))) + + def test(self): + """Run tests.""" + for field_size in range(2, 65): + for capacity in [0, 1, 2, 5, 10, field_size]: + self.field_size_capacity_test(field_size, min(capacity, (1 << field_size) - 1)) + +if __name__ == '__main__': + unittest.main() diff --git a/src/minisketchwrapper.cpp b/src/minisketchwrapper.cpp new file mode 100644 index 0000000000..fb176fb153 --- /dev/null +++ b/src/minisketchwrapper.cpp @@ -0,0 +1,77 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <minisketchwrapper.h> + +#include <logging.h> +#include <util/time.h> + +#include <minisketch.h> + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <utility> +#include <vector> + +namespace { + +static constexpr uint32_t BITS = 32; + +uint32_t FindBestImplementation() +{ + std::optional<std::pair<int64_t, uint32_t>> best; + + uint32_t max_impl = Minisketch::MaxImplementation(); + for (uint32_t impl = 0; impl <= max_impl; ++impl) { + std::vector<int64_t> benches; + uint64_t offset = 0; + /* Run a little benchmark with capacity 32, adding 184 entries, and decoding 11 of them once. */ + for (int b = 0; b < 11; ++b) { + if (!Minisketch::ImplementationSupported(BITS, impl)) break; + Minisketch sketch(BITS, impl, 32); + auto start = GetTimeMicros(); + for (uint64_t e = 0; e < 100; ++e) { + sketch.Add(e*1337 + b*13337 + offset); + } + for (uint64_t e = 0; e < 84; ++e) { + sketch.Add(e*1337 + b*13337 + offset); + } + offset += (*sketch.Decode(32))[0]; + auto stop = GetTimeMicros(); + benches.push_back(stop - start); + } + /* Remember which implementation has the best median benchmark time. */ + if (!benches.empty()) { + std::sort(benches.begin(), benches.end()); + if (!best || best->first > benches[5]) { + best = std::make_pair(benches[5], impl); + } + } + } + assert(best.has_value()); + LogPrintf("Using Minisketch implementation number %i\n", best->second); + return best->second; +} + +uint32_t Minisketch32Implementation() +{ + // Fast compute-once idiom. + static uint32_t best = FindBestImplementation(); + return best; +} + +} // namespace + + +Minisketch MakeMinisketch32(size_t capacity) +{ + return Minisketch(BITS, Minisketch32Implementation(), capacity); +} + +Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits) +{ + return Minisketch::CreateFP(BITS, Minisketch32Implementation(), max_elements, fpbits); +} diff --git a/src/minisketchwrapper.h b/src/minisketchwrapper.h new file mode 100644 index 0000000000..409221de79 --- /dev/null +++ b/src/minisketchwrapper.h @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_MINISKETCHWRAPPER_H +#define BITCOIN_MINISKETCHWRAPPER_H + +#include <minisketch.h> +#include <cstddef> +#include <cstdint> + +/** Wrapper around Minisketch::Minisketch(32, implementation, capacity). */ +Minisketch MakeMinisketch32(size_t capacity); +/** Wrapper around Minisketch::CreateFP. */ +Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits); + +#endif // BITCOIN_DBWRAPPER_H diff --git a/src/net.cpp b/src/net.cpp index c72cd75ba7..82e55d4189 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -25,6 +25,7 @@ #include <scheduler.h> #include <util/sock.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> #include <util/trace.h> @@ -123,7 +124,7 @@ void CConnman::AddAddrFetch(const std::string& strDest) uint16_t GetListenPort() { - return static_cast<uint16_t>(gArgs.GetArg("-port", Params().GetDefaultPort())); + return static_cast<uint16_t>(gArgs.GetIntArg("-port", Params().GetDefaultPort())); } // find 'best' local address for a particular peer @@ -192,8 +193,8 @@ CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) static int GetnScore(const CService& addr) { LOCK(cs_mapLocalHost); - if (mapLocalHost.count(addr) == 0) return 0; - return mapLocalHost[addr].nScore; + const auto it = mapLocalHost.find(addr); + return (it != mapLocalHost.end()) ? it->second.nScore : 0; } // Is our peer's addrLocal potentially useful as an external IP source? @@ -229,9 +230,27 @@ std::optional<CAddress> GetLocalAddrForPeer(CNode *pnode) return std::nullopt; } +/** + * If an IPv6 address belongs to the address range used by the CJDNS network and + * the CJDNS network is reachable (-cjdnsreachable config is set), then change + * the type from NET_IPV6 to NET_CJDNS. + * @param[in] service Address to potentially convert. + * @return a copy of `service` either unmodified or changed to CJDNS. + */ +CService MaybeFlipIPv6toCJDNS(const CService& service) +{ + CService ret{service}; + if (ret.m_net == NET_IPV6 && ret.m_addr[0] == 0xfc && IsReachable(NET_CJDNS)) { + ret.m_net = NET_CJDNS; + } + return ret; +} + // learn a new local address -bool AddLocal(const CService& addr, int nScore) +bool AddLocal(const CService& addr_, int nScore) { + CService addr{MaybeFlipIPv6toCJDNS(addr_)}; + if (!addr.IsRoutable()) return false; @@ -245,10 +264,10 @@ bool AddLocal(const CService& addr, int nScore) { LOCK(cs_mapLocalHost); - bool fAlready = mapLocalHost.count(addr) > 0; - LocalServiceInfo &info = mapLocalHost[addr]; - if (!fAlready || nScore >= info.nScore) { - info.nScore = nScore + (fAlready ? 1 : 0); + const auto [it, is_newly_added] = mapLocalHost.emplace(addr, LocalServiceInfo()); + LocalServiceInfo &info = it->second; + if (is_newly_added || nScore >= info.nScore) { + info.nScore = nScore + (is_newly_added ? 0 : 1); info.nPort = addr.GetPort(); } } @@ -290,12 +309,10 @@ bool IsReachable(const CNetAddr &addr) /** vote for a local address */ bool SeenLocal(const CService& addr) { - { - LOCK(cs_mapLocalHost); - if (mapLocalHost.count(addr) == 0) - return false; - mapLocalHost[addr].nScore++; - } + LOCK(cs_mapLocalHost); + const auto it = mapLocalHost.find(addr); + if (it == mapLocalHost.end()) return false; + ++it->second.nScore; return true; } @@ -410,7 +427,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo if (pszDest) { std::vector<CService> resolved; if (Lookup(pszDest, resolved, default_port, fNameLookup && !HaveNameProxy(), 256) && !resolved.empty()) { - addrConnect = CAddress(resolved[GetRand(resolved.size())], NODE_NONE); + const CService rnd{resolved[GetRand(resolved.size())]}; + addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; if (!addrConnect.IsValid()) { LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToString(), pszDest); return nullptr; @@ -627,25 +645,26 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) if (m_deserializer->Complete()) { // decompose a transport agnostic CNetMessage from the deserializer - uint32_t out_err_raw_size{0}; - std::optional<CNetMessage> result{m_deserializer->GetMessage(time, out_err_raw_size)}; - if (!result) { + bool reject_message{false}; + CNetMessage msg = m_deserializer->GetMessage(time, reject_message); + if (reject_message) { // Message deserialization failed. Drop the message but don't disconnect the peer. // store the size of the corrupt message - mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER)->second += out_err_raw_size; + mapRecvBytesPerMsgCmd.at(NET_MESSAGE_COMMAND_OTHER) += msg.m_raw_message_size; continue; } - //store received bytes per message command - //to prevent a memory DOS, only allow valid commands - mapMsgCmdSize::iterator i = mapRecvBytesPerMsgCmd.find(result->m_command); - if (i == mapRecvBytesPerMsgCmd.end()) + // Store received bytes per message command + // to prevent a memory DOS, only allow valid commands + auto i = mapRecvBytesPerMsgCmd.find(msg.m_command); + if (i == mapRecvBytesPerMsgCmd.end()) { i = mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER); + } assert(i != mapRecvBytesPerMsgCmd.end()); - i->second += result->m_raw_message_size; + i->second += msg.m_raw_message_size; // push the message to the process queue, - vRecvMsg.push_back(std::move(*result)); + vRecvMsg.push_back(std::move(msg)); complete = true; } @@ -719,16 +738,18 @@ const uint256& V1TransportDeserializer::GetMessageHash() const return data_hash; } -std::optional<CNetMessage> V1TransportDeserializer::GetMessage(const std::chrono::microseconds time, uint32_t& out_err_raw_size) +CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds time, bool& reject_message) { + // Initialize out parameter + reject_message = false; // decompose a single CNetMessage from the TransportDeserializer - std::optional<CNetMessage> msg(std::move(vRecv)); + CNetMessage msg(std::move(vRecv)); // store command string, time, and sizes - msg->m_command = hdr.GetCommand(); - msg->m_time = time; - msg->m_message_size = hdr.nMessageSize; - msg->m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; + msg.m_command = hdr.GetCommand(); + msg.m_time = time; + msg.m_message_size = hdr.nMessageSize; + msg.m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; uint256 hash = GetMessageHash(); @@ -738,17 +759,15 @@ std::optional<CNetMessage> V1TransportDeserializer::GetMessage(const std::chrono // Check checksum and header command string if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { LogPrint(BCLog::NET, "Header error: Wrong checksum (%s, %u bytes), expected %s was %s, peer=%d\n", - SanitizeString(msg->m_command), msg->m_message_size, + SanitizeString(msg.m_command), msg.m_message_size, HexStr(Span<uint8_t>(hash.begin(), hash.begin() + CMessageHeader::CHECKSUM_SIZE)), HexStr(hdr.pchChecksum), m_node_id); - out_err_raw_size = msg->m_raw_message_size; - msg = std::nullopt; + reject_message = true; } else if (!hdr.IsCommandValid()) { LogPrint(BCLog::NET, "Header error: Invalid message type (%s, %u bytes), peer=%d\n", - SanitizeString(hdr.GetCommand()), msg->m_message_size, m_node_id); - out_err_raw_size = msg->m_raw_message_size; - msg.reset(); + SanitizeString(hdr.GetCommand()), msg.m_message_size, m_node_id); + reject_message = true; } // Always reset the network deserializer (prepare for the next message) @@ -1092,9 +1111,11 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) { LogPrintf("Warning: Unknown socket family\n"); + } else { + addr = CAddress{MaybeFlipIPv6toCJDNS(addr), NODE_NONE}; } - const CAddress addr_bind = GetBindAddress(hSocket); + const CAddress addr_bind{MaybeFlipIPv6toCJDNS(GetBindAddress(hSocket)), NODE_NONE}; NetPermissionFlags permissionFlags = NetPermissionFlags::None; hListenSocket.AddSocketPermissionFlags(permissionFlags); @@ -1296,9 +1317,8 @@ void CConnman::NotifyNumConnectionsChanged() } } -bool CConnman::ShouldRunInactivityChecks(const CNode& node, std::optional<int64_t> now_in) const +bool CConnman::ShouldRunInactivityChecks(const CNode& node, int64_t now) const { - const int64_t now = now_in ? now_in.value() : GetTimeSeconds(); return node.nTimeConnected + m_peer_connect_timeout < now; } @@ -1617,6 +1637,7 @@ void CConnman::SocketHandler() void CConnman::ThreadSocketHandler() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET); while (!interruptNet) { DisconnectNodes(); @@ -1636,6 +1657,7 @@ void CConnman::WakeMessageHandler() void CConnman::ThreadDNSAddressSeed() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_DNS_SEED); FastRandomContext rng; std::vector<std::string> seeds = Params().DNSSeeds(); Shuffle(seeds.begin(), seeds.end(), rng); @@ -1818,6 +1840,7 @@ int CConnman::GetExtraBlockRelayCount() const void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_OPEN_CONNECTION); // Connect to specific addresses if (!connect.empty()) { @@ -2008,17 +2031,18 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) if (nTries > 100) break; - CAddrInfo addr; + CAddress addr; + int64_t addr_last_try{0}; if (fFeeler) { // First, try to get a tried table collision address. This returns // an empty (invalid) address if there are no collisions to try. - addr = addrman.SelectTriedCollision(); + std::tie(addr, addr_last_try) = addrman.SelectTriedCollision(); if (!addr.IsValid()) { // No tried table collisions. Select a new table address // for our feeler. - addr = addrman.Select(true); + std::tie(addr, addr_last_try) = addrman.Select(true); } else if (AlreadyConnectedToAddress(addr)) { // If test-before-evict logic would have us connect to a // peer that we're already connected to, just mark that @@ -2027,11 +2051,11 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // a currently-connected peer. addrman.Good(addr); // Select a new table address for our feeler instead. - addr = addrman.Select(true); + std::tie(addr, addr_last_try) = addrman.Select(true); } } else { // Not a feeler - addr = addrman.Select(); + std::tie(addr, addr_last_try) = addrman.Select(); } // Require outbound connections, other than feelers, to be to distinct network groups @@ -2048,7 +2072,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; // only consider very recently tried nodes after 30 failed attempts - if (nANow - addr.nLastTry < 600 && nTries < 30) + if (nANow - addr_last_try < 600 && nTries < 30) continue; // for non-feelers, require all the services we'll want, @@ -2157,6 +2181,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const void CConnman::ThreadOpenAddedConnections() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_ADD_CONNECTION); while (true) { CSemaphoreGrant grant(*semAddnode); @@ -2220,6 +2245,7 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai void CConnman::ThreadMessageHandler() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::MESSAGE_HANDLER); FastRandomContext rng; while (!flagInterruptMsgProc) { @@ -2439,7 +2465,7 @@ void CConnman::SetNetworkActive(bool active) } } -CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, CAddrMan& addrman_in, bool network_active) +CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, AddrMan& addrman_in, bool network_active) : addrman(addrman_in), nSeed0(nSeed0In), nSeed1(nSeed1In) { SetTryNewOutboundPeer(false); @@ -2455,7 +2481,10 @@ NodeId CConnman::GetNewNodeId() } -bool CConnman::Bind(const CService &addr, unsigned int flags, NetPermissionFlags permissions) { +bool CConnman::Bind(const CService& addr_, unsigned int flags, NetPermissionFlags permissions) +{ + const CService addr{MaybeFlipIPv6toCJDNS(addr_)}; + if (!(flags & BF_EXPLICIT) && !IsReachable(addr)) { return false; } @@ -2976,7 +3005,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const LogPrint(BCLog::NET, "Added connection peer=%d\n", id); } - m_deserializer = std::make_unique<V1TransportDeserializer>(V1TransportDeserializer(Params(), GetId(), SER_NETWORK, INIT_PROTO_VERSION)); + m_deserializer = std::make_unique<V1TransportDeserializer>(V1TransportDeserializer(Params(), id, SER_NETWORK, INIT_PROTO_VERSION)); m_serializer = std::make_unique<V1TransportSerializer>(V1TransportSerializer()); } @@ -7,10 +7,10 @@ #define BITCOIN_NET_H #include <addrman.h> -#include <amount.h> -#include <bloom.h> #include <chainparams.h> +#include <common/bloom.h> #include <compat.h> +#include <consensus/amount.h> #include <crypto/siphash.h> #include <hash.h> #include <i2p.h> @@ -308,7 +308,7 @@ public: /** read and deserialize data, advances msg_bytes data pointer */ virtual int Read(Span<const uint8_t>& msg_bytes) = 0; // decomposes a message from the context - virtual std::optional<CNetMessage> GetMessage(std::chrono::microseconds time, uint32_t& out_err) = 0; + virtual CNetMessage GetMessage(std::chrono::microseconds time, bool& reject_message) = 0; virtual ~TransportDeserializer() {} }; @@ -372,7 +372,7 @@ public: } return ret; } - std::optional<CNetMessage> GetMessage(std::chrono::microseconds time, uint32_t& out_err_raw_size) override; + CNetMessage GetMessage(std::chrono::microseconds time, bool& reject_message) override; }; /** The TransportSerializer prepares messages for the network transport @@ -797,7 +797,7 @@ public: m_onion_binds = connOptions.onion_binds; } - CConnman(uint64_t seed0, uint64_t seed1, CAddrMan& addrman, bool network_active = true); + CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, bool network_active = true); ~CConnman(); bool Start(CScheduler& scheduler, const Options& options); @@ -942,7 +942,7 @@ public: std::chrono::microseconds PoissonNextSendInbound(std::chrono::microseconds now, std::chrono::seconds average_interval); /** Return true if we should disconnect the peer for failing an inactivity check. */ - bool ShouldRunInactivityChecks(const CNode& node, std::optional<int64_t> now=std::nullopt) const; + bool ShouldRunInactivityChecks(const CNode& node, int64_t secs_now) const; private: struct ListenSocket { @@ -1049,7 +1049,7 @@ private: std::vector<ListenSocket> vhListenSocket; std::atomic<bool> fNetworkActive{true}; bool fAddressesInitialized{false}; - CAddrMan& addrman; + AddrMan& addrman; std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); RecursiveMutex m_addr_fetches_mutex; std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 80655c61e7..2185ccc700 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -10,6 +10,7 @@ #include <blockencodings.h> #include <blockfilter.h> #include <chainparams.h> +#include <consensus/amount.h> #include <consensus/validation.h> #include <deploymentstatus.h> #include <hash.h> @@ -291,7 +292,7 @@ using PeerRef = std::shared_ptr<Peer>; class PeerManagerImpl final : public PeerManager { public: - PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, + PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs); @@ -409,7 +410,7 @@ private: const CChainParams& m_chainparams; CConnman& m_connman; - CAddrMan& m_addrman; + AddrMan& m_addrman; /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* const m_banman; ChainstateManager& m_chainman; @@ -460,9 +461,9 @@ private: bool AlreadyHaveTx(const GenTxid& gtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** - * Filter for transactions that were recently rejected by - * AcceptToMemoryPool. These are not rerequested until the chain tip - * changes, at which point the entire filter is reset. + * Filter for transactions that were recently rejected by the mempool. + * These are not rerequested until the chain tip changes, at which point + * the entire filter is reset. * * Without this filter we'd be re-requesting txs from each of our peers, * increasing bandwidth consumption considerably. For instance, with 100 @@ -884,6 +885,12 @@ bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, st void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) { AssertLockHeld(cs_main); + + // Never request high-bandwidth mode from peers if we're blocks-only. Our + // mempool will not contain the transactions necessary to reconstruct the + // compact block. + if (m_ignore_incoming_txs) return; + CNodeState* nodestate = State(nodeid); if (!nodestate || !nodestate->fSupportsDesiredCmpctVersion) { // Never ask from peers who can't provide witnesses. @@ -1300,7 +1307,7 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c void PeerManagerImpl::AddToCompactExtraTransactions(const CTransactionRef& tx) { - size_t max_extra_txn = gArgs.GetArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); + size_t max_extra_txn = gArgs.GetIntArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); if (max_extra_txn <= 0) return; if (!vExtraTxnForCompact.size()) @@ -1402,6 +1409,7 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat case TxValidationResult::TX_WITNESS_STRIPPED: case TxValidationResult::TX_CONFLICT: case TxValidationResult::TX_MEMPOOL_POLICY: + case TxValidationResult::TX_NO_MEMPOOL: break; } if (message != "") { @@ -1419,14 +1427,14 @@ bool PeerManagerImpl::BlockRequestAllowed(const CBlockIndex* pindex) (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, m_chainparams.GetConsensus()) < STALE_RELAY_AGE_LIMIT); } -std::unique_ptr<PeerManager> PeerManager::make(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, +std::unique_ptr<PeerManager> PeerManager::make(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs) { return std::make_unique<PeerManagerImpl>(chainparams, connman, addrman, banman, chainman, pool, ignore_incoming_txs); } -PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, +PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs) : m_chainparams(chainparams), @@ -2165,7 +2173,11 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } if (vGetData.size() > 0) { - if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + if (!m_ignore_incoming_txs && + nodestate->fSupportsDesiredCmpctVersion && + vGetData.size() == 1 && + mapBlocksInFlight.size() == 1 && + pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } @@ -2231,7 +2243,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) const auto [porphanTx, from_peer] = m_orphanage.GetTx(orphanHash); if (porphanTx == nullptr) continue; - const MempoolAcceptResult result = AcceptToMemoryPool(m_chainman.ActiveChainstate(), m_mempool, porphanTx, false /* bypass_limits */); + const MempoolAcceptResult result = m_chainman.ProcessTransaction(porphanTx); const TxValidationState& state = result.m_state; if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { @@ -2288,7 +2300,6 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) break; } } - m_mempool.check(m_chainman.ActiveChainstate()); } bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, @@ -2653,7 +2664,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // table is also potentially detrimental because new-table entries // are subject to eviction in the event of addrman collisions. We // mitigate the information-leak by never calling - // CAddrMan::Connected() on block-relay-only peers; see + // AddrMan::Connected() on block-relay-only peers; see // FinalizeNode(). // // This moves an address from New to Tried table in Addrman, @@ -2949,16 +2960,17 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, best_block = &inv.hash; } } else if (inv.IsGenTxMsg()) { + if (reject_tx_invs) { + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } const GenTxid gtxid = ToGenTxid(inv); const bool fAlreadyHave = AlreadyHaveTx(gtxid); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); pfrom.AddKnownTx(inv.hash); - if (reject_tx_invs) { - LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); - pfrom.fDisconnect = true; - return; - } else if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { AddTxAnnouncement(pfrom, gtxid, current_time); } } else { @@ -3231,12 +3243,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // already; and an adversary can already relay us old transactions // (older than our recency filter) if trying to DoS us, without any need // for witness malleation. - if (AlreadyHaveTx(GenTxid(/* is_wtxid=*/true, wtxid))) { + if (AlreadyHaveTx(GenTxid::Wtxid(wtxid))) { if (pfrom.HasPermission(NetPermissionFlags::ForceRelay)) { // Always relay transactions received from peers with forcerelay // permission, even if they were already in the mempool, allowing // the node to function as a gateway for nodes hidden behind it. - if (!m_mempool.exists(tx.GetHash())) { + if (!m_mempool.exists(GenTxid::Txid(tx.GetHash()))) { LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); @@ -3246,11 +3258,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - const MempoolAcceptResult result = AcceptToMemoryPool(m_chainman.ActiveChainstate(), m_mempool, ptx, false /* bypass_limits */); + const MempoolAcceptResult result = m_chainman.ProcessTransaction(ptx); const TxValidationState& state = result.m_state; if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { - m_mempool.check(m_chainman.ActiveChainstate()); // As this version of the transaction was acceptable, we can forget about any // requests for it. m_txrequest.ForgetTxHash(tx.GetHash()); @@ -3301,7 +3312,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // wtxidrelay peers. // Eventually we should replace this with an improved // protocol for getting all unconfirmed parents. - const GenTxid gtxid{/* is_wtxid=*/false, parent_txid}; + const auto gtxid{GenTxid::Txid(parent_txid)}; pfrom.AddKnownTx(parent_txid); if (!AlreadyHaveTx(gtxid)) AddTxAnnouncement(pfrom, gtxid, current_time); } @@ -3315,7 +3326,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_txrequest.ForgetTxHash(tx.GetWitnessHash()); // DoS prevention: do not allow m_orphanage to grow unbounded (see CVE-2012-3789) - unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetIntArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); unsigned int nEvicted = m_orphanage.LimitOrphans(nMaxOrphanTx); if (nEvicted > 0) { LogPrint(BCLog::MEMPOOL, "orphanage overflow, removed %u tx\n", nEvicted); @@ -3369,8 +3380,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } // If a tx has been detected by m_recent_rejects, we will have reached - // this point and the tx will have been ignored. Because we haven't run - // the tx through AcceptToMemoryPool, we won't have computed a DoS + // this point and the tx will have been ignored. Because we haven't + // submitted the tx to our mempool, we won't have computed a DoS // score for it or determined exactly why we consider it invalid. // // This means we won't penalize any peer subsequently relaying a DoSy @@ -4301,8 +4312,11 @@ void PeerManagerImpl::CheckForStaleTipAndEvictPeers() void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::microseconds now) { - if (m_connman.ShouldRunInactivityChecks(node_to) && peer.m_ping_nonce_sent && + if (m_connman.ShouldRunInactivityChecks(node_to, std::chrono::duration_cast<std::chrono::seconds>(now).count()) && + peer.m_ping_nonce_sent && now > peer.m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL}) { + // The ping timeout is using mocktime. To disable the check during + // testing, increase -peertimeout. LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(now - peer.m_ping_start.load()), peer.m_id); node_to.fDisconnect = true; return; @@ -4416,7 +4430,7 @@ void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, std::chrono::microseconds c // peers with the forcerelay permission should not filter txs to us if (pto.HasPermission(NetPermissionFlags::ForceRelay)) return; - CAmount currentFilter = m_mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); + CAmount currentFilter = m_mempool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); static FeeFilterRounder g_filter_rounder{CFeeRate{DEFAULT_MIN_RELAY_TX_FEE}}; if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) { diff --git a/src/net_processing.h b/src/net_processing.h index 9d8d788583..27bc40687a 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -9,7 +9,7 @@ #include <net.h> #include <validationinterface.h> -class CAddrMan; +class AddrMan; class CChainParams; class CTxMemPool; class ChainstateManager; @@ -37,7 +37,7 @@ struct CNodeStateStats { class PeerManager : public CValidationInterface, public NetEventsInterface { public: - static std::unique_ptr<PeerManager> make(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, + static std::unique_ptr<PeerManager> make(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs); virtual ~PeerManager() { } diff --git a/src/netaddress.cpp b/src/netaddress.cpp index b2f4945e3b..7f1dd698b0 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -165,7 +165,7 @@ void CNetAddr::SetLegacyIPv6(Span<const uint8_t> ipv6) } /** - * Create an "internal" address that represents a name or FQDN. CAddrMan uses + * Create an "internal" address that represents a name or FQDN. AddrMan uses * these fake addresses to keep track of which DNS seeds were used. * @returns Whether or not the operation was successful. * @see NET_INTERNAL, INTERNAL_IN_IPV6_PREFIX, CNetAddr::IsInternal(), CNetAddr::IsRFC4193() @@ -663,7 +663,7 @@ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const } /** - * Try to get our IPv6 address. + * Try to get our IPv6 (or CJDNS) address. * * @param[out] pipv6Addr The in6_addr struct to which to copy. * @@ -674,7 +674,7 @@ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const */ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const { - if (!IsIPv6()) { + if (!IsIPv6() && !IsCJDNS()) { return false; } assert(sizeof(*pipv6Addr) == m_addr.size()); @@ -794,8 +794,14 @@ std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) co vchRet.push_back((ipv4 >> 24) & 0xFF); vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; - } else if (IsTor() || IsI2P() || IsCJDNS()) { + } else if (IsTor() || IsI2P()) { nBits = 4; + } else if (IsCJDNS()) { + // Treat in the same way as Tor and I2P because the address in all of + // them is "random" bytes (derived from a public key). However in CJDNS + // the first byte is a constant 0xfc, so the random bytes come after it. + // Thus skip the constant 8 bits at the start. + nBits = 12; } else if (IsHeNet()) { // for he.net, use /36 groups nBits = 36; @@ -892,6 +898,11 @@ int CNetAddr::GetReachabilityFrom(const CNetAddr *paddrPartner) const case NET_I2P: return REACH_PRIVATE; default: return REACH_DEFAULT; } + case NET_CJDNS: + switch (ourNet) { + case NET_CJDNS: return REACH_PRIVATE; + default: return REACH_DEFAULT; + } case NET_TEREDO: switch(ourNet) { default: return REACH_DEFAULT; @@ -993,7 +1004,7 @@ bool CService::GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const paddrin->sin_port = htons(port); return true; } - if (IsIPv6()) { + if (IsIPv6() || IsCJDNS()) { if (*addrlen < (socklen_t)sizeof(struct sockaddr_in6)) return false; *addrlen = sizeof(struct sockaddr_in6); diff --git a/src/netaddress.h b/src/netaddress.h index cfb2edcd34..b0b1c5ca9e 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -62,7 +62,7 @@ enum Network { NET_CJDNS, /// A set of addresses that represent the hash of a string or FQDN. We use - /// them in CAddrMan to keep track of which DNS seeds were used. + /// them in AddrMan to keep track of which DNS seeds were used. NET_INTERNAL, /// Dummy value to indicate the number of NET_* constants. @@ -224,7 +224,7 @@ public: */ bool IsRelayable() const { - return IsIPv4() || IsIPv6() || IsTor() || IsI2P(); + return IsIPv4() || IsIPv6() || IsTor() || IsI2P() || IsCJDNS(); } /** @@ -253,7 +253,6 @@ public: } } - friend class CNetAddrHash; friend class CSubNet; private: @@ -467,22 +466,6 @@ private: } }; -class CNetAddrHash -{ -public: - size_t operator()(const CNetAddr& a) const noexcept - { - CSipHasher hasher(m_salt_k0, m_salt_k1); - hasher.Write(a.m_net); - hasher.Write(a.m_addr.data(), a.m_addr.size()); - return static_cast<size_t>(hasher.Finalize()); - } - -private: - const uint64_t m_salt_k0 = GetRand(std::numeric_limits<uint64_t>::max()); - const uint64_t m_salt_k1 = GetRand(std::numeric_limits<uint64_t>::max()); -}; - class CSubNet { protected: @@ -565,6 +548,26 @@ public: READWRITEAS(CNetAddr, obj); READWRITE(Using<BigEndianFormatter<2>>(obj.port)); } + + friend class CServiceHash; + friend CService MaybeFlipIPv6toCJDNS(const CService& service); +}; + +class CServiceHash +{ +public: + size_t operator()(const CService& a) const noexcept + { + CSipHasher hasher(m_salt_k0, m_salt_k1); + hasher.Write(a.m_net); + hasher.Write(a.port); + hasher.Write(a.m_addr.data(), a.m_addr.size()); + return static_cast<size_t>(hasher.Finalize()); + } + +private: + const uint64_t m_salt_k0 = GetRand(std::numeric_limits<uint64_t>::max()); + const uint64_t m_salt_k1 = GetRand(std::numeric_limits<uint64_t>::max()); }; #endif // BITCOIN_NETADDRESS_H diff --git a/src/netbase.cpp b/src/netbase.cpp index 64d17189a6..6191f25cd9 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -96,6 +96,9 @@ enum Network ParseNetwork(const std::string& net_in) { if (net == "i2p") { return NET_I2P; } + if (net == "cjdns") { + return NET_CJDNS; + } return NET_UNROUTABLE; } @@ -120,7 +123,7 @@ std::vector<std::string> GetNetworkNames(bool append_unroutable) std::vector<std::string> names; for (int n = 0; n < NET_MAX; ++n) { const enum Network network{static_cast<Network>(n)}; - if (network == NET_UNROUTABLE || network == NET_CJDNS || network == NET_INTERNAL) continue; + if (network == NET_UNROUTABLE || network == NET_INTERNAL) continue; names.emplace_back(GetNetworkName(network)); } if (append_unroutable) { diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 5ddcf95c84..53bc2b5069 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -16,6 +16,7 @@ #include <signet.h> #include <streams.h> #include <undo.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <validation.h> @@ -67,13 +68,14 @@ void CleanupBlockRevFiles() LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); fs::path blocksdir = gArgs.GetBlocksDirPath(); for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + const std::string path = fs::PathToString(it->path().filename()); if (fs::is_regular_file(*it) && - it->path().filename().string().length() == 12 && - it->path().filename().string().substr(8,4) == ".dat") + path.length() == 12 && + path.substr(8,4) == ".dat") { - if (it->path().filename().string().substr(0, 3) == "blk") { - mapBlockFiles[it->path().filename().string().substr(3, 5)] = it->path(); - } else if (it->path().filename().string().substr(0, 3) == "rev") { + if (path.substr(0, 3) == "blk") { + mapBlockFiles[path.substr(3, 5)] = it->path(); + } else if (path.substr(0, 3) == "rev") { remove(it->path()); } } @@ -85,7 +87,7 @@ void CleanupBlockRevFiles() // start removing block files. int nContigCounter = 0; for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { - if (atoi(item.first) == nContigCounter) { + if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) { nContigCounter++; continue; } @@ -203,7 +205,7 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) FlatFilePos pos(*it, 0); fs::remove(BlockFileSeq().FileName(pos)); fs::remove(UndoFileSeq().FileName(pos)); - LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + LogPrint(BCLog::BLOCKSTORE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -260,7 +262,7 @@ bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, if ((int)nFile != nLastBlockFile) { if (!fKnown) { - LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + LogPrint(BCLog::BLOCKSTORE, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); } FlushBlockFile(!fKnown, finalize_undo); nLastBlockFile = nFile; @@ -489,6 +491,7 @@ struct CImportingNow { void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); { @@ -525,14 +528,14 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile for (const fs::path& path : vImportFiles) { FILE* file = fsbridge::fopen(path, "rb"); if (file) { - LogPrintf("Importing blocks file %s...\n", path.string()); + LogPrintf("Importing blocks file %s...\n", fs::PathToString(path)); chainman.ActiveChainstate().LoadExternalBlockFile(file); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; } } else { - LogPrintf("Warning: Could not open blocks file %s\n", path.string()); + LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path)); } } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 69e856dd15..9e9503ff5d 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_NODE_COINSTATS_H #define BITCOIN_NODE_COINSTATS_H -#include <amount.h> #include <chain.h> #include <coins.h> +#include <consensus/amount.h> #include <streams.h> #include <uint256.h> diff --git a/src/node/context.h b/src/node/context.h index 135f9ea1c6..26873345b4 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -12,7 +12,7 @@ class ArgsManager; class BanMan; -class CAddrMan; +class AddrMan; class CBlockPolicyEstimator; class CConnman; class CScheduler; @@ -39,7 +39,7 @@ class WalletClient; struct NodeContext { //! Init interface for initializing current process and connecting to other processes. interfaces::Init* init{nullptr}; - std::unique_ptr<CAddrMan> addrman; + std::unique_ptr<AddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; std::unique_ptr<CBlockPolicyEstimator> fee_estimator; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index b1d39cd00a..9f457570fa 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -67,6 +67,17 @@ using interfaces::WalletClient; namespace node { namespace { +#ifdef ENABLE_EXTERNAL_SIGNER +class ExternalSignerImpl : public interfaces::ExternalSigner +{ +public: + ExternalSignerImpl(::ExternalSigner signer) : m_signer(std::move(signer)) {} + std::string getName() override { return m_signer.m_name; } +private: + ::ExternalSigner m_signer; +}; +#endif + class NodeImpl : public Node { private: @@ -172,14 +183,18 @@ public: } return false; } - std::vector<ExternalSigner> externalSigners() override + std::vector<std::unique_ptr<interfaces::ExternalSigner>> listExternalSigners() override { #ifdef ENABLE_EXTERNAL_SIGNER std::vector<ExternalSigner> signers = {}; const std::string command = gArgs.GetArg("-signer", ""); - if (command == "") return signers; + if (command == "") return {}; ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); - return signers; + std::vector<std::unique_ptr<interfaces::ExternalSigner>> result; + for (auto& signer : signers) { + result.emplace_back(std::make_unique<ExternalSignerImpl>(std::move(signer))); + } + return result; #else // This result is indistinguishable from a successful call that returns // no signers. For the current GUI this doesn't matter, because the wallet @@ -559,7 +574,7 @@ public: { if (!m_node.mempool) return false; LOCK(m_node.mempool->cs); - return m_node.mempool->exists(txid); + return m_node.mempool->exists(GenTxid::Txid(txid)); } bool hasDescendantsInMempool(const uint256& txid) override { @@ -579,16 +594,16 @@ public: // that Chain clients do not need to know about. return TransactionError::OK == err; } - void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) override + void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize, CAmount* ancestorfees) override { ancestors = descendants = 0; if (!m_node.mempool) return; - m_node.mempool->GetTransactionAncestry(txid, ancestors, descendants); + m_node.mempool->GetTransactionAncestry(txid, ancestors, descendants, ancestorsize, ancestorfees); } void getPackageLimits(unsigned int& limit_ancestor_count, unsigned int& limit_descendant_count) override { - limit_ancestor_count = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); - limit_descendant_count = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + limit_ancestor_count = gArgs.GetIntArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + limit_descendant_count = gArgs.GetIntArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); } bool checkChainLimits(const CTransactionRef& tx) override { @@ -596,10 +611,10 @@ public: LockPoints lp; CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); CTxMemPool::setEntries ancestors; - auto limit_ancestor_count = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); - auto limit_ancestor_size = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; - auto limit_descendant_count = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); - auto limit_descendant_size = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; + auto limit_ancestor_count = gArgs.GetIntArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + auto limit_ancestor_size = gArgs.GetIntArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; + auto limit_descendant_count = gArgs.GetIntArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + auto limit_descendant_size = gArgs.GetIntArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; std::string unused_error_string; LOCK(m_node.mempool->cs); return m_node.mempool->CalculateMemPoolAncestors( @@ -619,7 +634,7 @@ public: CFeeRate mempoolMinFee() override { if (!m_node.mempool) return {}; - return m_node.mempool->GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + return m_node.mempool->GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } CFeeRate relayMinFee() override { return ::minRelayTxFee; } CFeeRate relayIncrementalFee() override { return ::incrementalRelayFee; } @@ -702,7 +717,7 @@ public: notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } - bool isTaprootActive() const override + bool isTaprootActive() override { LOCK(::cs_main); const CBlockIndex* tip = Assert(m_node.chainman)->ActiveChain().Tip(); diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index b013b6d579..9ad65d15d2 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/tx_verify.h> #include <node/psbt.h> #include <policy/policy.h> diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 2a7bcc057f..33b8e9351c 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -70,8 +70,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t if (max_tx_fee > 0) { // First, call ATMP with test_accept and check the fee. If ATMP // fails here, return error immediately. - const MempoolAcceptResult result = AcceptToMemoryPool(node.chainman->ActiveChainstate(), *node.mempool, tx, false /* bypass_limits */, - true /* test_accept */); + const MempoolAcceptResult result = node.chainman->ProcessTransaction(tx, /*test_accept=*/ true); if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { return HandleATMPError(result.m_state, err_string); } else if (result.m_base_fees.value() > max_tx_fee) { @@ -79,8 +78,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t } } // Try to submit the transaction to the mempool. - const MempoolAcceptResult result = AcceptToMemoryPool(node.chainman->ActiveChainstate(), *node.mempool, tx, false /* bypass_limits */, - false /* test_accept */); + const MempoolAcceptResult result = node.chainman->ProcessTransaction(tx, /*test_accept=*/ false); if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { return HandleATMPError(result.m_state, err_string); } diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp index 25b9282b4e..ce149067b7 100644 --- a/src/policy/feerate.cpp +++ b/src/policy/feerate.cpp @@ -7,6 +7,8 @@ #include <tinyformat.h> +#include <cmath> + CFeeRate::CFeeRate(const CAmount& nFeePaid, uint32_t num_bytes) { const int64_t nSize{num_bytes}; @@ -22,7 +24,9 @@ CAmount CFeeRate::GetFee(uint32_t num_bytes) const { const int64_t nSize{num_bytes}; - CAmount nFee = nSatoshisPerK * nSize / 1000; + // Be explicit that we're converting from a double to int64_t (CAmount) here. + // We've previously had issues with the silent double->int64_t conversion. + CAmount nFee{static_cast<CAmount>(std::ceil(nSatoshisPerK * nSize / 1000.0))}; if (nFee == 0 && nSize != 0) { if (nSatoshisPerK > 0) nFee = CAmount(1); diff --git a/src/policy/feerate.h b/src/policy/feerate.h index d296d32774..8ba896bb01 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_POLICY_FEERATE_H #define BITCOIN_POLICY_FEERATE_H -#include <amount.h> +#include <consensus/amount.h> #include <serialize.h> #include <string> @@ -48,6 +48,7 @@ public: CFeeRate(const CAmount& nFeePaid, uint32_t num_bytes); /** * Return the fee in satoshis for the given size in bytes. + * If the calculated fee would have fractional satoshis, then the returned fee will always be rounded up to the nearest satoshi. */ CAmount GetFee(uint32_t num_bytes) const; /** diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 2ae5798ebe..d8c21bd833 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -527,7 +527,7 @@ CBlockPolicyEstimator::CBlockPolicyEstimator() fs::path est_filepath = gArgs.GetDataDirNet() / FEE_ESTIMATES_FILENAME; CAutoFile est_file(fsbridge::fopen(est_filepath, "rb"), SER_DISK, CLIENT_VERSION); if (est_file.IsNull() || !Read(est_file)) { - LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", est_filepath.string()); + LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(est_filepath)); } } @@ -549,7 +549,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo if (txHeight != nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random they don't // affect the estimate. We'll potentially double count transactions in 1-block reorgs. - // Ignore txs if BlockPolicyEstimator is not in sync with ::ChainActive().Tip(). + // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip(). // It will be synced next time a block is processed. return; } @@ -887,7 +887,7 @@ void CBlockPolicyEstimator::Flush() { fs::path est_filepath = gArgs.GetDataDirNet() / FEE_ESTIMATES_FILENAME; CAutoFile est_file(fsbridge::fopen(est_filepath, "wb"), SER_DISK, CLIENT_VERSION); if (est_file.IsNull() || !Write(est_file)) { - LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", est_filepath.string()); + LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(est_filepath)); } } diff --git a/src/policy/fees.h b/src/policy/fees.h index c444d71a31..27f9120c64 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_POLICY_FEES_H #define BITCOIN_POLICY_FEES_H -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <uint256.h> #include <random.h> diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 9e433584e7..fced397e51 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -22,7 +22,7 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) // so dust is a spendable txout less than // 182*dustRelayFee/1000 (in satoshis). // 546 satoshis at the default rate of 3000 sat/kvB. - // A typical spendable segwit txout is 31 bytes big, and will + // A typical spendable segwit P2WPKH txout is 31 bytes big, and will // need a CTxIn of at least 67 bytes to spend: // so dust is a spendable txout less than // 98*dustRelayFee/1000 (in satoshis). @@ -34,6 +34,11 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) int witnessversion = 0; std::vector<unsigned char> witnessprogram; + // Note this computation is for spending a Segwit v0 P2WPKH output (a 33 bytes + // public key + an ECDSA signature). For Segwit v1 Taproot outputs the minimum + // satisfaction is lower (a single BIP340 signature) but this computation was + // kept to not further reduce the dust level. + // See discussion in https://github.com/bitcoin/bitcoin/pull/22779 for details. if (txout.scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { // sum the sizes of the parts of a transaction input // with 75% segwit discount applied to the script size. diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 15527afb8a..7e6b0cf245 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -22,7 +22,7 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) // If this transaction is not in our mempool, then we can't be sure // we will know about all its inputs. - if (!pool.exists(tx.GetHash())) { + if (!pool.exists(GenTxid::Txid(tx.GetHash()))) { return RBFTransactionState::UNKNOWN; } @@ -48,17 +48,19 @@ RBFTransactionState IsRBFOptInEmptyMempool(const CTransaction& tx) } std::optional<std::string> GetEntriesForConflicts(const CTransaction& tx, - CTxMemPool& pool, - const CTxMemPool::setEntries& iters_conflicting, - CTxMemPool::setEntries& all_conflicts) + CTxMemPool& pool, + const CTxMemPool::setEntries& iters_conflicting, + CTxMemPool::setEntries& all_conflicts) { AssertLockHeld(pool.cs); const uint256 txid = tx.GetHash(); uint64_t nConflictingCount = 0; for (const auto& mi : iters_conflicting) { nConflictingCount += mi->GetCountWithDescendants(); - // This potentially overestimates the number of actual descendants but we just want to be - // conservative to avoid doing too much work. + // BIP125 Rule #5: don't consider replacing more than MAX_BIP125_REPLACEMENT_CANDIDATES + // entries from the mempool. This potentially overestimates the number of actual + // descendants (i.e. if multiple conflicts share a descendant, it will be counted multiple + // times), but we just want to be conservative to avoid doing too much work. if (nConflictingCount > MAX_BIP125_REPLACEMENT_CANDIDATES) { return strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", txid.ToString(), @@ -66,8 +68,7 @@ std::optional<std::string> GetEntriesForConflicts(const CTransaction& tx, MAX_BIP125_REPLACEMENT_CANDIDATES); } } - // If not too many to replace, then calculate the set of - // transactions that would have to be evicted + // Calculate the set of all transactions that would have to be evicted. for (CTxMemPool::txiter it : iters_conflicting) { pool.CalculateDescendants(it, all_conflicts); } @@ -81,23 +82,23 @@ std::optional<std::string> HasNoNewUnconfirmed(const CTransaction& tx, AssertLockHeld(pool.cs); std::set<uint256> parents_of_conflicts; for (const auto& mi : iters_conflicting) { - for (const CTxIn &txin : mi->GetTx().vin) { + for (const CTxIn& txin : mi->GetTx().vin) { parents_of_conflicts.insert(txin.prevout.hash); } } for (unsigned int j = 0; j < tx.vin.size(); j++) { - // We don't want to accept replacements that require low feerate junk to be mined first. - // Ideally we'd keep track of the ancestor feerates and make the decision based on that, but - // for now requiring all new inputs to be confirmed works. + // BIP125 Rule #2: We don't want to accept replacements that require low feerate junk to be + // mined first. Ideally we'd keep track of the ancestor feerates and make the decision + // based on that, but for now requiring all new inputs to be confirmed works. // // Note that if you relax this to make RBF a little more useful, this may break the - // CalculateMempoolAncestors RBF relaxation, above. See the comment above the first - // CalculateMempoolAncestors call for more info. + // CalculateMempoolAncestors RBF relaxation which subtracts the conflict count/size from the + // descendant limit. if (!parents_of_conflicts.count(tx.vin[j].prevout.hash)) { // Rather than check the UTXO set - potentially expensive - it's cheaper to just check // if the new input refers to a tx that's in the mempool. - if (pool.exists(tx.vin[j].prevout.hash)) { + if (pool.exists(GenTxid::Txid(tx.vin[j].prevout.hash))) { return strprintf("replacement %s adds unconfirmed input, idx %d", tx.GetHash().ToString(), j); } @@ -111,7 +112,7 @@ std::optional<std::string> EntriesAndTxidsDisjoint(const CTxMemPool::setEntries& const uint256& txid) { for (CTxMemPool::txiter ancestorIt : ancestors) { - const uint256 &hashAncestor = ancestorIt->GetTx().GetHash(); + const uint256& hashAncestor = ancestorIt->GetTx().GetHash(); if (direct_conflicts.count(hashAncestor)) { return strprintf("%s spends conflicting transaction %s", txid.ToString(), @@ -150,24 +151,26 @@ std::optional<std::string> PaysMoreThanConflicts(const CTxMemPool::setEntries& i std::optional<std::string> PaysForRBF(CAmount original_fees, CAmount replacement_fees, size_t replacement_vsize, + CFeeRate relay_fee, const uint256& txid) { - // The replacement must pay greater fees than the transactions it - // replaces - if we did the bandwidth used by those conflicting - // transactions would not be paid for. + // BIP125 Rule #3: The replacement fees must be greater than or equal to fees of the + // transactions it replaces, otherwise the bandwidth used by those conflicting transactions + // would not be paid for. if (replacement_fees < original_fees) { return strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", txid.ToString(), FormatMoney(replacement_fees), FormatMoney(original_fees)); } - // Finally in addition to paying more fees than the conflicts the - // new transaction must pay for its own bandwidth. + // BIP125 Rule #4: The new transaction must pay for its own bandwidth. Otherwise, we have a DoS + // vector where attackers can cause a transaction to be replaced (and relayed) repeatedly by + // increasing the fee by tiny amounts. CAmount additional_fees = replacement_fees - original_fees; - if (additional_fees < ::incrementalRelayFee.GetFee(replacement_vsize)) { + if (additional_fees < relay_fee.GetFee(replacement_vsize)) { return strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", txid.ToString(), FormatMoney(additional_fees), - FormatMoney(::incrementalRelayFee.GetFee(replacement_vsize))); + FormatMoney(relay_fee.GetFee(replacement_vsize))); } return std::nullopt; } diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 56468a09b2..be8c2e5b8b 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -5,7 +5,12 @@ #ifndef BITCOIN_POLICY_RBF_H #define BITCOIN_POLICY_RBF_H +#include <primitives/transaction.h> #include <txmempool.h> +#include <uint256.h> + +#include <optional> +#include <string> /** Maximum number of transactions that can be replaced by BIP125 RBF (Rule #5). This includes all * mempool conflicts and their descendants. */ @@ -48,14 +53,14 @@ RBFTransactionState IsRBFOptInEmptyMempool(const CTransaction& tx); std::optional<std::string> GetEntriesForConflicts(const CTransaction& tx, CTxMemPool& pool, const CTxMemPool::setEntries& iters_conflicting, CTxMemPool::setEntries& all_conflicts) - EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + EXCLUSIVE_LOCKS_REQUIRED(pool.cs); /** BIP125 Rule #2: "The replacement transaction may only include an unconfirmed input if that input * was included in one of the original transactions." * @returns error message if Rule #2 is broken, otherwise std::nullopt. */ std::optional<std::string> HasNoNewUnconfirmed(const CTransaction& tx, const CTxMemPool& pool, const CTxMemPool::setEntries& iters_conflicting) - EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + EXCLUSIVE_LOCKS_REQUIRED(pool.cs); /** Check the intersection between two sets of transactions (a set of mempool entries and a set of * txids) to make sure they are disjoint. @@ -84,12 +89,14 @@ std::optional<std::string> PaysMoreThanConflicts(const CTxMemPool::setEntries& i * @param[in] original_fees Total modified fees of original transaction(s). * @param[in] replacement_fees Total modified fees of replacement transaction(s). * @param[in] replacement_vsize Total virtual size of replacement transaction(s). + * @param[in] relay_fee The node's minimum feerate for transaction relay. * @param[in] txid Transaction ID, included in the error message if violation occurs. * @returns error string if fees are insufficient, otherwise std::nullopt. */ std::optional<std::string> PaysForRBF(CAmount original_fees, CAmount replacement_fees, size_t replacement_vsize, + CFeeRate relay_fee, const uint256& txid); #endif // BITCOIN_POLICY_RBF_H diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 245206b906..a871912225 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -5,6 +5,7 @@ #include <primitives/transaction.h> +#include <consensus/amount.h> #include <hash.h> #include <tinyformat.h> #include <util/strencodings.h> diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 6bf36ee854..947c1c60bb 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -7,7 +7,7 @@ #define BITCOIN_PRIMITIVES_TRANSACTION_H #include <stdint.h> -#include <amount.h> +#include <consensus/amount.h> #include <script/script.h> #include <serialize.h> #include <uint256.h> @@ -391,8 +391,11 @@ class GenTxid { bool m_is_wtxid; uint256 m_hash; -public: GenTxid(bool is_wtxid, const uint256& hash) : m_is_wtxid(is_wtxid), m_hash(hash) {} + +public: + static GenTxid Txid(const uint256& hash) { return GenTxid{false, hash}; } + static GenTxid Wtxid(const uint256& hash) { return GenTxid{true, hash}; } bool IsWtxid() const { return m_is_wtxid; } const uint256& GetHash() const { return m_hash; } friend bool operator==(const GenTxid& a, const GenTxid& b) { return a.m_is_wtxid == b.m_is_wtxid && a.m_hash == b.m_hash; } diff --git a/src/protocol.cpp b/src/protocol.cpp index 2e70b41e4c..7506c81815 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -223,5 +223,5 @@ std::vector<std::string> serviceFlagsToStr(uint64_t flags) GenTxid ToGenTxid(const CInv& inv) { assert(inv.IsGenTxMsg()); - return {inv.IsMsgWtx(), inv.hash}; + return inv.IsMsgWtx() ? GenTxid::Wtxid(inv.hash) : GenTxid::Txid(inv.hash); } diff --git a/src/pubkey.cpp b/src/pubkey.cpp index d14a20b870..956ff2b34a 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -337,8 +337,7 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); - code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; - code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; + WriteBE32(code+5, nChild); memcpy(code+9, chaincode.begin(), 32); assert(pubkey.size() == CPubKey::COMPRESSED_SIZE); memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_SIZE); @@ -347,7 +346,7 @@ void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); - nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; + nChild = ReadBE32(code+5); memcpy(chaincode.begin(), code+9, 32); pubkey.Set(code+41, code+BIP32_EXTKEY_SIZE); if ((nDepth == 0 && (nChild != 0 || ReadLE32(vchFingerprint) != 0)) || !pubkey.IsFullyValid()) pubkey = CPubKey(); diff --git a/src/pubkey.h b/src/pubkey.h index 861a2cf500..f174ad8d85 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -230,8 +230,8 @@ public: XOnlyPubKey& operator=(const XOnlyPubKey&) = default; /** Determine if this pubkey is fully valid. This is true for approximately 50% of all - * possible 32-byte arrays. If false, VerifySchnorr and CreatePayToContract will always - * fail. */ + * possible 32-byte arrays. If false, VerifySchnorr, CheckTapTweak and CreateTapTweak + * will always fail. */ bool IsFullyValid() const; /** Test whether this is the 0 key (the result of default construction). This implies diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index e78594390b..a617bb4451 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -182,14 +182,14 @@ void AddressBookPage::onEditAction() if(indexes.isEmpty()) return; - EditAddressDialog dlg( + auto dlg = new EditAddressDialog( tab == SendingTab ? EditAddressDialog::EditSendingAddress : EditAddressDialog::EditReceivingAddress, this); - dlg.setModel(model); + dlg->setModel(model); QModelIndex origIndex = proxyModel->mapToSource(indexes.at(0)); - dlg.loadRow(origIndex.row()); - dlg.exec(); + dlg->loadRow(origIndex.row()); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void AddressBookPage::on_newAddress_clicked() diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index e30892aeb7..922aac531f 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -13,7 +13,6 @@ #include <interfaces/handler.h> #include <interfaces/init.h> #include <interfaces/node.h> -#include <node/context.h> #include <node/ui_interface.h> #include <noui.h> #include <qt/bitcoingui.h> @@ -46,7 +45,6 @@ #include <QApplication> #include <QDebug> -#include <QFontDatabase> #include <QLatin1String> #include <QLibraryInfo> #include <QLocale> @@ -55,6 +53,7 @@ #include <QThread> #include <QTimer> #include <QTranslator> +#include <QWindow> #if defined(QT_STATICPLUGIN) #include <QtPlugin> @@ -154,10 +153,11 @@ static bool InitSettings() std::vector<std::string> errors; if (!gArgs.ReadSettingsFile(&errors)) { - bilingual_str error = _("Settings file could not be read"); - InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors)))); + std::string error = QT_TRANSLATE_NOOP("bitcoin-core", "Settings file could not be read"); + std::string error_translated = QCoreApplication::translate("bitcoin-core", error.c_str()).toStdString(); + InitError(Untranslated(strprintf("%s:\n%s\n", error, MakeUnorderedList(errors)))); - QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Reset | QMessageBox::Abort); + QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error_translated)), QMessageBox::Reset | QMessageBox::Abort); /*: Explanatory text shown on startup when the settings file cannot be read. Prompts user to make a choice between resetting or aborting. */ messagebox.setInformativeText(QObject::tr("Do you want to reset settings to default values, or to abort without making changes?")); @@ -176,10 +176,11 @@ static bool InitSettings() errors.clear(); if (!gArgs.WriteSettingsFile(&errors)) { - bilingual_str error = _("Settings file could not be written"); - InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors)))); + std::string error = QT_TRANSLATE_NOOP("bitcoin-core", "Settings file could not be written"); + std::string error_translated = QCoreApplication::translate("bitcoin-core", error.c_str()).toStdString(); + InitError(Untranslated(strprintf("%s:\n%s\n", error, MakeUnorderedList(errors)))); - QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Ok); + QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error_translated)), QMessageBox::Ok); /*: Explanatory text shown on startup when the settings file could not be written. Prompts user to check that we have the ability to write to the file. Explains that the user has the option of running without a settings file.*/ @@ -259,6 +260,7 @@ void BitcoinApplication::createOptionsModel(bool resetSettings) void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) { window = new BitcoinGUI(node(), platformStyle, networkStyle, nullptr); + connect(window, &BitcoinGUI::quitRequested, this, &BitcoinApplication::requestShutdown); pollShutdownTimer = new QTimer(window); connect(pollShutdownTimer, &QTimer::timeout, window, &BitcoinGUI::detectShutdown); @@ -295,7 +297,7 @@ void BitcoinApplication::startThread() /* communication to and from thread */ connect(&m_executor.value(), &InitExecutor::initializeResult, this, &BitcoinApplication::initializeResult); - connect(&m_executor.value(), &InitExecutor::shutdownResult, this, &BitcoinApplication::shutdownResult); + connect(&m_executor.value(), &InitExecutor::shutdownResult, this, &QCoreApplication::quit); connect(&m_executor.value(), &InitExecutor::runawayException, this, &BitcoinApplication::handleRunawayException); connect(this, &BitcoinApplication::requestedInitialize, &m_executor.value(), &InitExecutor::initialize); connect(this, &BitcoinApplication::requestedShutdown, &m_executor.value(), &InitExecutor::shutdown); @@ -325,13 +327,17 @@ void BitcoinApplication::requestInitialize() void BitcoinApplication::requestShutdown() { + for (const auto w : QGuiApplication::topLevelWindows()) { + w->hide(); + } + // Show a simple window indicating shutdown status // Do this first as some of the steps may take some time below, // for example the RPC console may still be executing a command. shutdownWindow.reset(ShutdownWindow::showShutdownWindow(window)); qDebug() << __func__ << ": Requesting shutdown"; - window->hide(); + // Must disconnect node signals otherwise current thread can deadlock since // no event loop is running. window->unsubscribeFromCoreSignals(); @@ -408,15 +414,10 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead pollShutdownTimer->start(200); } else { Q_EMIT splashFinished(); // Make sure splash screen doesn't stick around during shutdown - quit(); // Exit first main loop invocation + requestShutdown(); } } -void BitcoinApplication::shutdownResult() -{ - quit(); // Exit second main loop invocation after shutdown finished -} - void BitcoinApplication::handleRunawayException(const QString &message) { QMessageBox::critical( @@ -461,9 +462,7 @@ int GuiMain(int argc, char* argv[]) std::tie(argc, argv) = winArgs.get(); #endif - NodeContext node_context; - int unused_exit_status; - std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status); + std::unique_ptr<interfaces::Init> init = interfaces::MakeGuiInit(argc, argv); SetupEnvironment(); util::ThreadSetInternalName("main"); @@ -490,7 +489,7 @@ int GuiMain(int argc, char* argv[]) #endif BitcoinApplication app; - QFontDatabase::addApplicationFont(":/fonts/monospace"); + GUIUtil::LoadFont(QStringLiteral(":/fonts/monospace")); /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: @@ -639,8 +638,6 @@ int GuiMain(int argc, char* argv[]) WinShutdownMonitor::registerShutdownBlockReason(QObject::tr("%1 didn't yet exit safely…").arg(PACKAGE_NAME), (HWND)app.getMainWinId()); #endif app.exec(); - app.requestShutdown(); - app.exec(); rv = app.getReturnValue(); } else { // A dialog with detailed error will have been shown by InitError() diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 602b76052c..5678ca90d2 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -61,8 +61,6 @@ public: /// Request core initialization void requestInitialize(); - /// Request core shutdown - void requestShutdown(); /// Get process return value int getReturnValue() const { return returnValue; } @@ -77,7 +75,8 @@ public: public Q_SLOTS: void initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info); - void shutdownResult(); + /// Request core shutdown + void requestShutdown(); /// Handle runaway exceptions. Shows a message box with the problem and quits the program. void handleRunawayException(const QString &message); diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index c60d9a2c90..4855ada513 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINAMOUNTFIELD_H #define BITCOIN_QT_BITCOINAMOUNTFIELD_H -#include <amount.h> +#include <consensus/amount.h> #include <QWidget> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 862bdd3bfe..b68ce39b53 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -107,7 +107,6 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty walletFrame = new WalletFrame(_platformStyle, this); connect(walletFrame, &WalletFrame::createWalletButtonClicked, [this] { auto activity = new CreateWalletActivity(getWalletController(), this); - connect(activity, &CreateWalletActivity::finished, activity, &QObject::deleteLater); activity->create(); }); connect(walletFrame, &WalletFrame::message, [this](const QString& title, const QString& message, unsigned int style) { @@ -171,7 +170,9 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty frameBlocksLayout->addWidget(unitDisplayControl); frameBlocksLayout->addStretch(); frameBlocksLayout->addWidget(labelWalletEncryptionIcon); + labelWalletEncryptionIcon->hide(); frameBlocksLayout->addWidget(labelWalletHDStatusIcon); + labelWalletHDStatusIcon->hide(); } frameBlocksLayout->addWidget(labelProxyIcon); frameBlocksLayout->addStretch(); @@ -371,7 +372,7 @@ void BitcoinGUI::createActions() m_mask_values_action->setStatusTip(tr("Mask the values in the Overview tab")); m_mask_values_action->setCheckable(true); - connect(quitAction, &QAction::triggered, qApp, QApplication::quit); + connect(quitAction, &QAction::triggered, this, &BitcoinGUI::quitRequested); connect(aboutAction, &QAction::triggered, this, &BitcoinGUI::aboutClicked); connect(aboutQtAction, &QAction::triggered, qApp, QApplication::aboutQt); connect(optionsAction, &QAction::triggered, this, &BitcoinGUI::optionsClicked); @@ -416,7 +417,6 @@ void BitcoinGUI::createActions() connect(action, &QAction::triggered, [this, path] { auto activity = new OpenWalletActivity(m_wallet_controller, this); connect(activity, &OpenWalletActivity::opened, this, &BitcoinGUI::setCurrentWallet); - connect(activity, &OpenWalletActivity::finished, activity, &QObject::deleteLater); activity->open(path); }); } @@ -431,7 +431,6 @@ void BitcoinGUI::createActions() connect(m_create_wallet_action, &QAction::triggered, [this] { auto activity = new CreateWalletActivity(m_wallet_controller, this); connect(activity, &CreateWalletActivity::created, this, &BitcoinGUI::setCurrentWallet); - connect(activity, &CreateWalletActivity::finished, activity, &QObject::deleteLater); activity->create(); }); connect(m_close_all_wallets_action, &QAction::triggered, [this] { @@ -662,9 +661,8 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) GUIUtil::ExceptionSafeConnect(wallet_controller, &WalletController::walletAdded, this, &BitcoinGUI::addWallet); connect(wallet_controller, &WalletController::walletRemoved, this, &BitcoinGUI::removeWallet); - for (WalletModel* wallet_model : m_wallet_controller->getOpenWallets()) { - addWallet(wallet_model); - } + auto activity = new LoadWalletsActivity(m_wallet_controller, this); + activity->load(); } WalletController* BitcoinGUI::getWalletController() @@ -848,8 +846,8 @@ void BitcoinGUI::aboutClicked() if(!clientModel) return; - HelpMessageDialog dlg(this, true); - dlg.exec(); + auto dlg = new HelpMessageDialog(this, /* about */ true); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void BitcoinGUI::showDebugWindow() @@ -990,10 +988,11 @@ void BitcoinGUI::openOptionsDialogWithTab(OptionsDialog::Tab tab) if (!clientModel || !clientModel->getOptionsModel()) return; - OptionsDialog dlg(this, enableWallet); - dlg.setCurrentTab(tab); - dlg.setModel(clientModel->getOptionsModel()); - dlg.exec(); + auto dlg = new OptionsDialog(this, enableWallet); + connect(dlg, &OptionsDialog::quitOnReset, this, &BitcoinGUI::quitRequested); + dlg->setCurrentTab(tab); + dlg->setModel(clientModel->getOptionsModel()); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header, SynchronizationState sync_state) @@ -1216,7 +1215,7 @@ void BitcoinGUI::closeEvent(QCloseEvent *event) // close rpcConsole in case it was open to make some space for the shutdown window rpcConsole->close(); - QApplication::quit(); + Q_EMIT quitRequested(); } else { @@ -1305,8 +1304,6 @@ void BitcoinGUI::setHDStatus(bool privkeyDisabled, int hdEnabled) labelWalletHDStatusIcon->setThemedPixmap(privkeyDisabled ? QStringLiteral(":/icons/eye") : hdEnabled ? QStringLiteral(":/icons/hd_enabled") : QStringLiteral(":/icons/hd_disabled"), STATUSBAR_ICONSIZE, STATUSBAR_ICONSIZE); labelWalletHDStatusIcon->setToolTip(privkeyDisabled ? tr("Private key <b>disabled</b>") : hdEnabled ? tr("HD key generation is <b>enabled</b>") : tr("HD key generation is <b>disabled</b>")); labelWalletHDStatusIcon->show(); - // eventually disable the QLabel to set its opacity to 50% - labelWalletHDStatusIcon->setEnabled(hdEnabled); } void BitcoinGUI::setEncryptionStatus(int status) @@ -1410,7 +1407,7 @@ void BitcoinGUI::detectShutdown() { if(rpcConsole) rpcConsole->hide(); - qApp->quit(); + Q_EMIT quitRequested(); } } diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index c83cd446a0..fa7ae4b87d 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -12,7 +12,7 @@ #include <qt/guiutil.h> #include <qt/optionsdialog.h> -#include <amount.h> +#include <consensus/amount.h> #include <QLabel> #include <QMainWindow> @@ -214,6 +214,7 @@ private: void openOptionsDialogWithTab(OptionsDialog::Tab tab); Q_SIGNALS: + void quitRequested(); /** Signal raised when a URI was entered or dragged to the GUI */ void receivedURI(const QString &uri); /** Signal raised when RPC console shown */ diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index 9660ba99f7..66d5eea7ac 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -4,6 +4,8 @@ #include <qt/bitcoinunits.h> +#include <consensus/amount.h> + #include <QStringList> #include <cassert> diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index e22ba0a938..e78a347bb1 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINUNITS_H #define BITCOIN_QT_BITCOINUNITS_H -#include <amount.h> +#include <consensus/amount.h> #include <QAbstractListModel> #include <QString> diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index d2a9365890..e93fedad28 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -55,7 +55,7 @@ CoinControlDialog::CoinControlDialog(CCoinControl& coin_control, WalletModel* _m contextMenu->addAction(tr("&Copy address"), this, &CoinControlDialog::copyAddress); contextMenu->addAction(tr("Copy &label"), this, &CoinControlDialog::copyLabel); contextMenu->addAction(tr("Copy &amount"), this, &CoinControlDialog::copyAmount); - copyTransactionHashAction = contextMenu->addAction(tr("Copy transaction &ID"), this, &CoinControlDialog::copyTransactionHash); + m_copy_transaction_outpoint_action = contextMenu->addAction(tr("Copy transaction &ID and output index"), this, &CoinControlDialog::copyTransactionOutpoint); contextMenu->addSeparator(); lockAction = contextMenu->addAction(tr("L&ock unspent"), this, &CoinControlDialog::lockCoin); unlockAction = contextMenu->addAction(tr("&Unlock unspent"), this, &CoinControlDialog::unlockCoin); @@ -180,7 +180,7 @@ void CoinControlDialog::showMenu(const QPoint &point) // disable some items (like Copy Transaction ID, lock, unlock) for tree roots in context menu if (item->data(COLUMN_ADDRESS, TxHashRole).toString().length() == 64) // transaction hash is 64 characters (this means it is a child node, so it is not a parent node in tree mode) { - copyTransactionHashAction->setEnabled(true); + m_copy_transaction_outpoint_action->setEnabled(true); if (model->wallet().isLockedCoin(COutPoint(uint256S(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt()))) { lockAction->setEnabled(false); @@ -194,7 +194,7 @@ void CoinControlDialog::showMenu(const QPoint &point) } else // this means click on parent node in tree mode -> disable all { - copyTransactionHashAction->setEnabled(false); + m_copy_transaction_outpoint_action->setEnabled(false); lockAction->setEnabled(false); unlockAction->setEnabled(false); } @@ -228,10 +228,14 @@ void CoinControlDialog::copyAddress() GUIUtil::setClipboard(contextMenuItem->text(COLUMN_ADDRESS)); } -// context menu action: copy transaction id -void CoinControlDialog::copyTransactionHash() +// context menu action: copy transaction id and vout index +void CoinControlDialog::copyTransactionOutpoint() { - GUIUtil::setClipboard(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString()); + const QString address = contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString(); + const QString vout = contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toString(); + const QString outpoint = QString("%1:%2").arg(address).arg(vout); + + GUIUtil::setClipboard(outpoint); } // context menu action: lock coin @@ -241,7 +245,7 @@ void CoinControlDialog::lockCoin() contextMenuItem->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); COutPoint outpt(uint256S(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toUInt()); - model->wallet().lockCoin(outpt); + model->wallet().lockCoin(outpt, /* write_to_db = */ true); contextMenuItem->setDisabled(true); contextMenuItem->setIcon(COLUMN_CHECKBOX, platformStyle->SingleColorIcon(":/icons/lock_closed")); updateLabelLocked(); diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index 3a03341c9e..ec2619d115 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_COINCONTROLDIALOG_H #define BITCOIN_QT_COINCONTROLDIALOG_H -#include <amount.h> +#include <consensus/amount.h> #include <QAbstractButton> #include <QAction> @@ -63,7 +63,7 @@ private: QMenu *contextMenu; QTreeWidgetItem *contextMenuItem; - QAction *copyTransactionHashAction; + QAction* m_copy_transaction_outpoint_action; QAction *lockAction; QAction *unlockAction; @@ -95,7 +95,7 @@ private Q_SLOTS: void copyAmount(); void copyLabel(); void copyAddress(); - void copyTransactionHash(); + void copyTransactionOutpoint(); void lockCoin(); void unlockCoin(); void clipboardQuantity(); diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index f9a61c3e60..eba70331f8 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -6,7 +6,7 @@ #include <config/bitcoin-config.h> #endif -#include <external_signer.h> +#include <interfaces/node.h> #include <qt/createwalletdialog.h> #include <qt/forms/ui_createwalletdialog.h> @@ -113,7 +113,7 @@ CreateWalletDialog::~CreateWalletDialog() delete ui; } -void CreateWalletDialog::setSigners(const std::vector<ExternalSigner>& signers) +void CreateWalletDialog::setSigners(const std::vector<std::unique_ptr<interfaces::ExternalSigner>>& signers) { m_has_signers = !signers.empty(); if (m_has_signers) { @@ -126,7 +126,7 @@ void CreateWalletDialog::setSigners(const std::vector<ExternalSigner>& signers) ui->blank_wallet_checkbox->setChecked(false); ui->disable_privkeys_checkbox->setEnabled(false); ui->disable_privkeys_checkbox->setChecked(true); - const std::string label = signers[0].m_name; + const std::string label = signers[0]->getName(); ui->wallet_name_line_edit->setText(QString::fromStdString(label)); ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(true); } else { diff --git a/src/qt/createwalletdialog.h b/src/qt/createwalletdialog.h index fc13cc44eb..63a5e012d8 100644 --- a/src/qt/createwalletdialog.h +++ b/src/qt/createwalletdialog.h @@ -7,7 +7,12 @@ #include <QDialog> +#include <memory> + +namespace interfaces { class ExternalSigner; +} // namespace interfaces + class WalletModel; namespace Ui { @@ -24,7 +29,7 @@ public: explicit CreateWalletDialog(QWidget* parent); virtual ~CreateWalletDialog(); - void setSigners(const std::vector<ExternalSigner>& signers); + void setSigners(const std::vector<std::unique_ptr<interfaces::ExternalSigner>>& signers); QString walletName() const; bool isEncryptWalletChecked() const; diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index b11fb026b0..56adbe17a5 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -107,6 +107,9 @@ <property name="text"> <string>Descriptor Wallet</string> </property> + <property name="checked"> + <bool>true</bool> + </property> </widget> </item> <item> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 59d220636d..1c22124616 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -33,7 +33,7 @@ <string>Automatically start %1 after logging in to the system.</string> </property> <property name="text"> - <string>&Start %1 on system login</string> + <string>Start %1 on system &login</string> </property> </widget> </item> @@ -179,7 +179,7 @@ <property name="sizeHint" stdset="0"> <size> <width>40</width> - <height>20</height> + <height>40</height> </size> </property> </spacer> @@ -187,6 +187,16 @@ </layout> </item> <item> + <widget class="QCheckBox" name="enableServer"> + <property name="toolTip"> + <string extracomment="Tooltip text for Options window setting that enables the RPC server.">This allows you or a third party tool to communicate with the node through command-line and JSON-RPC commands.</string> + </property> + <property name="text"> + <string extracomment="An Options window setting to enable the RPC server.">Enable RPC &server</string> + </property> + </widget> + </item> + <item> <spacer name="verticalSpacer_Main"> <property name="orientation"> <enum>Qt::Vertical</enum> @@ -729,10 +739,10 @@ <item> <widget class="QLabel" name="thirdPartyTxUrlsLabel"> <property name="toolTip"> - <string>Third party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> + <string>Third-party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> </property> <property name="text"> - <string>&Third party transaction URLs</string> + <string>&Third-party transaction URLs</string> </property> <property name="buddy"> <cstring>thirdPartyTxUrls</cstring> @@ -742,7 +752,7 @@ <item> <widget class="QLineEdit" name="thirdPartyTxUrls"> <property name="toolTip"> - <string>Third party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> + <string>Third-party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> </property> <property name="placeholderText"> <string notr="true">https://example.com/tx/%s</string> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index e98e50ba14..4262866f32 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -36,6 +36,7 @@ #include <QClipboard> #include <QDateTime> #include <QDesktopServices> +#include <QDialog> #include <QDoubleValidator> #include <QFileDialog> #include <QFont> @@ -271,6 +272,12 @@ bool hasEntryData(const QAbstractItemView *view, int column, int role) return !selection.at(0).data(role).toString().isEmpty(); } +void LoadFont(const QString& file_name) +{ + const int id = QFontDatabase::addApplicationFont(file_name); + assert(id != -1); +} + QString getDefaultDataDirectory() { return boostPathToQString(GetDefaultDataDir()); @@ -646,12 +653,12 @@ void setClipboard(const QString& str) fs::path qstringToBoostPath(const QString &path) { - return fs::path(path.toStdString()); + return fs::u8path(path.toStdString()); } QString boostPathToQString(const fs::path &path) { - return QString::fromStdString(path.string()); + return QString::fromStdString(path.u8string()); } QString NetworkToQString(Network net) @@ -673,14 +680,26 @@ QString ConnectionTypeToQString(ConnectionType conn_type, bool prepend_direction { QString prefix; if (prepend_direction) { - prefix = (conn_type == ConnectionType::INBOUND) ? QObject::tr("Inbound") : QObject::tr("Outbound") + " "; + prefix = (conn_type == ConnectionType::INBOUND) ? + /*: An inbound connection from a peer. An inbound connection + is a connection initiated by a peer. */ + QObject::tr("Inbound") : + /*: An outbound connection to a peer. An outbound connection + is a connection initiated by us. */ + QObject::tr("Outbound") + " "; } switch (conn_type) { case ConnectionType::INBOUND: return prefix; + //: Peer connection type that relays all network information. case ConnectionType::OUTBOUND_FULL_RELAY: return prefix + QObject::tr("Full Relay"); + /*: Peer connection type that relays network information about + blocks and not transactions or addresses. */ case ConnectionType::BLOCK_RELAY: return prefix + QObject::tr("Block Relay"); + //: Peer connection type established manually through one of several methods. case ConnectionType::MANUAL: return prefix + QObject::tr("Manual"); + //: Short-lived peer connection type that tests the aliveness of known addresses. case ConnectionType::FEELER: return prefix + QObject::tr("Feeler"); + //: Short-lived peer connection type that solicits known addresses from a peer. case ConnectionType::ADDR_FETCH: return prefix + QObject::tr("Address Fetch"); } // no default case, so the compiler can warn about missing cases assert(false); @@ -958,4 +977,11 @@ void PrintSlotException( PrintExceptionContinue(exception, description.c_str()); } +void ShowModalDialogAndDeleteOnClose(QDialog* dialog) +{ + dialog->setAttribute(Qt::WA_DeleteOnClose); + dialog->setWindowModality(Qt::ApplicationModal); + dialog->show(); +} + } // namespace GUIUtil diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 06a3b63668..211f3f506d 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_GUIUTIL_H #define BITCOIN_QT_GUIUTIL_H -#include <amount.h> +#include <consensus/amount.h> #include <fs.h> #include <net.h> #include <netaddress.h> @@ -41,6 +41,7 @@ class QAbstractButton; class QAbstractItemView; class QAction; class QDateTime; +class QDialog; class QFont; class QKeySequence; class QLineEdit; @@ -113,6 +114,11 @@ namespace GUIUtil void setClipboard(const QString& str); /** + * Loads the font from the file specified by file_name, aborts if it fails. + */ + void LoadFont(const QString& file_name); + + /** * Determine default data directory for operating system. */ QString getDefaultDataDirectory(); @@ -417,6 +423,11 @@ namespace GUIUtil type); } + /** + * Shows a QDialog instance asynchronously, and deletes it on close. + */ + void ShowModalDialogAndDeleteOnClose(QDialog* dialog); + } // namespace GUIUtil #endif // BITCOIN_QT_GUIUTIL_H diff --git a/src/qt/initexecutor.cpp b/src/qt/initexecutor.cpp index 7060f74dab..24ae7ba73d 100644 --- a/src/qt/initexecutor.cpp +++ b/src/qt/initexecutor.cpp @@ -5,6 +5,7 @@ #include <qt/initexecutor.h> #include <interfaces/node.h> +#include <qt/guiutil.h> #include <util/system.h> #include <util/threadnames.h> @@ -18,7 +19,7 @@ InitExecutor::InitExecutor(interfaces::Node& node) : QObject(), m_node(node) { - this->moveToThread(&m_thread); + m_context.moveToThread(&m_thread); m_thread.start(); } @@ -38,29 +39,33 @@ void InitExecutor::handleRunawayException(const std::exception* e) void InitExecutor::initialize() { - try { - util::ThreadRename("qt-init"); - qDebug() << __func__ << ": Running initialization in thread"; - interfaces::BlockAndHeaderTipInfo tip_info; - bool rv = m_node.appInitMain(&tip_info); - Q_EMIT initializeResult(rv, tip_info); - } catch (const std::exception& e) { - handleRunawayException(&e); - } catch (...) { - handleRunawayException(nullptr); - } + GUIUtil::ObjectInvoke(&m_context, [this] { + try { + util::ThreadRename("qt-init"); + qDebug() << "Running initialization in thread"; + interfaces::BlockAndHeaderTipInfo tip_info; + bool rv = m_node.appInitMain(&tip_info); + Q_EMIT initializeResult(rv, tip_info); + } catch (const std::exception& e) { + handleRunawayException(&e); + } catch (...) { + handleRunawayException(nullptr); + } + }); } void InitExecutor::shutdown() { - try { - qDebug() << __func__ << ": Running Shutdown in thread"; - m_node.appShutdown(); - qDebug() << __func__ << ": Shutdown finished"; - Q_EMIT shutdownResult(); - } catch (const std::exception& e) { - handleRunawayException(&e); - } catch (...) { - handleRunawayException(nullptr); - } + GUIUtil::ObjectInvoke(&m_context, [this] { + try { + qDebug() << "Running Shutdown in thread"; + m_node.appShutdown(); + qDebug() << "Shutdown finished"; + Q_EMIT shutdownResult(); + } catch (const std::exception& e) { + handleRunawayException(&e); + } catch (...) { + handleRunawayException(nullptr); + } + }); } diff --git a/src/qt/initexecutor.h b/src/qt/initexecutor.h index 319ce40465..410c44fa2d 100644 --- a/src/qt/initexecutor.h +++ b/src/qt/initexecutor.h @@ -40,6 +40,7 @@ private: void handleRunawayException(const std::exception* e); interfaces::Node& m_node; + QObject m_context; QThread m_thread; }; diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index a698a96857..2ca4b6a21e 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -113,7 +113,7 @@ namespace { //! Return pruning size that will be used if automatic pruning is enabled. int GetPruneTargetGB() { - int64_t prune_target_mib = gArgs.GetArg("-prune", 0); + int64_t prune_target_mib = gArgs.GetIntArg("-prune", 0); // >1 means automatic pruning is enabled by config, 1 means manual pruning, 0 means no pruning. return prune_target_mib > 1 ? PruneMiBtoGB(prune_target_mib) : DEFAULT_PRUNE_TARGET_GB; } @@ -142,7 +142,7 @@ Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_si const int min_prune_target_GB = std::ceil(MIN_DISK_SPACE_FOR_BLOCK_FILES / 1e9); ui->pruneGB->setRange(min_prune_target_GB, std::numeric_limits<int>::max()); - if (gArgs.GetArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB + if (gArgs.GetIntArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB ui->prune->setChecked(true); ui->prune->setEnabled(false); } @@ -263,7 +263,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) * (to be consistent with bitcoind behavior) */ if(dataDir != GUIUtil::getDefaultDataDirectory()) { - gArgs.SoftSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); // use OS locale for path setting + gArgs.SoftSetArg("-datadir", fs::PathToString(GUIUtil::qstringToBoostPath(dataDir))); // use OS locale for path setting } return true; } diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 92644ef24b..0cc2d61df6 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -210,6 +210,7 @@ void OptionsDialog::setModel(OptionsModel *_model) connect(ui->spendZeroConfChange, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); /* Network */ connect(ui->allowIncoming, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); + connect(ui->enableServer, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); connect(ui->connectSocks, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); connect(ui->connectSocksTor, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); /* Display */ @@ -246,6 +247,7 @@ void OptionsDialog::setMapper() mapper->addMapping(ui->mapPortUpnp, OptionsModel::MapPortUPnP); mapper->addMapping(ui->mapPortNatpmp, OptionsModel::MapPortNatpmp); mapper->addMapping(ui->allowIncoming, OptionsModel::Listen); + mapper->addMapping(ui->enableServer, OptionsModel::Server); mapper->addMapping(ui->connectSocks, OptionsModel::ProxyUse); mapper->addMapping(ui->proxyIp, OptionsModel::ProxyIP); @@ -290,7 +292,8 @@ void OptionsDialog::on_resetButton_clicked() /* reset all options and close GUI */ model->Reset(); - QApplication::quit(); + close(); + Q_EMIT quitOnReset(); } } diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index ba35ff3b67..f14aec3449 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -68,6 +68,7 @@ private Q_SLOTS: Q_SIGNALS: void proxyIpChecks(QValidatedLineEdit *pUiProxyIp, uint16_t nProxyPort); + void quitOnReset(); private: Ui::OptionsDialog *ui; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index d87fc1f84a..9e2f38f7ec 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -149,6 +149,13 @@ void OptionsModel::Init(bool resetSettings) if (!gArgs.SoftSetBoolArg("-listen", settings.value("fListen").toBool())) addOverriddenOption("-listen"); + if (!settings.contains("server")) { + settings.setValue("server", false); + } + if (!gArgs.SoftSetBoolArg("-server", settings.value("server").toBool())) { + addOverriddenOption("-server"); + } + if (!settings.contains("fUseProxy")) settings.setValue("fUseProxy", false); if (!settings.contains("addrProxy")) @@ -363,6 +370,8 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const return settings.value("nThreadsScriptVerif"); case Listen: return settings.value("fListen"); + case Server: + return settings.value("server"); default: return QVariant(); } @@ -528,6 +537,12 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in setRestartRequired(true); } break; + case Server: + if (settings.value("server") != value) { + settings.setValue("server", value); + setRestartRequired(true); + } + break; default: break; } diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 203ee27ad8..8f1513e48d 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -5,7 +5,6 @@ #ifndef BITCOIN_QT_OPTIONSMODEL_H #define BITCOIN_QT_OPTIONSMODEL_H -#include <amount.h> #include <cstdint> #include <qt/guiconstants.h> @@ -69,6 +68,7 @@ public: ExternalSignerPath, // QString SpendZeroConfChange, // bool Listen, // bool + Server, // bool OptionIDRowCount, }; diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 289fb9f7c8..34d56e5506 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -71,6 +71,9 @@ void PSBTOperationsDialog::signTransaction() { bool complete; size_t n_signed; + + WalletModel::UnlockContext ctx(m_wallet_model->requestUnlock()); + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, true /* sign */, true /* bip32derivs */, &n_signed, m_transaction_data, complete); if (err != TransactionError::OK) { @@ -81,7 +84,9 @@ void PSBTOperationsDialog::signTransaction() updateTransactionDisplay(); - if (!complete && n_signed < 1) { + if (!complete && !ctx.isValid()) { + showStatus(tr("Cannot sign inputs while wallet is locked."), StatusLevel::WARN); + } else if (!complete && n_signed < 1) { showStatus(tr("Could not sign any more inputs."), StatusLevel::WARN); } else if (!complete) { showStatus(tr("Signed %1 inputs, but more signatures are still required.").arg(n_signed), diff --git a/src/qt/res/animation/makespinner.sh b/src/qt/res/animation/makespinner.sh index 83142f5034..647ee1aed8 100755 --- a/src/qt/res/animation/makespinner.sh +++ b/src/qt/res/animation/makespinner.sh @@ -5,10 +5,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -FRAMEDIR=$(dirname $0) +FRAMEDIR=$(dirname "$0") for i in {0..35} do - frame=$(printf "%03d" $i) + frame=$(printf "%03d" "$i") angle=$((i * 10)) - convert $FRAMEDIR/../src/spinner.png -background "rgba(0,0,0,0.0)" -distort SRT $angle $FRAMEDIR/spinner-$frame.png + convert "${FRAMEDIR}/../src/spinner.png" -background "rgba(0,0,0,0.0)" -distort SRT $angle "${FRAMEDIR}/spinner-${frame}.png" done diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 4554f11a41..0c3332ab76 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -247,10 +247,11 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes UniValue subelement; if (lastResult.isArray()) { - for(char argch: curarg) - if (!IsDigit(argch)) - throw std::runtime_error("Invalid result query"); - subelement = lastResult[atoi(curarg.c_str())]; + const auto parsed{ToIntegral<size_t>(curarg)}; + if (!parsed) { + throw std::runtime_error("Invalid result query"); + } + subelement = lastResult[parsed.value()]; } else if (lastResult.isObject()) subelement = find_value(lastResult, curarg); @@ -495,14 +496,28 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty constexpr QChar nonbreaking_hyphen(8209); const std::vector<QString> CONNECTION_TYPE_DOC{ + //: Explanatory text for an inbound peer connection. tr("Inbound: initiated by peer"), + /*: Explanatory text for an outbound peer connection that + relays all network information. This is the default behavior for + outbound connections. */ tr("Outbound Full Relay: default"), + /*: Explanatory text for an outbound peer connection that relays + network information about blocks and not transactions or addresses. */ tr("Outbound Block Relay: does not relay transactions or addresses"), + /*: Explanatory text for an outbound peer connection that was + established manually through one of several methods. The numbered + arguments are stand-ins for the methods available to establish + manual connections. */ tr("Outbound Manual: added using RPC %1 or %2/%3 configuration options") .arg("addnode") .arg(QString(nonbreaking_hyphen) + "addnode") .arg(QString(nonbreaking_hyphen) + "connect"), + /*: Explanatory text for a short-lived outbound peer connection that + is used to test the aliveness of known addresses. */ tr("Outbound Feeler: short-lived, for testing addresses"), + /*: Explanatory text for a short-lived outbound peer connection that is used + to request addresses from a peer. */ tr("Outbound Address Fetch: short-lived, for soliciting addresses")}; const QString list{"<ul><li>" + Join(CONNECTION_TYPE_DOC, QString("</li><li>")) + "</li></ul>"}; ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg(list)); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index ff53d8160f..2718392940 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -399,9 +399,10 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) const QString confirmation = model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner() ? tr("Confirm transaction proposal") : tr("Confirm send coins"); const QString confirmButtonText = model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner() ? tr("Create Unsigned") : tr("Sign and send"); - SendConfirmationDialog confirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); - confirmationDialog.exec(); - QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); + auto confirmationDialog = new SendConfirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); + confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); + // TODO: Replace QDialog::exec() with safer QDialog::show(). + const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); if(retval != QMessageBox::Yes) { @@ -914,9 +915,9 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) // Coin Control: button inputs -> show actual coin control dialog void SendCoinsDialog::coinControlButtonClicked() { - CoinControlDialog dlg(*m_coin_control, model, platformStyle); - dlg.exec(); - coinControlUpdateLabels(); + auto dlg = new CoinControlDialog(*m_coin_control, model, platformStyle); + connect(dlg, &QDialog::finished, this, &SendCoinsDialog::coinControlUpdateLabels); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } // Coin Control: checkbox custom change address diff --git a/src/qt/sendcoinsrecipient.h b/src/qt/sendcoinsrecipient.h index 01135cdfef..c23afcab24 100644 --- a/src/qt/sendcoinsrecipient.h +++ b/src/qt/sendcoinsrecipient.h @@ -9,7 +9,7 @@ #include <config/bitcoin-config.h> #endif -#include <amount.h> +#include <consensus/amount.h> #include <serialize.h> #include <string> diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 736af27acc..85703b3350 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -184,8 +184,8 @@ static void InitMessage(SplashScreen *splash, const std::string &message) static void ShowProgress(SplashScreen *splash, const std::string &title, int nProgress, bool resume_possible) { InitMessage(splash, title + std::string("\n") + - (resume_possible ? _("(press q to shutdown and continue later)").translated - : _("press q to shutdown").translated) + + (resume_possible ? SplashScreen::tr("(press q to shutdown and continue later)").toStdString() + : SplashScreen::tr("press q to shutdown").toStdString()) + strprintf("\n%d", nProgress) + "%"); } diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 0de781661a..39cc1c1e1d 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -63,9 +63,13 @@ void TestAddAddressesToSendBook(interfaces::Node& node) auto wallet_client = interfaces::MakeWalletClient(*test.m_node.chain, *Assert(test.m_node.args)); test.m_node.wallet_client = wallet_client.get(); node.setContext(&test.m_node); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); - wallet->SetupLegacyScriptPubKeyMan(); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", gArgs, CreateMockWalletDatabase()); wallet->LoadWallet(); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + { + LOCK(wallet->cs_wallet); + wallet->SetupDescriptorScriptPubKeyMans(); + } auto build_address = [&wallet]() { CKey key; diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 884ed25637..b26cddf4ae 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -9,7 +9,6 @@ #include <interfaces/init.h> #include <interfaces/node.h> #include <qt/bitcoin.h> -#include <qt/initexecutor.h> #include <qt/test/apptests.h> #include <qt/test/rpcnestedtests.h> #include <qt/test/uritests.h> @@ -53,9 +52,7 @@ int main(int argc, char* argv[]) BasicTestingSetup dummy{CBaseChainParams::REGTEST}; } - NodeContext node_context; - int unused_exit_status; - std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status); + std::unique_ptr<interfaces::Init> init = interfaces::MakeGuiInit(argc, argv); gArgs.ForceSetArg("-listen", "0"); gArgs.ForceSetArg("-listenonion", "0"); gArgs.ForceSetArg("-discover", "0"); diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 62b135d3f1..93d6aa805f 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -141,13 +141,22 @@ void TestGUI(interfaces::Node& node) auto wallet_client = interfaces::MakeWalletClient(*test.m_node.chain, *Assert(test.m_node.args)); test.m_node.wallet_client = wallet_client.get(); node.setContext(&test.m_node); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", gArgs, CreateMockWalletDatabase()); wallet->LoadWallet(); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); { - auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); - wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); - spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + LOCK(wallet->cs_wallet); + wallet->SetupDescriptorScriptPubKeyMans(); + + // Add the coinbase key + FlatSigningProvider provider; + std::string error; + std::unique_ptr<Descriptor> desc = Parse("combo(" + EncodeSecret(test.coinbaseKey) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet->AddWalletDescriptor(w_desc, provider, "", false)) assert(false); + CTxDestination dest = GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type); + wallet->SetAddressBook(dest, "", "receive"); wallet->SetLastBlockProcessed(105, node.context()->chainman->ActiveChain().Tip()->GetBlockHash()); } { diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index 09bc9e75db..270b8ef78c 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TRANSACTIONFILTERPROXY_H #define BITCOIN_QT_TRANSACTIONFILTERPROXY_H -#include <amount.h> +#include <consensus/amount.h> #include <QDateTime> #include <QSortFilterProxyModel> diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index e10243a28a..fb88ca424f 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TRANSACTIONRECORD_H #define BITCOIN_QT_TRANSACTIONRECORD_H -#include <amount.h> +#include <consensus/amount.h> #include <uint256.h> #include <QList> diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 2f16e6edb4..653f3dda6d 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -222,17 +222,21 @@ void TransactionView::setModel(WalletModel *_model) { // Add third party transaction URLs to context menu QStringList listUrls = GUIUtil::SplitSkipEmptyParts(_model->getOptionsModel()->getThirdPartyTxUrls(), "|"); + bool actions_created = false; for (int i = 0; i < listUrls.size(); ++i) { QString url = listUrls[i].trimmed(); QString host = QUrl(url, QUrl::StrictMode).host(); if (!host.isEmpty()) { - QAction *thirdPartyTxUrlAction = new QAction(host, this); // use host as menu item label - if (i == 0) + if (!actions_created) { contextMenu->addSeparator(); - contextMenu->addAction(thirdPartyTxUrlAction); - connect(thirdPartyTxUrlAction, &QAction::triggered, [this, url] { openThirdPartyTxUrl(url); }); + actions_created = true; + } + /*: Transactions table context menu action to show the + selected transaction in a third-party block explorer. + %1 is a stand-in argument for the URL of the explorer. */ + contextMenu->addAction(tr("Show in %1").arg(host), [this, url] { openThirdPartyTxUrl(url); }); } } } @@ -500,22 +504,22 @@ void TransactionView::editLabel() // Determine type of address, launch appropriate editor dialog type QString type = modelIdx.data(AddressTableModel::TypeRole).toString(); - EditAddressDialog dlg( + auto dlg = new EditAddressDialog( type == AddressTableModel::Receive ? EditAddressDialog::EditReceivingAddress : EditAddressDialog::EditSendingAddress, this); - dlg.setModel(addressBook); - dlg.loadRow(idx); - dlg.exec(); + dlg->setModel(addressBook); + dlg->loadRow(idx); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } else { // Add sending address - EditAddressDialog dlg(EditAddressDialog::NewSendingAddress, + auto dlg = new EditAddressDialog(EditAddressDialog::NewSendingAddress, this); - dlg.setModel(addressBook); - dlg.setAddress(address); - dlg.exec(); + dlg->setModel(addressBook); + dlg->setAddress(address); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } } } diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 3cceb5ca5a..b9a9fcf3d1 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -41,10 +41,6 @@ WalletController::WalletController(ClientModel& client_model, const PlatformStyl getOrCreateWallet(std::move(wallet)); }); - for (std::unique_ptr<interfaces::Wallet>& wallet : m_node.walletClient().getWallets()) { - getOrCreateWallet(std::move(wallet)); - } - m_activity_worker->moveToThread(m_activity_thread); m_activity_thread->start(); QTimer::singleShot(0, m_activity_worker, []() { @@ -61,12 +57,6 @@ WalletController::~WalletController() delete m_activity_worker; } -std::vector<WalletModel*> WalletController::getOpenWallets() const -{ - QMutexLocker locker(&m_mutex); - return m_wallets; -} - std::map<std::string, bool> WalletController::listWalletDir() const { QMutexLocker locker(&m_mutex); @@ -191,33 +181,24 @@ WalletControllerActivity::WalletControllerActivity(WalletController* wallet_cont , m_wallet_controller(wallet_controller) , m_parent_widget(parent_widget) { + connect(this, &WalletControllerActivity::finished, this, &QObject::deleteLater); } -WalletControllerActivity::~WalletControllerActivity() +void WalletControllerActivity::showProgressDialog(const QString& title_text, const QString& label_text) { - delete m_progress_dialog; -} - -void WalletControllerActivity::showProgressDialog(const QString& label_text) -{ - assert(!m_progress_dialog); - m_progress_dialog = new QProgressDialog(m_parent_widget); - - m_progress_dialog->setLabelText(label_text); - m_progress_dialog->setRange(0, 0); - m_progress_dialog->setCancelButton(nullptr); - m_progress_dialog->setWindowModality(Qt::ApplicationModal); - GUIUtil::PolishProgressDialog(m_progress_dialog); + auto progress_dialog = new QProgressDialog(m_parent_widget); + progress_dialog->setAttribute(Qt::WA_DeleteOnClose); + connect(this, &WalletControllerActivity::finished, progress_dialog, &QWidget::close); + + progress_dialog->setWindowTitle(title_text); + progress_dialog->setLabelText(label_text); + progress_dialog->setRange(0, 0); + progress_dialog->setCancelButton(nullptr); + progress_dialog->setWindowModality(Qt::ApplicationModal); + GUIUtil::PolishProgressDialog(progress_dialog); // The setValue call forces QProgressDialog to start the internal duration estimation. // See details in https://bugreports.qt.io/browse/QTBUG-47042. - m_progress_dialog->setValue(0); -} - -void WalletControllerActivity::destroyProgressDialog() -{ - assert(m_progress_dialog); - delete m_progress_dialog; - m_progress_dialog = nullptr; + progress_dialog->setValue(0); } CreateWalletActivity::CreateWalletActivity(WalletController* wallet_controller, QWidget* parent_widget) @@ -251,7 +232,12 @@ void CreateWalletActivity::askPassphrase() void CreateWalletActivity::createWallet() { - showProgressDialog(tr("Creating Wallet <b>%1</b>…").arg(m_create_wallet_dialog->walletName().toHtmlEscaped())); + showProgressDialog( + //: Title of window indicating the progress of creation of a new wallet. + tr("Create Wallet"), + /*: Descriptive text of the create wallet progress window which indicates + to the user which wallet is currently being created. */ + tr("Creating Wallet <b>%1</b>…").arg(m_create_wallet_dialog->walletName().toHtmlEscaped())); std::string name = m_create_wallet_dialog->walletName().toStdString(); uint64_t flags = 0; @@ -279,8 +265,6 @@ void CreateWalletActivity::createWallet() void CreateWalletActivity::finish() { - destroyProgressDialog(); - if (!m_error_message.empty()) { QMessageBox::critical(m_parent_widget, tr("Create wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { @@ -296,9 +280,9 @@ void CreateWalletActivity::create() { m_create_wallet_dialog = new CreateWalletDialog(m_parent_widget); - std::vector<ExternalSigner> signers; + std::vector<std::unique_ptr<interfaces::ExternalSigner>> signers; try { - signers = node().externalSigners(); + signers = node().listExternalSigners(); } catch (const std::runtime_error& e) { QMessageBox::critical(nullptr, tr("Can't list signers"), e.what()); } @@ -329,8 +313,6 @@ OpenWalletActivity::OpenWalletActivity(WalletController* wallet_controller, QWid void OpenWalletActivity::finish() { - destroyProgressDialog(); - if (!m_error_message.empty()) { QMessageBox::critical(m_parent_widget, tr("Open wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { @@ -346,7 +328,12 @@ void OpenWalletActivity::open(const std::string& path) { QString name = path.empty() ? QString("["+tr("default wallet")+"]") : QString::fromStdString(path); - showProgressDialog(tr("Opening Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); + showProgressDialog( + //: Title of window indicating the progress of opening of a wallet. + tr("Open Wallet"), + /*: Descriptive text of the open wallet progress window which indicates + to the user which wallet is currently being opened. */ + tr("Opening Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); QTimer::singleShot(0, worker(), [this, path] { std::unique_ptr<interfaces::Wallet> wallet = node().walletClient().loadWallet(path, m_error_message, m_warning_message); @@ -356,3 +343,26 @@ void OpenWalletActivity::open(const std::string& path) QTimer::singleShot(0, this, &OpenWalletActivity::finish); }); } + +LoadWalletsActivity::LoadWalletsActivity(WalletController* wallet_controller, QWidget* parent_widget) + : WalletControllerActivity(wallet_controller, parent_widget) +{ +} + +void LoadWalletsActivity::load() +{ + showProgressDialog( + //: Title of progress window which is displayed when wallets are being loaded. + tr("Load Wallets"), + /*: Descriptive text of the load wallets progress window which indicates to + the user that wallets are currently being loaded.*/ + tr("Loading wallets…")); + + QTimer::singleShot(0, worker(), [this] { + for (auto& wallet : node().walletClient().getWallets()) { + m_wallet_controller->getOrCreateWallet(std::move(wallet)); + } + + QTimer::singleShot(0, this, [this] { Q_EMIT finished(); }); + }); +} diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index f7e366878d..bbd990228f 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -52,9 +52,6 @@ public: WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent); ~WalletController(); - //! Returns wallet models currently open. - std::vector<WalletModel*> getOpenWallets() const; - WalletModel* getOrCreateWallet(std::unique_ptr<interfaces::Wallet> wallet); //! Returns all wallet names in the wallet dir mapped to whether the wallet @@ -90,7 +87,7 @@ class WalletControllerActivity : public QObject public: WalletControllerActivity(WalletController* wallet_controller, QWidget* parent_widget); - virtual ~WalletControllerActivity(); + virtual ~WalletControllerActivity() = default; Q_SIGNALS: void finished(); @@ -99,12 +96,10 @@ protected: interfaces::Node& node() const { return m_wallet_controller->m_node; } QObject* worker() const { return m_wallet_controller->m_activity_worker; } - void showProgressDialog(const QString& label_text); - void destroyProgressDialog(); + void showProgressDialog(const QString& title_text, const QString& label_text); WalletController* const m_wallet_controller; QWidget* const m_parent_widget; - QProgressDialog* m_progress_dialog{nullptr}; WalletModel* m_wallet_model{nullptr}; bilingual_str m_error_message; std::vector<bilingual_str> m_warning_message; @@ -150,4 +145,14 @@ private: void finish(); }; +class LoadWalletsActivity : public WalletControllerActivity +{ + Q_OBJECT + +public: + LoadWalletsActivity(WalletController* wallet_controller, QWidget* parent_widget); + + void load(); +}; + #endif // BITCOIN_QT_WALLETCONTROLLER_H diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 5eeb2d5308..4ff92bf82c 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -221,10 +221,9 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) return; } - PSBTOperationsDialog* dlg = new PSBTOperationsDialog(this, currentWalletModel(), clientModel); + auto dlg = new PSBTOperationsDialog(this, currentWalletModel(), clientModel); dlg->openWithPSBT(psbtx); - dlg->setAttribute(Qt::WA_DeleteOnClose); - dlg->exec(); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void WalletFrame::encryptWallet() diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 967dd588b4..052453cf65 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -506,9 +506,10 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); } - SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); - confirmationDialog.exec(); - QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); + auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString); + confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); + // TODO: Replace QDialog::exec() with safer QDialog::show(). + const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); // cancel sign&broadcast if user doesn't want to bump the fee if (retval != QMessageBox::Yes) { diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index 120d240d91..0bae4bade3 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -8,7 +8,7 @@ #include <primitives/transaction.h> #include <qt/sendcoinsrecipient.h> -#include <amount.h> +#include <consensus/amount.h> #include <QObject> diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 309806a1c4..7813b89e41 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -205,11 +205,10 @@ void WalletView::showOutOfSyncWarning(bool fShow) void WalletView::encryptWallet() { - AskPassphraseDialog dlg(AskPassphraseDialog::Encrypt, this); - dlg.setModel(walletModel); - dlg.exec(); - - Q_EMIT encryptionStatusChanged(); + auto dlg = new AskPassphraseDialog(AskPassphraseDialog::Encrypt, this); + dlg->setModel(walletModel); + connect(dlg, &QDialog::finished, this, &WalletView::encryptionStatusChanged); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void WalletView::backupWallet() @@ -234,19 +233,18 @@ void WalletView::backupWallet() void WalletView::changePassphrase() { - AskPassphraseDialog dlg(AskPassphraseDialog::ChangePass, this); - dlg.setModel(walletModel); - dlg.exec(); + auto dlg = new AskPassphraseDialog(AskPassphraseDialog::ChangePass, this); + dlg->setModel(walletModel); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void WalletView::unlockWallet() { // Unlock wallet when requested by wallet model - if (walletModel->getEncryptionStatus() == WalletModel::Locked) - { - AskPassphraseDialog dlg(AskPassphraseDialog::Unlock, this); - dlg.setModel(walletModel); - dlg.exec(); + if (walletModel->getEncryptionStatus() == WalletModel::Locked) { + auto dlg = new AskPassphraseDialog(AskPassphraseDialog::Unlock, this); + dlg->setModel(walletModel); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } } diff --git a/src/qt/walletview.h b/src/qt/walletview.h index eebc163624..86a835c484 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_WALLETVIEW_H #define BITCOIN_QT_WALLETVIEW_H -#include <amount.h> +#include <consensus/amount.h> #include <QStackedWidget> diff --git a/src/randomenv.cpp b/src/randomenv.cpp index fa2a3a0607..bf23ea4a12 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -53,7 +53,7 @@ #include <sys/vmmeter.h> #endif #endif -#if defined(HAVE_STRONG_GETAUXVAL) || defined(HAVE_WEAK_GETAUXVAL) +#if defined(HAVE_STRONG_GETAUXVAL) #include <sys/auxv.h> #endif @@ -326,7 +326,7 @@ void RandAddStaticEnv(CSHA512& hasher) // Bitcoin client version hasher << CLIENT_VERSION; -#if defined(HAVE_STRONG_GETAUXVAL) || defined(HAVE_WEAK_GETAUXVAL) +#if defined(HAVE_STRONG_GETAUXVAL) // Information available through getauxval() # ifdef AT_HWCAP hasher << getauxval(AT_HWCAP); @@ -346,7 +346,7 @@ void RandAddStaticEnv(CSHA512& hasher) const char* exec_str = (const char*)getauxval(AT_EXECFN); if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); # endif -#endif // HAVE_STRONG_GETAUXVAL || HAVE_WEAK_GETAUXVAL +#endif // HAVE_STRONG_GETAUXVAL #ifdef HAVE_GETCPUID AddAllCPUID(hasher); diff --git a/src/rest.cpp b/src/rest.cpp index e50ab33e54..3746fd752a 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -189,9 +189,10 @@ static bool rest_headers(const std::any& context, if (path.size() != 2) return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>."); - long count = strtol(path[0].c_str(), nullptr, 10); - if (count < 1 || count > 2000) + const auto parsed_count{ToIntegral<size_t>(path[0])}; + if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > 2000) { return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); + } std::string hashStr = path[1]; uint256 hash; @@ -199,8 +200,8 @@ static bool rest_headers(const std::any& context, return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); const CBlockIndex* tip = nullptr; - std::vector<const CBlockIndex *> headers; - headers.reserve(count); + std::vector<const CBlockIndex*> headers; + headers.reserve(*parsed_count); { ChainstateManager* maybe_chainman = GetChainman(context, req); if (!maybe_chainman) return false; @@ -211,8 +212,9 @@ static bool rest_headers(const std::any& context, const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); while (pindex != nullptr && active_chain.Contains(pindex)) { headers.push_back(pindex); - if (headers.size() == (unsigned long)count) + if (headers.size() == *parsed_count) { break; + } pindex = active_chain.Next(pindex); } } @@ -260,7 +262,7 @@ static bool rest_headers(const std::any& context, static bool rest_block(const std::any& context, HTTPRequest* req, const std::string& strURIPart, - bool showTxDetails) + TxVerbosity tx_verbosity) { if (!CheckWarmup(req)) return false; @@ -312,7 +314,7 @@ static bool rest_block(const std::any& context, } case RetFormat::JSON: { - UniValue objBlock = blockToJSON(block, tip, pblockindex, showTxDetails); + UniValue objBlock = blockToJSON(block, tip, pblockindex, tx_verbosity); std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -327,12 +329,12 @@ static bool rest_block(const std::any& context, static bool rest_block_extended(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { - return rest_block(context, req, strURIPart, true); + return rest_block(context, req, strURIPart, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); } static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { - return rest_block(context, req, strURIPart, false); + return rest_block(context, req, strURIPart, TxVerbosity::SHOW_TXID); } // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 909019d796..55048f6811 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -5,11 +5,11 @@ #include <rpc/blockchain.h> -#include <amount.h> #include <blockfilter.h> #include <chain.h> #include <chainparams.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/params.h> #include <consensus/validation.h> #include <core_io.h> @@ -200,7 +200,7 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex return result; } -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails) +UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) { UniValue result = blockheaderToJSON(tip, blockindex); @@ -208,22 +208,29 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); result.pushKV("weight", (int)::GetBlockWeight(block)); UniValue txs(UniValue::VARR); - if (txDetails) { - CBlockUndo blockUndo; - const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); - for (size_t i = 0; i < block.vtx.size(); ++i) { - const CTransactionRef& tx = block.vtx.at(i); - // coinbase transaction (i == 0) doesn't have undo data - const CTxUndo* txundo = (have_undo && i) ? &blockUndo.vtxundo.at(i - 1) : nullptr; - UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo); - txs.push_back(objTx); - } - } else { - for (const CTransactionRef& tx : block.vtx) { - txs.push_back(tx->GetHash().GetHex()); - } + + switch (verbosity) { + case TxVerbosity::SHOW_TXID: + for (const CTransactionRef& tx : block.vtx) { + txs.push_back(tx->GetHash().GetHex()); + } + break; + + case TxVerbosity::SHOW_DETAILS: + case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: + CBlockUndo blockUndo; + const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); + + for (size_t i = 0; i < block.vtx.size(); ++i) { + const CTransactionRef& tx = block.vtx.at(i); + // coinbase transaction (i.e. i == 0) doesn't have undo data + const CTxUndo* txundo = (have_undo && i > 0) ? &blockUndo.vtxundo.at(i - 1) : nullptr; + UniValue objTx(UniValue::VOBJ); + TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo, verbosity); + txs.push_back(objTx); + } } + result.pushKV("tx", txs); return result; @@ -509,7 +516,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool std::set<std::string> setDepends; for (const CTxIn& txin : tx.vin) { - if (pool.exists(txin.prevout.hash)) + if (pool.exists(GenTxid::Txid(txin.prevout.hash))) setDepends.insert(txin.prevout.hash.ToString()); } @@ -931,7 +938,8 @@ static RPCHelpMan getblock() return RPCHelpMan{"getblock", "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" "If verbosity is 1, returns an Object with information about block <hash>.\n" - "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n", + "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction.\n" + "If verbosity is 3, returns an Object with information about block <hash> and information about each transaction, including prevout information for inputs (only for unpruned blocks in the current best chain).\n", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{1}, "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, @@ -1018,7 +1026,16 @@ static RPCHelpMan getblock() return strHex; } - return blockToJSON(block, tip, pblockindex, verbosity >= 2); + TxVerbosity tx_verbosity; + if (verbosity == 1) { + tx_verbosity = TxVerbosity::SHOW_TXID; + } else if (verbosity == 2) { + tx_verbosity = TxVerbosity::SHOW_DETAILS; + } else { + tx_verbosity = TxVerbosity::SHOW_DETAILS_AND_PREVOUT; + } + + return blockToJSON(block, tip, pblockindex, tx_verbosity); }, }; } @@ -1115,11 +1132,11 @@ static RPCHelpMan gettxoutsetinfo() {RPCResult::Type::NUM, "bogosize", "Database-independent, meaningless metric indicating the UTXO set size"}, {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"}, - {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs (not available when coinstatsindex is used)"}, - {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"}, + {RPCResult::Type::NUM, "transactions", /* optional */ true, "The number of transactions with unspent outputs (not available when coinstatsindex is used)"}, + {RPCResult::Type::NUM, "disk_size", /* optional */ true, "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of coins in the UTXO set"}, - {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"}, - {RPCResult::Type::OBJ, "block_info", "Info on amounts in the block at this block height (only available if coinstatsindex is used)", + {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", /* optional */ true, "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"}, + {RPCResult::Type::OBJ, "block_info", /* optional */ true, "Info on amounts in the block at this block height (only available if coinstatsindex is used)", { {RPCResult::Type::STR_AMOUNT, "prevout_spent", "Total amount of all prevouts spent in this block"}, {RPCResult::Type::STR_AMOUNT, "coinbase", "Coinbase subsidy amount of this block"}, @@ -1256,11 +1273,8 @@ static RPCHelpMan gettxout() {RPCResult::Type::OBJ, "scriptPubKey", "", { {RPCResult::Type::STR, "asm", ""}, {RPCResult::Type::STR_HEX, "hex", ""}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, {RPCResult::Type::STR, "type", "The type, eg pubkeyhash"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - {{RPCResult::Type::STR, "address", "bitcoin address"}}}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, }}, {RPCResult::Type::BOOL, "coinbase", "Coinbase or not"}, }}, @@ -1436,32 +1450,32 @@ RPCHelpMan getblockchaininfo() {RPCResult::Type::STR_HEX, "chainwork", "total amount of work in active chain, in hexadecimal"}, {RPCResult::Type::NUM, "size_on_disk", "the estimated size of the block and undo files on disk"}, {RPCResult::Type::BOOL, "pruned", "if the blocks are subject to pruning"}, - {RPCResult::Type::NUM, "pruneheight", "lowest-height complete block stored (only present if pruning is enabled)"}, - {RPCResult::Type::BOOL, "automatic_pruning", "whether automatic pruning is enabled (only present if pruning is enabled)"}, - {RPCResult::Type::NUM, "prune_target_size", "the target size used by pruning (only present if automatic pruning is enabled)"}, + {RPCResult::Type::NUM, "pruneheight", /* optional */ true, "lowest-height complete block stored (only present if pruning is enabled)"}, + {RPCResult::Type::BOOL, "automatic_pruning", /* optional */ true, "whether automatic pruning is enabled (only present if pruning is enabled)"}, + {RPCResult::Type::NUM, "prune_target_size", /* optional */ true, "the target size used by pruning (only present if automatic pruning is enabled)"}, {RPCResult::Type::OBJ_DYN, "softforks", "status of softforks", { {RPCResult::Type::OBJ, "xxxx", "name of the softfork", { {RPCResult::Type::STR, "type", "one of \"buried\", \"bip9\""}, - {RPCResult::Type::OBJ, "bip9", "status of bip9 softforks (only for \"bip9\" type)", + {RPCResult::Type::OBJ, "bip9", /* optional */ true, "status of bip9 softforks (only for \"bip9\" type)", { {RPCResult::Type::STR, "status", "one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\""}, - {RPCResult::Type::NUM, "bit", "the bit (0-28) in the block version field used to signal this softfork (only for \"started\" and \"locked_in\" status)"}, + {RPCResult::Type::NUM, "bit", /* optional */ true, "the bit (0-28) in the block version field used to signal this softfork (only for \"started\" and \"locked_in\" status)"}, {RPCResult::Type::NUM_TIME, "start_time", "the minimum median time past of a block at which the bit gains its meaning"}, {RPCResult::Type::NUM_TIME, "timeout", "the median time past of a block at which the deployment is considered failed if not yet locked in"}, {RPCResult::Type::NUM, "since", "height of the first block to which the status applies"}, {RPCResult::Type::NUM, "min_activation_height", "minimum height of blocks for which the rules may be enforced"}, - {RPCResult::Type::OBJ, "statistics", "numeric statistics about signalling for a softfork (only for \"started\" and \"locked_in\" status)", + {RPCResult::Type::OBJ, "statistics", /* optional */ true, "numeric statistics about signalling for a softfork (only for \"started\" and \"locked_in\" status)", { {RPCResult::Type::NUM, "period", "the length in blocks of the signalling period"}, - {RPCResult::Type::NUM, "threshold", "the number of blocks with the version bit set required to activate the feature (only for \"started\" status)"}, + {RPCResult::Type::NUM, "threshold", /* optional */ true, "the number of blocks with the version bit set required to activate the feature (only for \"started\" status)"}, {RPCResult::Type::NUM, "elapsed", "the number of blocks elapsed since the beginning of the current period"}, {RPCResult::Type::NUM, "count", "the number of blocks with the version bit set in the current period"}, - {RPCResult::Type::BOOL, "possible", "returns false if there are not enough blocks left in this period to pass activation threshold (only for \"started\" status)"}, + {RPCResult::Type::BOOL, "possible", /* optional */ true, "returns false if there are not enough blocks left in this period to pass activation threshold (only for \"started\" status)"}, }}, }}, - {RPCResult::Type::NUM, "height", "height of the first block which the rules are or will be enforced (only for \"buried\" type, or \"bip9\" type with \"active\" status)"}, + {RPCResult::Type::NUM, "height", /* optional */ true, "height of the first block which the rules are or will be enforced (only for \"buried\" type, or \"bip9\" type with \"active\" status)"}, {RPCResult::Type::BOOL, "active", "true if the rules are enforced for the mempool and the next block"}, }}, }}, @@ -1503,7 +1517,7 @@ RPCHelpMan getblockchaininfo() obj.pushKV("pruneheight", block->nHeight); // if 0, execution bypasses the whole if block. - bool automatic_pruning = (gArgs.GetArg("-prune", 0) != 1); + bool automatic_pruning = (gArgs.GetIntArg("-prune", 0) != 1); obj.pushKV("automatic_pruning", automatic_pruning); if (automatic_pruning) { obj.pushKV("prune_target_size", nPruneTarget); @@ -1650,7 +1664,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); - size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + size_t maxmempool = gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t) maxmempool); ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); @@ -1933,16 +1947,6 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], } } -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex) -{ - ScriptPubKeyToUniv(scriptPubKey, out, fIncludeHex, IsDeprecatedRPCEnabled("addresses")); -} - -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo) -{ - TxToUniv(tx, hashBlock, IsDeprecatedRPCEnabled("addresses"), entry, include_hex, serialize_flags, txundo); -} - template<typename T> static inline bool SetHasKeys(const std::set<T>& set) {return false;} template<typename T, typename Tk, typename... Args> @@ -1971,11 +1975,11 @@ static RPCHelpMan getblockstats() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::NUM, "avgfee", "Average fee in the block"}, - {RPCResult::Type::NUM, "avgfeerate", "Average feerate (in satoshis per virtual byte)"}, - {RPCResult::Type::NUM, "avgtxsize", "Average transaction size"}, - {RPCResult::Type::STR_HEX, "blockhash", "The block hash (to check for potential reorgs)"}, - {RPCResult::Type::ARR_FIXED, "feerate_percentiles", "Feerates at the 10th, 25th, 50th, 75th, and 90th percentile weight unit (in satoshis per virtual byte)", + {RPCResult::Type::NUM, "avgfee", /* optional */ true, "Average fee in the block"}, + {RPCResult::Type::NUM, "avgfeerate", /* optional */ true, "Average feerate (in satoshis per virtual byte)"}, + {RPCResult::Type::NUM, "avgtxsize", /* optional */ true, "Average transaction size"}, + {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "The block hash (to check for potential reorgs)"}, + {RPCResult::Type::ARR_FIXED, "feerate_percentiles", /* optional */ true, "Feerates at the 10th, 25th, 50th, 75th, and 90th percentile weight unit (in satoshis per virtual byte)", { {RPCResult::Type::NUM, "10th_percentile_feerate", "The 10th percentile feerate"}, {RPCResult::Type::NUM, "25th_percentile_feerate", "The 25th percentile feerate"}, @@ -1983,30 +1987,30 @@ static RPCHelpMan getblockstats() {RPCResult::Type::NUM, "75th_percentile_feerate", "The 75th percentile feerate"}, {RPCResult::Type::NUM, "90th_percentile_feerate", "The 90th percentile feerate"}, }}, - {RPCResult::Type::NUM, "height", "The height of the block"}, - {RPCResult::Type::NUM, "ins", "The number of inputs (excluding coinbase)"}, - {RPCResult::Type::NUM, "maxfee", "Maximum fee in the block"}, - {RPCResult::Type::NUM, "maxfeerate", "Maximum feerate (in satoshis per virtual byte)"}, - {RPCResult::Type::NUM, "maxtxsize", "Maximum transaction size"}, - {RPCResult::Type::NUM, "medianfee", "Truncated median fee in the block"}, - {RPCResult::Type::NUM, "mediantime", "The block median time past"}, - {RPCResult::Type::NUM, "mediantxsize", "Truncated median transaction size"}, - {RPCResult::Type::NUM, "minfee", "Minimum fee in the block"}, - {RPCResult::Type::NUM, "minfeerate", "Minimum feerate (in satoshis per virtual byte)"}, - {RPCResult::Type::NUM, "mintxsize", "Minimum transaction size"}, - {RPCResult::Type::NUM, "outs", "The number of outputs"}, - {RPCResult::Type::NUM, "subsidy", "The block subsidy"}, - {RPCResult::Type::NUM, "swtotal_size", "Total size of all segwit transactions"}, - {RPCResult::Type::NUM, "swtotal_weight", "Total weight of all segwit transactions"}, - {RPCResult::Type::NUM, "swtxs", "The number of segwit transactions"}, - {RPCResult::Type::NUM, "time", "The block time"}, - {RPCResult::Type::NUM, "total_out", "Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee])"}, - {RPCResult::Type::NUM, "total_size", "Total size of all non-coinbase transactions"}, - {RPCResult::Type::NUM, "total_weight", "Total weight of all non-coinbase transactions"}, - {RPCResult::Type::NUM, "totalfee", "The fee total"}, - {RPCResult::Type::NUM, "txs", "The number of transactions (including coinbase)"}, - {RPCResult::Type::NUM, "utxo_increase", "The increase/decrease in the number of unspent outputs"}, - {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, + {RPCResult::Type::NUM, "height", /* optional */ true, "The height of the block"}, + {RPCResult::Type::NUM, "ins", /* optional */ true, "The number of inputs (excluding coinbase)"}, + {RPCResult::Type::NUM, "maxfee", /* optional */ true, "Maximum fee in the block"}, + {RPCResult::Type::NUM, "maxfeerate", /* optional */ true, "Maximum feerate (in satoshis per virtual byte)"}, + {RPCResult::Type::NUM, "maxtxsize", /* optional */ true, "Maximum transaction size"}, + {RPCResult::Type::NUM, "medianfee", /* optional */ true, "Truncated median fee in the block"}, + {RPCResult::Type::NUM, "mediantime", /* optional */ true, "The block median time past"}, + {RPCResult::Type::NUM, "mediantxsize", /* optional */ true, "Truncated median transaction size"}, + {RPCResult::Type::NUM, "minfee", /* optional */ true, "Minimum fee in the block"}, + {RPCResult::Type::NUM, "minfeerate", /* optional */ true, "Minimum feerate (in satoshis per virtual byte)"}, + {RPCResult::Type::NUM, "mintxsize", /* optional */ true, "Minimum transaction size"}, + {RPCResult::Type::NUM, "outs", /* optional */ true, "The number of outputs"}, + {RPCResult::Type::NUM, "subsidy", /* optional */ true, "The block subsidy"}, + {RPCResult::Type::NUM, "swtotal_size", /* optional */ true, "Total size of all segwit transactions"}, + {RPCResult::Type::NUM, "swtotal_weight", /* optional */ true, "Total weight of all segwit transactions"}, + {RPCResult::Type::NUM, "swtxs", /* optional */ true, "The number of segwit transactions"}, + {RPCResult::Type::NUM, "time", /* optional */ true, "The block time"}, + {RPCResult::Type::NUM, "total_out", /* optional */ true, "Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee])"}, + {RPCResult::Type::NUM, "total_size", /* optional */ true, "Total size of all non-coinbase transactions"}, + {RPCResult::Type::NUM, "total_weight", /* optional */ true, "Total weight of all non-coinbase transactions"}, + {RPCResult::Type::NUM, "totalfee", /* optional */ true, "The fee total"}, + {RPCResult::Type::NUM, "txs", /* optional */ true, "The number of transactions (including coinbase)"}, + {RPCResult::Type::NUM, "utxo_increase", /* optional */ true, "The increase/decrease in the number of unspent outputs"}, + {RPCResult::Type::NUM, "utxo_size_inc", /* optional */ true, "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, }}, RPCExamples{ HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") + @@ -2198,7 +2202,11 @@ static RPCHelpMan savemempool() return RPCHelpMan{"savemempool", "\nDumps the mempool to disk. It will fail until the previous dump is fully loaded.\n", {}, - RPCResult{RPCResult::Type::NONE, "", ""}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "filename", "the directory and file where the mempool was saved"}, + }}, RPCExamples{ HelpExampleCli("savemempool", "") + HelpExampleRpc("savemempool", "") @@ -2207,6 +2215,8 @@ static RPCHelpMan savemempool() { const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + const NodeContext& node = EnsureAnyNodeContext(request.context); + if (!mempool.IsLoaded()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } @@ -2215,7 +2225,10 @@ static RPCHelpMan savemempool() throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } - return NullUniValue; + UniValue ret(UniValue::VOBJ); + ret.pushKV("filename", fs::path((node.args->GetDataDirNet() / "mempool.dat")).u8string()); + + return ret; }, }; } @@ -2550,15 +2563,15 @@ static RPCHelpMan dumptxoutset() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - const fs::path path = fsbridge::AbsPathJoin(gArgs.GetDataDirNet(), request.params[0].get_str()); + const fs::path path = fsbridge::AbsPathJoin(gArgs.GetDataDirNet(), fs::u8path(request.params[0].get_str())); // Write to a temporary path and then move into `path` on completion // to avoid confusion due to an interruption. - const fs::path temppath = fsbridge::AbsPathJoin(gArgs.GetDataDirNet(), request.params[0].get_str() + ".incomplete"); + const fs::path temppath = fsbridge::AbsPathJoin(gArgs.GetDataDirNet(), fs::u8path(request.params[0].get_str() + ".incomplete")); if (fs::exists(path)) { throw JSONRPCError( RPC_INVALID_PARAMETER, - path.string() + " already exists. If you are sure this is what you want, " + path.u8string() + " already exists. If you are sure this is what you want, " "move it out of the way first"); } @@ -2568,7 +2581,7 @@ static RPCHelpMan dumptxoutset() UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), afile); fs::rename(temppath, path); - result.pushKV("path", path.string()); + result.pushKV("path", path.u8string()); return result; }, }; diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index ffb6f03b47..d9c6761f47 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_RPC_BLOCKCHAIN_H #define BITCOIN_RPC_BLOCKCHAIN_H -#include <amount.h> +#include <consensus/amount.h> #include <core_io.h> #include <streams.h> #include <sync.h> @@ -39,7 +39,7 @@ double GetDifficulty(const CBlockIndex* blockindex); void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false) LOCKS_EXCLUDED(cs_main); +UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); /** Mempool information to JSON */ UniValue MempoolInfoToJSON(const CTxMemPool& pool); @@ -53,9 +53,6 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex /** Used by getblockstats to get feerates at different percentiles by weight */ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight); -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); - NodeContext& EnsureAnyNodeContext(const std::any& context); CTxMemPool& EnsureMemPool(const NodeContext& node); CTxMemPool& EnsureAnyMemPool(const std::any& context); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 4357ab2bb3..93e49cb9a8 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -131,6 +131,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxoutsetinfo", 2, "use_index"}, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, + { "lockunspent", 2, "persistent" }, { "send", 0, "outputs" }, { "send", 1, "conf_target" }, { "send", 3, "fee_rate"}, @@ -192,6 +193,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "unloadwallet", 1, "load_on_startup"}, { "getnodeaddresses", 0, "count"}, { "addpeeraddress", 1, "port"}, + { "addpeeraddress", 2, "tried"}, { "stop", 0, "wait" }, }; // clang-format on diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 6ec2b1a07f..60ec15e904 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -24,8 +24,11 @@ static RPCHelpMan enumeratesigners() { {RPCResult::Type::ARR, "signers", /* optional */ false, "", { - {RPCResult::Type::STR_HEX, "masterkeyfingerprint", "Master key fingerprint"}, - {RPCResult::Type::STR, "name", "Device name"}, + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "fingerprint", "Master key fingerprint"}, + {RPCResult::Type::STR, "name", "Device name"}, + }}, }, } } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 27cbb3a702..518c41d12a 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -3,9 +3,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <chain.h> #include <chainparams.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/params.h> #include <consensus/validation.h> @@ -553,6 +553,10 @@ static RPCHelpMan getblocktemplate() { {RPCResult::Type::NUM, "rulename", "identifies the bit number as indicating acceptance and readiness for the named softfork rule"}, }}, + {RPCResult::Type::ARR, "capabilities", "", + { + {RPCResult::Type::STR, "value", "A supported feature, for example 'proposal'"}, + }}, {RPCResult::Type::NUM, "vbrequired", "bit mask of versionbits the server requires set in submissions"}, {RPCResult::Type::STR, "previousblockhash", "The hash of current highest block"}, {RPCResult::Type::ARR, "transactions", "contents of non-coinbase transactions that should be included in the next block", @@ -586,11 +590,12 @@ static RPCHelpMan getblocktemplate() {RPCResult::Type::STR_HEX, "noncerange", "A range of valid nonces"}, {RPCResult::Type::NUM, "sigoplimit", "limit of sigops in blocks"}, {RPCResult::Type::NUM, "sizelimit", "limit of block size"}, - {RPCResult::Type::NUM, "weightlimit", "limit of block weight"}, + {RPCResult::Type::NUM, "weightlimit", /* optional */ true, "limit of block weight"}, {RPCResult::Type::NUM_TIME, "curtime", "current timestamp in " + UNIX_EPOCH_TIME}, {RPCResult::Type::STR, "bits", "compressed target of next block"}, {RPCResult::Type::NUM, "height", "The height of the next block"}, - {RPCResult::Type::STR, "default_witness_commitment", /* optional */ true, "a valid witness commitment for the unmodified block template"}, + {RPCResult::Type::STR_HEX, "signet_challenge", /* optional */ true, "Only on signet"}, + {RPCResult::Type::STR_HEX, "default_witness_commitment", /* optional */ true, "a valid witness commitment for the unmodified block template"}, }}, }, RPCExamples{ @@ -697,7 +702,7 @@ static RPCHelpMan getblocktemplate() std::string lpstr = lpval.get_str(); hashWatchedChain = ParseHashV(lpstr.substr(0, 64), "longpollid"); - nTransactionsUpdatedLastLP = atoi64(lpstr.substr(64)); + nTransactionsUpdatedLastLP = LocaleIndependentAtoi<int64_t>(lpstr.substr(64)); } else { @@ -1098,6 +1103,8 @@ static RPCHelpMan estimatesmartfee() RPCTypeCheckArgument(request.params[0], UniValue::VNUM); CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); + const NodeContext& node = EnsureAnyNodeContext(request.context); + const CTxMemPool& mempool = EnsureMemPool(node); unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); @@ -1113,7 +1120,10 @@ static RPCHelpMan estimatesmartfee() UniValue result(UniValue::VOBJ); UniValue errors(UniValue::VARR); FeeCalculation feeCalc; - CFeeRate feeRate = fee_estimator.estimateSmartFee(conf_target, &feeCalc, conservative); + CFeeRate feeRate{fee_estimator.estimateSmartFee(conf_target, &feeCalc, conservative)}; + CFeeRate min_mempool_feerate{mempool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000)}; + CFeeRate min_relay_feerate{::minRelayTxFee}; + feeRate = std::max({feeRate, min_mempool_feerate, min_relay_feerate}); if (feeRate != CFeeRate(0)) { result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); } else { diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 14b0e5a984..39bd9c6091 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -22,6 +22,7 @@ #include <util/check.h> #include <util/message.h> // For MessageSign(), MessageVerify() #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <optional> @@ -44,10 +45,10 @@ static RPCHelpMan validateaddress() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, - {RPCResult::Type::STR, "address", "The bitcoin address validated"}, - {RPCResult::Type::STR_HEX, "scriptPubKey", "The hex-encoded scriptPubKey generated by the address"}, - {RPCResult::Type::BOOL, "isscript", "If the key is a script"}, - {RPCResult::Type::BOOL, "iswitness", "If the address is a witness address"}, + {RPCResult::Type::STR, "address", /* optional */ true, "The bitcoin address validated"}, + {RPCResult::Type::STR_HEX, "scriptPubKey", /* optional */ true, "The hex-encoded scriptPubKey generated by the address"}, + {RPCResult::Type::BOOL, "isscript", /* optional */ true, "If the key is a script"}, + {RPCResult::Type::BOOL, "iswitness", /* optional */ true, "If the address is a witness address"}, {RPCResult::Type::NUM, "witness_version", /* optional */ true, "The version number of the witness program"}, {RPCResult::Type::STR_HEX, "witness_program", /* optional */ true, "The hex value of the witness program"}, {RPCResult::Type::STR, "error", /* optional */ true, "Error message, if any"}, @@ -417,6 +418,27 @@ static RPCHelpMan setmocktime() }; } +#if defined(USE_SYSCALL_SANDBOX) +static RPCHelpMan invokedisallowedsyscall() +{ + return RPCHelpMan{ + "invokedisallowedsyscall", + "\nInvoke a disallowed syscall to trigger a syscall sandbox violation. Used for testing purposes.\n", + {}, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + HelpExampleCli("invokedisallowedsyscall", "") + HelpExampleRpc("invokedisallowedsyscall", "")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + if (!Params().IsTestChain()) { + throw std::runtime_error("invokedisallowedsyscall is used for testing only."); + } + TestDisallowedSandboxCall(); + return NullUniValue; + }, + }; +} +#endif // USE_SYSCALL_SANDBOX + static RPCHelpMan mockscheduler() { return RPCHelpMan{"mockscheduler", @@ -717,7 +739,7 @@ static RPCHelpMan getindexinfo() {"index_name", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Filter results for an index with a specific name."}, }, RPCResult{ - RPCResult::Type::OBJ, "", "", { + RPCResult::Type::OBJ_DYN, "", "", { { RPCResult::Type::OBJ, "name", "The name of the index", { @@ -777,6 +799,9 @@ static const CRPCCommand commands[] = { "hidden", &echo, }, { "hidden", &echojson, }, { "hidden", &echoipc, }, +#if defined(USE_SYSCALL_SANDBOX) + { "hidden", &invokedisallowedsyscall, }, +#endif // USE_SYSCALL_SANDBOX }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 0f554ec5e7..e33f1ce4a3 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -116,10 +116,10 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::NUM, "id", "Peer index"}, {RPCResult::Type::STR, "addr", "(host:port) The IP address and port of the peer"}, - {RPCResult::Type::STR, "addrbind", "(ip:port) Bind address of the connection to the peer"}, - {RPCResult::Type::STR, "addrlocal", "(ip:port) Local address as reported by the peer"}, + {RPCResult::Type::STR, "addrbind", /* optional */ true, "(ip:port) Bind address of the connection to the peer"}, + {RPCResult::Type::STR, "addrlocal", /* optional */ true, "(ip:port) Local address as reported by the peer"}, {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/* append_unroutable */ true), ", ") + ")"}, - {RPCResult::Type::NUM, "mapped_as", "The AS in the BGP route to the peer used for diversifying\n" + {RPCResult::Type::NUM, "mapped_as", /* optional */ true, "The AS in the BGP route to the peer used for diversifying\n" "peer selection (only available if the asmap config flag is set)"}, {RPCResult::Type::STR_HEX, "services", "The services offered"}, {RPCResult::Type::ARR, "servicesnames", "the services offered, in human-readable form", @@ -135,9 +135,9 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, {RPCResult::Type::NUM, "timeoffset", "The time offset in seconds"}, - {RPCResult::Type::NUM, "pingtime", "ping time (if available)"}, - {RPCResult::Type::NUM, "minping", "minimum observed ping time (if any at all)"}, - {RPCResult::Type::NUM, "pingwait", "ping wait (if non-zero)"}, + {RPCResult::Type::NUM, "pingtime", /* optional */ true, "ping time (if available)"}, + {RPCResult::Type::NUM, "minping", /* optional */ true, "minimum observed ping time (if any at all)"}, + {RPCResult::Type::NUM, "pingwait", /* optional */ true, "ping wait (if non-zero)"}, {RPCResult::Type::NUM, "version", "The peer version, such as 70001"}, {RPCResult::Type::STR, "subver", "The string version"}, {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, @@ -566,7 +566,7 @@ static UniValue GetNetworksInfo() UniValue networks(UniValue::VARR); for (int n = 0; n < NET_MAX; ++n) { enum Network network = static_cast<enum Network>(n); - if (network == NET_UNROUTABLE || network == NET_CJDNS || network == NET_INTERNAL) continue; + if (network == NET_UNROUTABLE || network == NET_INTERNAL) continue; proxyType proxy; UniValue obj(UniValue::VOBJ); GetProxy(network, proxy); @@ -921,6 +921,7 @@ static RPCHelpMan addpeeraddress() { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address of the peer"}, {"port", RPCArg::Type::NUM, RPCArg::Optional::NO, "The port of the peer"}, + {"tried", RPCArg::Type::BOOL, RPCArg::Default{false}, "If true, attempt to add the peer to the tried addresses table"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -929,8 +930,8 @@ static RPCHelpMan addpeeraddress() }, }, RPCExamples{ - HelpExampleCli("addpeeraddress", "\"1.2.3.4\" 8333") - + HelpExampleRpc("addpeeraddress", "\"1.2.3.4\", 8333") + HelpExampleCli("addpeeraddress", "\"1.2.3.4\" 8333 true") + + HelpExampleRpc("addpeeraddress", "\"1.2.3.4\", 8333, true") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -941,6 +942,7 @@ static RPCHelpMan addpeeraddress() const std::string& addr_string{request.params[0].get_str()}; const uint16_t port{static_cast<uint16_t>(request.params[1].get_int())}; + const bool tried{request.params[2].isTrue()}; UniValue obj(UniValue::VOBJ); CNetAddr net_addr; @@ -951,7 +953,13 @@ static RPCHelpMan addpeeraddress() address.nTime = GetAdjustedTime(); // The source address is set equal to the address. This is equivalent to the peer // announcing itself. - if (node.addrman->Add({address}, address)) success = true; + if (node.addrman->Add({address}, address)) { + success = true; + if (tried) { + // Attempt to move the address to the tried addresses table. + node.addrman->Good(address); + } + } } obj.pushKV("success", success); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 00e77d89e5..89f2309cb7 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -5,6 +5,7 @@ #include <chain.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/validation.h> #include <core_io.h> #include <index/txindex.h> @@ -94,7 +95,7 @@ static RPCHelpMan getrawtransaction() RPCResult{"if verbose is set to true", RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "in_active_chain", "Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)"}, + {RPCResult::Type::BOOL, "in_active_chain", /* optional */ true, "Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)"}, {RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded data for 'txid'"}, {RPCResult::Type::STR_HEX, "txid", "The transaction id (same as provided)"}, {RPCResult::Type::STR_HEX, "hash", "The transaction hash (differs from txid for witness transactions)"}, @@ -115,7 +116,7 @@ static RPCHelpMan getrawtransaction() {RPCResult::Type::STR_HEX, "hex", "hex"}, }}, {RPCResult::Type::NUM, "sequence", "The script sequence number"}, - {RPCResult::Type::ARR, "txinwitness", "", + {RPCResult::Type::ARR, "txinwitness", /* optional */ true, "", { {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, }}, @@ -131,20 +132,15 @@ static RPCHelpMan getrawtransaction() { {RPCResult::Type::STR, "asm", "the asm"}, {RPCResult::Type::STR, "hex", "the hex"}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - { - {RPCResult::Type::STR, "address", "bitcoin address"}, - }}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, }}, }}, }}, - {RPCResult::Type::STR_HEX, "blockhash", "the block hash"}, - {RPCResult::Type::NUM, "confirmations", "The confirmations"}, - {RPCResult::Type::NUM_TIME, "blocktime", "The block time expressed in " + UNIX_EPOCH_TIME}, - {RPCResult::Type::NUM, "time", "Same as \"blocktime\""}, + {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "the block hash"}, + {RPCResult::Type::NUM, "confirmations", /* optional */ true, "The confirmations"}, + {RPCResult::Type::NUM_TIME, "blocktime", /* optional */ true, "The block time expressed in " + UNIX_EPOCH_TIME}, + {RPCResult::Type::NUM, "time", /* optional */ true, "Same as \"blocktime\""}, } }, }, @@ -470,14 +466,15 @@ static RPCHelpMan decoderawtransaction() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, - {RPCResult::Type::NUM, "vout", "The output number"}, - {RPCResult::Type::OBJ, "scriptSig", "The script", + {RPCResult::Type::STR_HEX, "coinbase", /* optional */ true, ""}, + {RPCResult::Type::STR_HEX, "txid", /* optional */ true, "The transaction id"}, + {RPCResult::Type::NUM, "vout", /* optional */ true, "The output number"}, + {RPCResult::Type::OBJ, "scriptSig", /* optional */ true, "The script", { {RPCResult::Type::STR, "asm", "asm"}, {RPCResult::Type::STR_HEX, "hex", "hex"}, }}, - {RPCResult::Type::ARR, "txinwitness", "", + {RPCResult::Type::ARR, "txinwitness", /* optional */ true, "", { {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, }}, @@ -494,13 +491,8 @@ static RPCHelpMan decoderawtransaction() { {RPCResult::Type::STR, "asm", "the asm"}, {RPCResult::Type::STR_HEX, "hex", "the hex"}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - { - {RPCResult::Type::STR, "address", "bitcoin address"}, - }}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, }}, }}, }}, @@ -553,24 +545,14 @@ static RPCHelpMan decodescript() { {RPCResult::Type::STR, "asm", "Script public key"}, {RPCResult::Type::STR, "type", "The output type (e.g. "+GetAllOutputTypes()+")"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - { - {RPCResult::Type::STR, "address", "bitcoin address"}, - }}, - {RPCResult::Type::STR, "p2sh", "address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH)"}, - {RPCResult::Type::OBJ, "segwit", "Result of a witness script public key wrapping this redeem script (not returned if the script is a P2SH or witness)", + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "p2sh", /* optional */ true, "address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH)"}, + {RPCResult::Type::OBJ, "segwit", /* optional */ true, "Result of a witness script public key wrapping this redeem script (not returned if the script is a P2SH or witness)", { {RPCResult::Type::STR, "asm", "String representation of the script public key"}, {RPCResult::Type::STR_HEX, "hex", "Hex string of the script public key"}, {RPCResult::Type::STR, "type", "The type of the script public key (e.g. witness_v0_keyhash or witness_v0_scripthash)"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - { - {RPCResult::Type::STR, "address", "segwit address"}, - }}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "p2sh-segwit", "address of the P2SH script wrapping this witness redeem script"}, }}, } @@ -591,7 +573,7 @@ static RPCHelpMan decodescript() } else { // Empty scripts are valid } - ScriptPubKeyToUniv(script, r, /* fIncludeHex */ false); + ScriptPubKeyToUniv(script, r, /* include_hex */ false); UniValue type; type = find_value(r, "type"); @@ -625,7 +607,7 @@ static RPCHelpMan decodescript() // Newer segwit program versions should be considered when then become available. segwitScr = GetScriptForDestination(WitnessV0ScriptHash(script)); } - ScriptPubKeyToUniv(segwitScr, sr, /* fIncludeHex */ true); + ScriptPubKeyToUniv(segwitScr, sr, /* include_hex */ true); sr.pushKV("p2sh-segwit", EncodeDestination(ScriptHash(segwitScr))); r.pushKV("segwit", sr); } @@ -772,6 +754,10 @@ static RPCHelpMan signrawtransactionwithkey() { {RPCResult::Type::STR_HEX, "txid", "The hash of the referenced, previous transaction"}, {RPCResult::Type::NUM, "vout", "The index of the output to spent and used as input"}, + {RPCResult::Type::ARR, "witness", "", + { + {RPCResult::Type::STR_HEX, "witness", ""}, + }}, {RPCResult::Type::STR_HEX, "scriptSig", "The hex-encoded signature script"}, {RPCResult::Type::NUM, "sequence", "Script sequence number"}, {RPCResult::Type::STR, "error", "Verification or signing error related to the input"}, @@ -909,15 +895,15 @@ static RPCHelpMan testmempoolaccept() { {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, {RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, - {RPCResult::Type::STR, "package-error", "Package validation error, if any (only possible if rawtxs had more than 1 transaction)."}, - {RPCResult::Type::BOOL, "allowed", "Whether this tx would be accepted to the mempool and pass client-specified maxfeerate." + {RPCResult::Type::STR, "package-error", /* optional */ true, "Package validation error, if any (only possible if rawtxs had more than 1 transaction)."}, + {RPCResult::Type::BOOL, "allowed", /* optional */ true, "Whether this tx would be accepted to the mempool and pass client-specified maxfeerate. " "If not present, the tx was not fully validated due to a failure in another tx in the list."}, - {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, - {RPCResult::Type::OBJ, "fees", "Transaction fees (only present if 'allowed' is true)", + {RPCResult::Type::NUM, "vsize", /* optional */ true, "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, + {RPCResult::Type::OBJ, "fees", /* optional */ true, "Transaction fees (only present if 'allowed' is true)", { {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, }}, - {RPCResult::Type::STR, "reject-reason", "Rejection string (only present when 'allowed' is false)"}, + {RPCResult::Type::STR, "reject-reason", /* optional */ true, "Rejection string (only present when 'allowed' is false)"}, }}, } }, @@ -960,12 +946,13 @@ static RPCHelpMan testmempoolaccept() NodeContext& node = EnsureAnyNodeContext(request.context); CTxMemPool& mempool = EnsureMemPool(node); - CChainState& chainstate = EnsureChainman(node).ActiveChainstate(); + ChainstateManager& chainman = EnsureChainman(node); + CChainState& chainstate = chainman.ActiveChainstate(); const PackageMempoolAcceptResult package_result = [&] { LOCK(::cs_main); if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /* test_accept */ true); return PackageMempoolAcceptResult(txns[0]->GetWitnessHash(), - AcceptToMemoryPool(chainstate, mempool, txns[0], /* bypass_limits */ false, /* test_accept*/ true)); + chainman.ProcessTransaction(txns[0], /*test_accept=*/ true)); }(); UniValue rpc_result(UniValue::VARR); @@ -991,7 +978,7 @@ static RPCHelpMan testmempoolaccept() if (tx_result.m_result_type == MempoolAcceptResult::ResultType::VALID) { const CAmount fee = tx_result.m_base_fees.value(); // Check that fee does not exceed maximum fee - const int64_t virtual_size = GetVirtualTransactionSize(*tx); + const int64_t virtual_size = tx_result.m_vsize.value(); const CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); if (max_raw_tx_fee && fee > max_raw_tx_fee) { result_inner.pushKV("allowed", false); @@ -1056,7 +1043,7 @@ static RPCHelpMan decodepsbt() {RPCResult::Type::STR, "asm", "The asm"}, {RPCResult::Type::STR_HEX, "hex", "The hex"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address"," Bitcoin address if there is one"}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, }}, }}, {RPCResult::Type::OBJ_DYN, "partial_signatures", /* optional */ true, "", @@ -1078,22 +1065,23 @@ static RPCHelpMan decodepsbt() }}, {RPCResult::Type::ARR, "bip32_derivs", /* optional */ true, "", { - {RPCResult::Type::OBJ, "pubkey", /* optional */ true, "The public key with the derivation path as the value.", + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "pubkey", "The public key with the derivation path as the value."}, {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, {RPCResult::Type::STR, "path", "The path"}, }}, }}, - {RPCResult::Type::OBJ, "final_scriptsig", /* optional */ true, "", + {RPCResult::Type::OBJ, "final_scriptSig", /* optional */ true, "", { {RPCResult::Type::STR, "asm", "The asm"}, {RPCResult::Type::STR, "hex", "The hex"}, }}, - {RPCResult::Type::ARR, "final_scriptwitness", "", + {RPCResult::Type::ARR, "final_scriptwitness", /* optional */ true, "", { {RPCResult::Type::STR_HEX, "", "hex-encoded witness data (if any)"}, }}, - {RPCResult::Type::OBJ_DYN, "unknown", "The unknown global fields", + {RPCResult::Type::OBJ_DYN, "unknown", /* optional */ true, "The unknown global fields", { {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, }}, @@ -1124,7 +1112,7 @@ static RPCHelpMan decodepsbt() {RPCResult::Type::STR, "path", "The path"}, }}, }}, - {RPCResult::Type::OBJ_DYN, "unknown", "The unknown global fields", + {RPCResult::Type::OBJ_DYN, "unknown", /* optional */ true, "The unknown global fields", { {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, }}, @@ -1175,7 +1163,7 @@ static RPCHelpMan decodepsbt() txout = input.witness_utxo; UniValue o(UniValue::VOBJ); - ScriptToUniv(txout.scriptPubKey, o, true); + ScriptPubKeyToUniv(txout.scriptPubKey, o, /* include_hex */ true); UniValue out(UniValue::VOBJ); out.pushKV("amount", ValueFromAmount(txout.nValue)); @@ -1222,12 +1210,12 @@ static RPCHelpMan decodepsbt() // Redeem script and witness script if (!input.redeem_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(input.redeem_script, r, false); + ScriptToUniv(input.redeem_script, r); in.pushKV("redeem_script", r); } if (!input.witness_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(input.witness_script, r, false); + ScriptToUniv(input.witness_script, r); in.pushKV("witness_script", r); } @@ -1282,12 +1270,12 @@ static RPCHelpMan decodepsbt() // Redeem script and witness script if (!output.redeem_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(output.redeem_script, r, false); + ScriptToUniv(output.redeem_script, r); out.pushKV("redeem_script", r); } if (!output.witness_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(output.witness_script, r, false); + ScriptToUniv(output.witness_script, r); out.pushKV("witness_script", r); } @@ -1398,8 +1386,8 @@ static RPCHelpMan finalizepsbt() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "psbt", "The base64-encoded partially signed transaction if not extracted"}, - {RPCResult::Type::STR_HEX, "hex", "The hex-encoded network transaction if extracted"}, + {RPCResult::Type::STR, "psbt", /* optional */ true, "The base64-encoded partially signed transaction if not extracted"}, + {RPCResult::Type::STR_HEX, "hex", /* optional */ true, "The hex-encoded network transaction if extracted"}, {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, } }, @@ -1791,7 +1779,7 @@ static RPCHelpMan analyzepsbt() RPCResult { RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::ARR, "inputs", "", + {RPCResult::Type::ARR, "inputs", /* optional */ true, "", { {RPCResult::Type::OBJ, "", "", { diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index f21eddf56c..d550160260 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -6,6 +6,7 @@ #include <rpc/rawtransaction_util.h> #include <coins.h> +#include <consensus/amount.h> #include <core_io.h> #include <key_io.h> #include <policy/policy.h> diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index a7866474e1..3245e04cdf 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -70,7 +70,7 @@ static fs::path GetAuthCookieFile(bool temp=false) if (temp) { arg += ".tmp"; } - return AbsPathForConfigVal(fs::path(arg)); + return AbsPathForConfigVal(fs::PathFromString(arg)); } bool GenerateAuthCookie(std::string *cookie_out) @@ -87,7 +87,7 @@ bool GenerateAuthCookie(std::string *cookie_out) fs::path filepath_tmp = GetAuthCookieFile(true); file.open(filepath_tmp); if (!file.is_open()) { - LogPrintf("Unable to open cookie authentication file %s for writing\n", filepath_tmp.string()); + LogPrintf("Unable to open cookie authentication file %s for writing\n", fs::PathToString(filepath_tmp)); return false; } file << cookie; @@ -95,10 +95,10 @@ bool GenerateAuthCookie(std::string *cookie_out) fs::path filepath = GetAuthCookieFile(false); if (!RenameOver(filepath_tmp, filepath)) { - LogPrintf("Unable to rename cookie authentication file %s to %s\n", filepath_tmp.string(), filepath.string()); + LogPrintf("Unable to rename cookie authentication file %s to %s\n", fs::PathToString(filepath_tmp), fs::PathToString(filepath)); return false; } - LogPrintf("Generated RPC authentication cookie %s\n", filepath.string()); + LogPrintf("Generated RPC authentication cookie %s\n", fs::PathToString(filepath)); if (cookie_out) *cookie_out = cookie; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index cf80b08b96..9bcfba3507 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -239,7 +239,7 @@ static RPCHelpMan getrpcinfo() UniValue result(UniValue::VOBJ); result.pushKV("active_commands", active_commands); - const std::string path = LogInstance().m_file_path.string(); + const std::string path = LogInstance().m_file_path.u8string(); UniValue log_path(UniValue::VSTR, path); result.pushKV("logpath", log_path); @@ -540,7 +540,7 @@ void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nS int RPCSerializationFlags() { int flag = 0; - if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) + if (gArgs.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) flag |= SERIALIZE_TRANSACTION_NO_WITNESS; return flag; } diff --git a/src/rpc/server.h b/src/rpc/server.h index 03967020c2..e6bb35fc33 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -6,7 +6,6 @@ #ifndef BITCOIN_RPC_SERVER_H #define BITCOIN_RPC_SERVER_H -#include <amount.h> #include <rpc/request.h> #include <rpc/util.h> diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 2059628b54..2d7f5f2894 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <key_io.h> #include <outputtype.h> #include <rpc/util.h> diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 02ada969a4..162cced6c7 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -5,6 +5,7 @@ #include <scheduler.h> #include <random.h> +#include <util/syscall_sandbox.h> #include <util/time.h> #include <assert.h> @@ -24,6 +25,7 @@ CScheduler::~CScheduler() void CScheduler::serviceQueue() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::SCHEDULER); WAIT_LOCK(newTaskMutex, lock); ++nThreadsServicingQueue; diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 65867c1c14..6f911f4fe7 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -96,7 +96,7 @@ void InitSignatureCache() { // nMaxCacheSize is unsigned. If -maxsigcachesize is set to zero, // setup_bytes creates the minimum possible cache (2 elements). - size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); + size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetIntArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); size_t nElems = signatureCache.setup_bytes(nMaxCacheSize); LogPrintf("Using %zu MiB out of %zu/2 requested for signature cache, able to store %zu elements\n", (nElems*sizeof(uint256)) >>20, (nMaxCacheSize*2)>>20, nElems); diff --git a/src/script/sign.cpp b/src/script/sign.cpp index b912b00365..4cb2125747 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -5,6 +5,7 @@ #include <script/sign.h> +#include <consensus/amount.h> #include <key.h> #include <policy/policy.h> #include <primitives/transaction.h> diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 67a79a157c..d9656c781d 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -266,47 +266,6 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) assert(false); } -// TODO: from v23 ("addresses" and "reqSigs" deprecated) "ExtractDestinations" should be removed -bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) -{ - addressRet.clear(); - std::vector<valtype> vSolutions; - typeRet = Solver(scriptPubKey, vSolutions); - if (typeRet == TxoutType::NONSTANDARD) { - return false; - } else if (typeRet == TxoutType::NULL_DATA) { - // This is data, not addresses - return false; - } - - if (typeRet == TxoutType::MULTISIG) - { - nRequiredRet = vSolutions.front()[0]; - for (unsigned int i = 1; i < vSolutions.size()-1; i++) - { - CPubKey pubKey(vSolutions[i]); - if (!pubKey.IsValid()) - continue; - - CTxDestination address = PKHash(pubKey); - addressRet.push_back(address); - } - - if (addressRet.empty()) - return false; - } - else - { - nRequiredRet = 1; - CTxDestination address; - if (!ExtractDestination(scriptPubKey, address)) - return false; - addressRet.push_back(address); - } - - return true; -} - namespace { class CScriptVisitor { diff --git a/src/script/standard.h b/src/script/standard.h index 78492733db..a8e57231bf 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -176,28 +176,12 @@ TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned c /** * Parse a standard scriptPubKey for the destination address. Assigns result to - * the addressRet parameter and returns true if successful. For multisig - * scripts, instead use ExtractDestinations. Currently only works for P2PK, + * the addressRet parameter and returns true if successful. Currently only works for P2PK, * P2PKH, P2SH, P2WPKH, and P2WSH scripts. */ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet); /** - * Parse a standard scriptPubKey with one or more destination addresses. For - * multisig scripts, this populates the addressRet vector with the pubkey IDs - * and nRequiredRet with the n required to spend. For other destinations, - * addressRet is populated with a single value and nRequiredRet is set to 1. - * Returns true if successful. - * - * Note: this function confuses destinations (a subset of CScripts that are - * encodable as an address) with key identifiers (of keys involved in a - * CScript), and its use should be phased out. - * - * TODO: from v23 ("addresses" and "reqSigs" deprecated) "ExtractDestinations" should be removed - */ -bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet); - -/** * Generate a Bitcoin scriptPubKey for the given CTxDestination. Returns a P2PKH * script for a CKeyID destination, a P2SH script for a CScriptID, and an empty * script for CNoDestination. diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 0e31ad3ce3..c4923dc56f 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -9,6 +9,7 @@ #include <support/lockedpool.h> #include <support/cleanse.h> +#include <memory> #include <string> // @@ -17,15 +18,13 @@ // template <typename T> struct secure_allocator : public std::allocator<T> { - // MSVC8 default copy constructor is broken - typedef std::allocator<T> base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; + using base = std::allocator<T>; + using traits = std::allocator_traits<base>; + using size_type = typename traits::size_type; + using difference_type = typename traits::difference_type; + using pointer = typename traits::pointer; + using const_pointer = typename traits::const_pointer; + using value_type = typename traits::value_type; secure_allocator() noexcept {} secure_allocator(const secure_allocator& a) noexcept : base(a) {} template <typename U> diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index 418f0ee656..77de4b1e69 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -13,15 +13,13 @@ template <typename T> struct zero_after_free_allocator : public std::allocator<T> { - // MSVC8 default copy constructor is broken - typedef std::allocator<T> base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; + using base = std::allocator<T>; + using traits = std::allocator_traits<base>; + using size_type = typename traits::size_type; + using difference_type = typename traits::difference_type; + using pointer = typename traits::pointer; + using const_pointer = typename traits::const_pointer; + using value_type = typename traits::value_type; zero_after_free_allocator() noexcept {} zero_after_free_allocator(const zero_after_free_allocator& a) noexcept : base(a) {} template <typename U> diff --git a/src/sync.cpp b/src/sync.cpp index 98e6d3d65d..c9fd8e347e 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -97,27 +97,29 @@ static void potential_deadlock_detected(const LockPair& mismatch, const LockStac LogPrintf("POTENTIAL DEADLOCK DETECTED\n"); LogPrintf("Previous lock order was:\n"); for (const LockStackItem& i : s1) { + std::string prefix{}; if (i.first == mismatch.first) { - LogPrintf(" (1)"); /* Continued */ + prefix = " (1)"; } if (i.first == mismatch.second) { - LogPrintf(" (2)"); /* Continued */ + prefix = " (2)"; } - LogPrintf(" %s\n", i.second.ToString()); + LogPrintf("%s %s\n", prefix, i.second.ToString()); } std::string mutex_a, mutex_b; LogPrintf("Current lock order is:\n"); for (const LockStackItem& i : s2) { + std::string prefix{}; if (i.first == mismatch.first) { - LogPrintf(" (1)"); /* Continued */ + prefix = " (1)"; mutex_a = i.second.Name(); } if (i.first == mismatch.second) { - LogPrintf(" (2)"); /* Continued */ + prefix = " (2)"; mutex_b = i.second.Name(); } - LogPrintf(" %s\n", i.second.ToString()); + LogPrintf("%s %s\n", prefix, i.second.ToString()); } if (g_debug_lockorder_abort) { tfm::format(std::cerr, "Assertion failed: detected inconsistent lock order for %s, details in debug log.\n", s2.back().second.ToString()); @@ -131,10 +133,11 @@ static void double_lock_detected(const void* mutex, const LockStack& lock_stack) LogPrintf("DOUBLE LOCK DETECTED\n"); LogPrintf("Lock order:\n"); for (const LockStackItem& i : lock_stack) { + std::string prefix{}; if (i.first == mutex) { - LogPrintf(" (*)"); /* Continued */ + prefix = " (*)"; } - LogPrintf(" %s\n", i.second.ToString()); + LogPrintf("%s %s\n", prefix, i.second.ToString()); } if (g_debug_lockorder_abort) { tfm::format(std::cerr, diff --git a/src/test/README.md b/src/test/README.md index 57cda26d7c..d03411c3ed 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -74,3 +74,29 @@ start debugging, just like you would with any other program: ```bash gdb src/test/test_bitcoin ``` + +#### Segmentation faults + +If you hit a segmentation fault during a test run, you can diagnose where the fault +is happening by running `gdb ./src/test/test_bitcoin` and then using the `bt` command +within gdb. + +Another tool that can be used to resolve segmentation faults is +[valgrind](https://valgrind.org/). + +If for whatever reason you want to produce a core dump file for this fault, you can do +that as well. By default, the boost test runner will intercept system errors and not +produce a core file. To bypass this, add `--catch_system_errors=no` to the +`test_bitcoin` arguments and ensure that your ulimits are set properly (e.g. `ulimit -c +unlimited`). + +Running the tests and hitting a segmentation fault should now produce a file called `core` +(on Linux platforms, the file name will likely depend on the contents of +`/proc/sys/kernel/core_pattern`). + +You can then explore the core dump using +``` bash +gdb src/test/test_bitcoin core + +(gbd) bt # produce a backtrace for where a segfault occurred +``` diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 01a492a20b..f82864b421 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -4,6 +4,7 @@ #include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> #include <chainparams.h> #include <clientversion.h> #include <hash.h> @@ -21,99 +22,39 @@ using namespace std::literals; -class CAddrManSerializationMock : public CAddrMan +class AddrManTest : public AddrMan { public: - virtual void Serialize(CDataStream& s) const = 0; - - CAddrManSerializationMock() - : CAddrMan(/* asmap */ std::vector<bool>(), /* deterministic */ true, /* consistency_check_ratio */ 100) + explicit AddrManTest(std::vector<bool> asmap = std::vector<bool>()) + : AddrMan(asmap, /*deterministic=*/true, /* consistency_check_ratio */ 100) {} -}; - -class CAddrManUncorrupted : public CAddrManSerializationMock -{ -public: - void Serialize(CDataStream& s) const override - { - CAddrMan::Serialize(s); - } -}; - -class CAddrManCorrupted : public CAddrManSerializationMock -{ -public: - void Serialize(CDataStream& s) const override - { - // Produces corrupt output that claims addrman has 20 addrs when it only has one addr. - unsigned char nVersion = 1; - s << nVersion; - s << ((unsigned char)32); - s << uint256::ONE; - s << 10; // nNew - s << 10; // nTried - - int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); - s << nUBuckets; - - CService serv; - BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); - CAddress addr = CAddress(serv, NODE_NONE); - CNetAddr resolved; - BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); - CAddrInfo info = CAddrInfo(addr, resolved); - s << info; - } -}; - -static CDataStream AddrmanToStream(const CAddrManSerializationMock& _addrman) -{ - CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); - ssPeersIn << Params().MessageStart(); - ssPeersIn << _addrman; - std::string str = ssPeersIn.str(); - std::vector<unsigned char> vchData(str.begin(), str.end()); - return CDataStream(vchData, SER_DISK, CLIENT_VERSION); -} - -class CAddrManTest : public CAddrMan -{ -private: - bool deterministic; -public: - explicit CAddrManTest(bool makeDeterministic = true, - std::vector<bool> asmap = std::vector<bool>()) - : CAddrMan(asmap, makeDeterministic, /* consistency_check_ratio */ 100) - { - deterministic = makeDeterministic; - } - CAddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) + AddrInfo* Find(const CService& addr) { - LOCK(cs); - return CAddrMan::Find(addr, pnId); + LOCK(m_impl->cs); + return m_impl->Find(addr); } - CAddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) { - LOCK(cs); - return CAddrMan::Create(addr, addrSource, pnId); + LOCK(m_impl->cs); + return m_impl->Create(addr, addrSource, pnId); } void Delete(int nId) { - LOCK(cs); - CAddrMan::Delete(nId); + LOCK(m_impl->cs); + m_impl->Delete(nId); } // Used to test deserialization std::pair<int, int> GetBucketAndEntry(const CAddress& addr) { - LOCK(cs); - int nId = mapAddr[addr]; + LOCK(m_impl->cs); + int nId = m_impl->mapAddr[addr]; for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { - if (nId == vvNew[bucket][entry]) { + if (nId == m_impl->vvNew[bucket][entry]) { return std::pair<int, int>(bucket, entry); } } @@ -124,14 +65,14 @@ public: // Simulates connection failure so that we can test eviction of offline nodes void SimConnFail(const CService& addr) { - int64_t nLastSuccess = 1; - // Set last good connection in the deep past. - Good(addr, nLastSuccess); - - bool count_failure = false; - int64_t nLastTry = GetAdjustedTime()-61; - Attempt(addr, count_failure, nLastTry); - } + int64_t nLastSuccess = 1; + // Set last good connection in the deep past. + Good(addr, nLastSuccess); + + bool count_failure = false; + int64_t nLastTry = GetAdjustedTime() - 61; + Attempt(addr, count_failure, nLastTry); + } }; static CNetAddr ResolveIP(const std::string& ip) @@ -149,7 +90,8 @@ static CService ResolveService(const std::string& ip, uint16_t port = 0) } -static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) { +static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) +{ std::vector<bool> result(vector_size); for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) { unsigned char cur_byte = source[byte_i]; @@ -165,20 +107,20 @@ BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) { - auto addrman = std::make_unique<CAddrManTest>(); + auto addrman = std::make_unique<AddrManTest>(); CNetAddr source = ResolveIP("252.2.2.2"); // Test: Does Addrman respond correctly when empty. BOOST_CHECK_EQUAL(addrman->size(), 0U); - CAddrInfo addr_null = addrman->Select(); + auto addr_null = addrman->Select().first; BOOST_CHECK_EQUAL(addr_null.ToString(), "[::]:0"); // Test: Does Addrman::Add work as expected. CService addr1 = ResolveService("250.1.1.1", 8333); BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); BOOST_CHECK_EQUAL(addrman->size(), 1U); - CAddrInfo addr_ret1 = addrman->Select(); + auto addr_ret1 = addrman->Select().first; BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); // Test: Does IP address deduplication work correctly. @@ -199,7 +141,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_CHECK(addrman->size() >= 1); // Test: reset addrman and test AddrMan::Add multiple addresses works as expected - addrman = std::make_unique<CAddrManTest>(); + addrman = std::make_unique<AddrManTest>(); std::vector<CAddress> vAddr; vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); @@ -209,7 +151,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_AUTO_TEST_CASE(addrman_ports) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -221,24 +163,24 @@ BOOST_AUTO_TEST_CASE(addrman_ports) BOOST_CHECK_EQUAL(addrman.size(), 1U); CService addr1_port = ResolveService("250.1.1.1", 8334); - BOOST_CHECK(!addrman.Add({CAddress(addr1_port, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman.size(), 1U); - CAddrInfo addr_ret2 = addrman.Select(); - BOOST_CHECK_EQUAL(addr_ret2.ToString(), "250.1.1.1:8333"); + BOOST_CHECK(addrman.Add({CAddress(addr1_port, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman.size(), 2U); + auto addr_ret2 = addrman.Select().first; + BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333" || addr_ret2.ToString() == "250.1.1.1:8334"); - // Test: Add same IP but diff port to tried table, it doesn't get added. - // Perhaps this is not ideal behavior but it is the current behavior. + // Test: Add same IP but diff port to tried table; this converts the entry with + // the specified port to tried, but not the other. addrman.Good(CAddress(addr1_port, NODE_NONE)); - BOOST_CHECK_EQUAL(addrman.size(), 1U); + BOOST_CHECK_EQUAL(addrman.size(), 2U); bool newOnly = true; - CAddrInfo addr_ret3 = addrman.Select(newOnly); + auto addr_ret3 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_select) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -248,16 +190,16 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_CHECK_EQUAL(addrman.size(), 1U); bool newOnly = true; - CAddrInfo addr_ret1 = addrman.Select(newOnly); + auto addr_ret1 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); // Test: move addr to tried, select from new expected nothing returned. addrman.Good(CAddress(addr1, NODE_NONE)); BOOST_CHECK_EQUAL(addrman.size(), 1U); - CAddrInfo addr_ret2 = addrman.Select(newOnly); + auto addr_ret2 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); - CAddrInfo addr_ret3 = addrman.Select(); + auto addr_ret3 = addrman.Select().first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); BOOST_CHECK_EQUAL(addrman.size(), 1U); @@ -290,14 +232,14 @@ BOOST_AUTO_TEST_CASE(addrman_select) // Test: Select pulls from new and tried regardless of port number. std::set<uint16_t> ports; for (int i = 0; i < 20; ++i) { - ports.insert(addrman.Select().GetPort()); + ports.insert(addrman.Select().first.GetPort()); } BOOST_CHECK_EQUAL(ports.size(), 3U); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -305,15 +247,15 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) BOOST_CHECK_EQUAL(addrman.size(), num_addrs); - while (num_addrs < 22) { // Magic number! 250.1.1.1 - 250.1.1.22 do not collide with deterministic key = 1 + while (num_addrs < 22) { // Magic number! 250.1.1.1 - 250.1.1.22 do not collide with deterministic key = 1 CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); - //Test: No collision in new table yet. + // Test: No collision in new table yet. BOOST_CHECK_EQUAL(addrman.size(), num_addrs); } - //Test: new table collision! + // Test: new table collision! CService addr1 = ResolveService("250.1.1." + ToString(++num_addrs)); uint32_t collisions{1}; BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); @@ -326,7 +268,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) BOOST_AUTO_TEST_CASE(addrman_tried_collisions) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -334,19 +276,19 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) BOOST_CHECK_EQUAL(addrman.size(), num_addrs); - while (num_addrs < 64) { // Magic number! 250.1.1.1 - 250.1.1.64 do not collide with deterministic key = 1 + while (num_addrs < 64) { // Magic number! 250.1.1.1 - 250.1.1.64 do not collide with deterministic key = 1 CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(CAddress(addr, NODE_NONE)); - //Test: No collision in tried table yet. + // Test: No collision in tried table yet. BOOST_CHECK_EQUAL(addrman.size(), num_addrs); } - //Test: tried table collision! + // Test: tried table collision! CService addr1 = ResolveService("250.1.1." + ToString(++num_addrs)); uint32_t collisions{1}; - BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); + BOOST_CHECK(!addrman.Add({CAddress(addr1, NODE_NONE)}, source)); BOOST_CHECK_EQUAL(addrman.size(), num_addrs - collisions); CService addr2 = ResolveService("250.1.1." + ToString(++num_addrs)); @@ -356,7 +298,7 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) BOOST_AUTO_TEST_CASE(addrman_find) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -368,28 +310,28 @@ BOOST_AUTO_TEST_CASE(addrman_find) CNetAddr source2 = ResolveIP("250.1.2.2"); BOOST_CHECK(addrman.Add({addr1}, source1)); - BOOST_CHECK(!addrman.Add({addr2}, source2)); + BOOST_CHECK(addrman.Add({addr2}, source2)); BOOST_CHECK(addrman.Add({addr3}, source1)); - // Test: ensure Find returns an IP matching what we searched on. - CAddrInfo* info1 = addrman.Find(addr1); + // Test: ensure Find returns an IP/port matching what we searched on. + AddrInfo* info1 = addrman.Find(addr1); BOOST_REQUIRE(info1); BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); - // Test 18; Find does not discriminate by port number. - CAddrInfo* info2 = addrman.Find(addr2); + // Test; Find discriminates by port number. + AddrInfo* info2 = addrman.Find(addr2); BOOST_REQUIRE(info2); - BOOST_CHECK_EQUAL(info2->ToString(), info1->ToString()); + BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:9999"); // Test: Find returns another IP matching what we searched on. - CAddrInfo* info3 = addrman.Find(addr3); + AddrInfo* info3 = addrman.Find(addr3); BOOST_REQUIRE(info3); BOOST_CHECK_EQUAL(info3->ToString(), "251.255.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_create) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -397,19 +339,19 @@ BOOST_AUTO_TEST_CASE(addrman_create) CNetAddr source1 = ResolveIP("250.1.2.1"); int nId; - CAddrInfo* pinfo = addrman.Create(addr1, source1, &nId); + AddrInfo* pinfo = addrman.Create(addr1, source1, &nId); // Test: The result should be the same as the input addr. BOOST_CHECK_EQUAL(pinfo->ToString(), "250.1.2.1:8333"); - CAddrInfo* info2 = addrman.Find(addr1); + AddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_delete) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -423,13 +365,13 @@ BOOST_AUTO_TEST_CASE(addrman_delete) BOOST_CHECK_EQUAL(addrman.size(), 1U); addrman.Delete(nId); BOOST_CHECK_EQUAL(addrman.size(), 0U); - CAddrInfo* info2 = addrman.Find(addr1); + AddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK(info2 == nullptr); } BOOST_AUTO_TEST_CASE(addrman_getaddr) { - CAddrManTest addrman; + AddrManTest addrman; // Test: Sanity check, GetAddr should never return anything if addrman // is empty. @@ -489,7 +431,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -497,7 +439,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) CNetAddr source1 = ResolveIP("250.1.1.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -512,14 +454,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetTriedBucket(nKey1, asmap); @@ -531,7 +473,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) buckets.clear(); for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -544,14 +486,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -567,13 +509,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetNewBucket(nKey1, asmap); @@ -585,7 +527,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) buckets.clear(); for (int j = 0; j < 4 * 255; j++) { - CAddrInfo infoj = CAddrInfo(CAddress( + AddrInfo infoj = AddrInfo(CAddress( ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); @@ -598,7 +540,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -622,7 +564,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) // 101.8.0.0/16 AS8 BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -630,7 +572,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) CNetAddr source1 = ResolveIP("250.1.1.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -645,14 +587,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("101." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("101." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -664,7 +606,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) buckets.clear(); for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -677,14 +619,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -700,13 +642,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetNewBucket(nKey1, asmap); @@ -718,7 +660,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int j = 0; j < 4 * 255; j++) { - CAddrInfo infoj = CAddrInfo(CAddress( + AddrInfo infoj = AddrInfo(CAddress( ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); @@ -731,7 +673,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("101." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -743,7 +685,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -752,16 +694,15 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE // than 1 bucket. BOOST_CHECK(buckets.size() == 1); - } BOOST_AUTO_TEST_CASE(addrman_serialization) { std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - auto addrman_asmap1 = std::make_unique<CAddrManTest>(true, asmap1); - auto addrman_asmap1_dup = std::make_unique<CAddrManTest>(true, asmap1); - auto addrman_noasmap = std::make_unique<CAddrManTest>(); + auto addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); + auto addrman_asmap1_dup = std::make_unique<AddrManTest>(asmap1); + auto addrman_noasmap = std::make_unique<AddrManTest>(); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); @@ -791,8 +732,8 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); // deserializing non-asmaped peers.dat to asmaped addrman - addrman_asmap1 = std::make_unique<CAddrManTest>(true, asmap1); - addrman_noasmap = std::make_unique<CAddrManTest>(); + addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); + addrman_noasmap = std::make_unique<AddrManTest>(); addrman_noasmap->Add({addr}, default_source); stream << *addrman_noasmap; stream >> *addrman_asmap1; @@ -803,8 +744,8 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); // used to map to different buckets, now maps to the same bucket. - addrman_asmap1 = std::make_unique<CAddrManTest>(true, asmap1); - addrman_noasmap = std::make_unique<CAddrManTest>(); + addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); + addrman_noasmap = std::make_unique<AddrManTest>(); CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); addrman_noasmap->Add({addr, addr2}, default_source); @@ -824,7 +765,7 @@ BOOST_AUTO_TEST_CASE(remove_invalid) { // Confirm that invalid addresses are ignored in unserialization. - auto addrman = std::make_unique<CAddrManTest>(); + auto addrman = std::make_unique<AddrManTest>(); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); const CAddress new1{ResolveService("5.5.5.5"), NODE_NONE}; @@ -856,57 +797,56 @@ BOOST_AUTO_TEST_CASE(remove_invalid) BOOST_REQUIRE(pos + sizeof(tried2_raw_replacement) <= stream.size()); memcpy(stream.data() + pos, tried2_raw_replacement, sizeof(tried2_raw_replacement)); - addrman = std::make_unique<CAddrManTest>(); + addrman = std::make_unique<AddrManTest>(); stream >> *addrman; BOOST_CHECK_EQUAL(addrman->size(), 2); } BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK(addrman.size() == 0); // Empty addrman should return blank addrman info. - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // Add twenty two addresses. CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 23; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); // No collisions yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Ensure Good handles duplicates well. for (unsigned int i = 1; i < 23; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); addrman.Good(addr); BOOST_CHECK(addrman.size() == 22); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } - } BOOST_AUTO_TEST_CASE(addrman_noevict) { - CAddrManTest addrman; + AddrManTest addrman; // Add 35 addresses. CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 36; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); // No collision yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Collision between 36 and 19. @@ -915,20 +855,20 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) addrman.Good(addr36); BOOST_CHECK(addrman.size() == 36); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().ToString(), "250.1.1.19:0"); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.19:0"); // 36 should be discarded and 19 not evicted. addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // Lets create two collisions. for (unsigned int i = 37; i < 59; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Cause a collision. @@ -937,37 +877,37 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) addrman.Good(addr59); BOOST_CHECK(addrman.size() == 59); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().ToString(), "250.1.1.10:0"); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.10:0"); // Cause a second collision. BOOST_CHECK(!addrman.Add({CAddress(addr36, NODE_NONE)}, source)); addrman.Good(addr36); BOOST_CHECK(addrman.size() == 59); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() != "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() != "[::]:0"); addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } BOOST_AUTO_TEST_CASE(addrman_evictionworks) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK(addrman.size() == 0); // Empty addrman should return blank addrman info. - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // Add 35 addresses CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 36; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); // No collision yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Collision between 36 and 19. @@ -976,7 +916,7 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) addrman.Good(addr); BOOST_CHECK_EQUAL(addrman.size(), 36); - CAddrInfo info = addrman.SelectTriedCollision(); + auto info = addrman.SelectTriedCollision().first; BOOST_CHECK_EQUAL(info.ToString(), "250.1.1.19:0"); // Ensure test of address fails, so that it is evicted. @@ -984,28 +924,37 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) // Should swap 36 for 19. addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // If 36 was swapped for 19, then this should cause no collisions. BOOST_CHECK(!addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // If we insert 19 it should collide with 36 CService addr19 = ResolveService("250.1.1.19"); BOOST_CHECK(!addrman.Add({CAddress(addr19, NODE_NONE)}, source)); addrman.Good(addr19); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().ToString(), "250.1.1.36:0"); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.36:0"); addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); +} + +static CDataStream AddrmanToStream(const AddrMan& addrman) +{ + CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); + ssPeersIn << Params().MessageStart(); + ssPeersIn << addrman; + return ssPeersIn; } BOOST_AUTO_TEST_CASE(load_addrman) { - CAddrManUncorrupted addrmanUncorrupted; + AddrMan addrman{/*asmap=*/ std::vector<bool>(), /*deterministic=*/ true, + /*consistency_check_ratio=*/ 100}; CService addr1, addr2, addr3; BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); @@ -1018,13 +967,13 @@ BOOST_AUTO_TEST_CASE(load_addrman) CService source; BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false)); std::vector<CAddress> addresses{CAddress(addr1, NODE_NONE), CAddress(addr2, NODE_NONE), CAddress(addr3, NODE_NONE)}; - BOOST_CHECK(addrmanUncorrupted.Add(addresses, source)); - BOOST_CHECK(addrmanUncorrupted.size() == 3); + BOOST_CHECK(addrman.Add(addresses, source)); + BOOST_CHECK(addrman.size() == 3); // Test that the de-serialization does not throw an exception. - CDataStream ssPeers1 = AddrmanToStream(addrmanUncorrupted); + CDataStream ssPeers1 = AddrmanToStream(addrman); bool exceptionThrown = false; - CAddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); BOOST_CHECK(addrman1.size() == 0); try { @@ -1039,23 +988,47 @@ BOOST_AUTO_TEST_CASE(load_addrman) BOOST_CHECK(exceptionThrown == false); // Test that ReadFromStream creates an addrman with the correct number of addrs. - CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); + CDataStream ssPeers2 = AddrmanToStream(addrman); - CAddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); BOOST_CHECK(addrman2.size() == 0); ReadFromStream(addrman2, ssPeers2); BOOST_CHECK(addrman2.size() == 3); } +// Produce a corrupt peers.dat that claims 20 addrs when it only has one addr. +static CDataStream MakeCorruptPeersDat() +{ + CDataStream s(SER_DISK, CLIENT_VERSION); + s << ::Params().MessageStart(); + + unsigned char nVersion = 1; + s << nVersion; + s << ((unsigned char)32); + s << uint256::ONE; + s << 10; // nNew + s << 10; // nTried + + int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); + s << nUBuckets; + + CService serv; + BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); + CAddress addr = CAddress(serv, NODE_NONE); + CNetAddr resolved; + BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); + AddrInfo info = AddrInfo(addr, resolved); + s << info; + + return s; +} BOOST_AUTO_TEST_CASE(load_addrman_corrupted) { - CAddrManCorrupted addrmanCorrupted; - - // Test that the de-serialization of corrupted addrman throws an exception. - CDataStream ssPeers1 = AddrmanToStream(addrmanCorrupted); + // Test that the de-serialization of corrupted peers.dat throws an exception. + CDataStream ssPeers1 = MakeCorruptPeersDat(); bool exceptionThrown = false; - CAddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); BOOST_CHECK(addrman1.size() == 0); try { unsigned char pchMsgTmp[4]; @@ -1069,9 +1042,9 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) BOOST_CHECK(exceptionThrown); // Test that ReadFromStream fails if peers.dat is corrupt - CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); + CDataStream ssPeers2 = MakeCorruptPeersDat(); - CAddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); BOOST_CHECK(addrman2.size() == 0); BOOST_CHECK_THROW(ReadFromStream(addrman2, ssPeers2), std::ios_base::failure); } diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 77b7758a17..aa23d71671 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <limits> @@ -48,13 +48,13 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) BOOST_CHECK_EQUAL(feeRate.GetFee(9e3), CAmount(-9e3)); feeRate = CFeeRate(123); - // Truncates the result, if not integer + // Rounds up the result, if not integer BOOST_CHECK_EQUAL(feeRate.GetFee(0), CAmount(0)); BOOST_CHECK_EQUAL(feeRate.GetFee(8), CAmount(1)); // Special case: returns 1 instead of 0 - BOOST_CHECK_EQUAL(feeRate.GetFee(9), CAmount(1)); - BOOST_CHECK_EQUAL(feeRate.GetFee(121), CAmount(14)); - BOOST_CHECK_EQUAL(feeRate.GetFee(122), CAmount(15)); - BOOST_CHECK_EQUAL(feeRate.GetFee(999), CAmount(122)); + BOOST_CHECK_EQUAL(feeRate.GetFee(9), CAmount(2)); + BOOST_CHECK_EQUAL(feeRate.GetFee(121), CAmount(15)); + BOOST_CHECK_EQUAL(feeRate.GetFee(122), CAmount(16)); + BOOST_CHECK_EQUAL(feeRate.GetFee(999), CAmount(123)); BOOST_CHECK_EQUAL(feeRate.GetFee(1e3), CAmount(123)); BOOST_CHECK_EQUAL(feeRate.GetFee(9e3), CAmount(1107)); diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index a89868e1ef..0fa6b7784f 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -124,7 +124,7 @@ void RunTest(const TestVector &test) { std::vector<unsigned char> seed = ParseHex(test.strHexMaster); CExtKey key; CExtPubKey pubkey; - key.SetSeed(seed.data(), seed.size()); + key.SetSeed(seed); pubkey = key.Neuter(); for (const TestDerivation &derive : test.vDerive) { unsigned char data[74]; diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 5a98558240..fe5ed0a3c8 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <bloom.h> +#include <common/bloom.h> #include <clientversion.h> #include <key.h> @@ -83,7 +83,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) CBloomFilter filter(2, 0.001, 0, BLOOM_UPDATE_ALL); filter.insert(vchPubKey); uint160 hash = pubkey.GetID(); - filter.insert(std::vector<unsigned char>(hash.begin(), hash.end())); + filter.insert(hash); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 64c6d7f634..4d34407ca8 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -18,7 +18,17 @@ #include <utility> #include <vector> -BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) +/** + * Identical to TestingSetup but excludes lock contention logging, as some of + * these tests are designed to be heavily contested to trigger race conditions + * or other issues. + */ +struct NoLockLoggingTestingSetup : public TestingSetup { + NoLockLoggingTestingSetup() + : TestingSetup{CBaseChainParams::MAIN, /*extra_args=*/{"-debugexclude=lock"}} {} +}; + +BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, NoLockLoggingTestingSetup) static const unsigned int QUEUE_BATCH_SIZE = 128; static const int SCRIPT_CHECK_THREADS = 3; diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 5b3b39fdb8..1483bd3cb3 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -694,8 +694,8 @@ BOOST_AUTO_TEST_CASE(chacha20_poly1305_aead_testvector) TestChaCha20Poly1305AEAD(true, 0, /* m */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k1 (payload) */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k2 (AAD) */ "0000000000000000000000000000000000000000000000000000000000000000", + /* k1 (AAD) */ "0000000000000000000000000000000000000000000000000000000000000000", + /* k2 (payload) */ "0000000000000000000000000000000000000000000000000000000000000000", /* AAD keystream */ "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", /* encrypted message & MAC */ "76b8e09f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32d2fc11829c1b6c1df1f551cd6131ff08", /* encrypted message & MAC at sequence 999 */ "b0a03d5bd2855d60699e7d3a3133fa47be740fe4e4c1f967555e2d9271f31c3aaa7aa16ec62c5e24f040c08bb20c3598"); diff --git a/src/test/data/README.md b/src/test/data/README.md index 2463daa42a..a05d9c668b 100644 --- a/src/test/data/README.md +++ b/src/test/data/README.md @@ -8,5 +8,5 @@ License The data files in this directory are distributed under the MIT software license, see the accompanying file COPYING or -http://www.opensource.org/licenses/mit-license.php. +https://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 0bfe6eecd9..668ff150ee 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -52,6 +52,8 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { const CChainParams& chainparams = Params(); auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); + // Disable inactivity checks for this test to avoid interference + static_cast<ConnmanTestMsg*>(connman.get())->SetPeerConnectTimeout(99999); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, false); diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index 526a3c27be..ecb838a7dd 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -11,6 +11,33 @@ BOOST_FIXTURE_TEST_SUITE(fs_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(fsbridge_pathtostring) +{ + std::string u8_str = "fs_tests_₿_🏃"; + BOOST_CHECK_EQUAL(fs::PathToString(fs::PathFromString(u8_str)), u8_str); + BOOST_CHECK_EQUAL(fs::u8path(u8_str).u8string(), u8_str); + BOOST_CHECK_EQUAL(fs::PathFromString(u8_str).u8string(), u8_str); + BOOST_CHECK_EQUAL(fs::PathToString(fs::u8path(u8_str)), u8_str); +#ifndef WIN32 + // On non-windows systems, verify that arbitrary byte strings containing + // invalid UTF-8 can be round tripped successfully with PathToString and + // PathFromString. On non-windows systems, paths are just byte strings so + // these functions do not do any encoding. On windows, paths are Unicode, + // and these functions do encoding and decoding, so the behavior of this + // test would be undefined. + std::string invalid_u8_str = "\xf0"; + BOOST_CHECK_EQUAL(invalid_u8_str.size(), 1); + BOOST_CHECK_EQUAL(fs::PathToString(fs::PathFromString(invalid_u8_str)), invalid_u8_str); +#endif +} + +BOOST_AUTO_TEST_CASE(fsbridge_stem) +{ + std::string test_filename = "fs_tests_₿_🏃.dat"; + std::string expected_stem = "fs_tests_₿_🏃"; + BOOST_CHECK_EQUAL(fs::PathToString(fs::PathFromString(test_filename).stem()), expected_stem); +} + BOOST_AUTO_TEST_CASE(fsbridge_fstream) { fs::path tmpfolder = m_args.GetDataDirBase(); diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index fdbfb3b93b..d427d12a3c 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -4,8 +4,10 @@ #include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> #include <chainparams.h> #include <merkleblock.h> +#include <random.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -23,111 +25,97 @@ void initialize_addrman() SelectParams(CBaseChainParams::REGTEST); } -class CAddrManDeterministic : public CAddrMan +FUZZ_TARGET_INIT(data_stream_addr_man, initialize_addrman) { -public: - FuzzedDataProvider& m_fuzzed_data_provider; - - explicit CAddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) - : CAddrMan(std::move(asmap), /* deterministic */ true, /* consistency_check_ratio */ 0) - , m_fuzzed_data_provider(fuzzed_data_provider) - { - WITH_LOCK(cs, insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); + AddrMan addr_man(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + try { + ReadFromStream(addr_man, data_stream); + } catch (const std::exception&) { } +} - /** - * Generate a random address. Always returns a valid address. - */ - CNetAddr RandAddr() EXCLUSIVE_LOCKS_REQUIRED(cs) - { - CNetAddr addr; - if (m_fuzzed_data_provider.remaining_bytes() > 1 && m_fuzzed_data_provider.ConsumeBool()) { - addr = ConsumeNetAddr(m_fuzzed_data_provider); - } else { - // The networks [1..6] correspond to CNetAddr::BIP155Network (private). - static const std::map<uint8_t, uint8_t> net_len_map = {{1, ADDR_IPV4_SIZE}, - {2, ADDR_IPV6_SIZE}, - {4, ADDR_TORV3_SIZE}, - {5, ADDR_I2P_SIZE}, - {6, ADDR_CJDNS_SIZE}}; - uint8_t net = insecure_rand.randrange(5) + 1; // [1..5] - if (net == 3) { - net = 6; - } - - CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT); +/** + * Generate a random address. Always returns a valid address. + */ +CNetAddr RandAddr(FuzzedDataProvider& fuzzed_data_provider, FastRandomContext& fast_random_context) +{ + CNetAddr addr; + if (fuzzed_data_provider.remaining_bytes() > 1 && fuzzed_data_provider.ConsumeBool()) { + addr = ConsumeNetAddr(fuzzed_data_provider); + } else { + // The networks [1..6] correspond to CNetAddr::BIP155Network (private). + static const std::map<uint8_t, uint8_t> net_len_map = {{1, ADDR_IPV4_SIZE}, + {2, ADDR_IPV6_SIZE}, + {4, ADDR_TORV3_SIZE}, + {5, ADDR_I2P_SIZE}, + {6, ADDR_CJDNS_SIZE}}; + uint8_t net = fast_random_context.randrange(5) + 1; // [1..5] + if (net == 3) { + net = 6; + } - s << net; - s << insecure_rand.randbytes(net_len_map.at(net)); + CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT); - s >> addr; - } + s << net; + s << fast_random_context.randbytes(net_len_map.at(net)); - // Return a dummy IPv4 5.5.5.5 if we generated an invalid address. - if (!addr.IsValid()) { - in_addr v4_addr = {}; - v4_addr.s_addr = 0x05050505; - addr = CNetAddr{v4_addr}; - } + s >> addr; + } - return addr; + // Return a dummy IPv4 5.5.5.5 if we generated an invalid address. + if (!addr.IsValid()) { + in_addr v4_addr = {}; + v4_addr.s_addr = 0x05050505; + addr = CNetAddr{v4_addr}; } - /** - * Fill this addrman with lots of addresses from lots of sources. - */ - void Fill() - { - LOCK(cs); - - // Add some of the addresses directly to the "tried" table. - - // 0, 1, 2, 3 corresponding to 0%, 100%, 50%, 33% - const size_t n = m_fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 3); - - const size_t num_sources = m_fuzzed_data_provider.ConsumeIntegralInRange<size_t>(10, 50); - CNetAddr prev_source; - // Use insecure_rand inside the loops instead of m_fuzzed_data_provider because when - // the latter is exhausted it just returns 0. - for (size_t i = 0; i < num_sources; ++i) { - const auto source = RandAddr(); - const size_t num_addresses = insecure_rand.randrange(500) + 1; // [1..500] - - for (size_t j = 0; j < num_addresses; ++j) { - const auto addr = CAddress{CService{RandAddr(), 8333}, NODE_NETWORK}; - const auto time_penalty = insecure_rand.randrange(100000001); -#if 1 - // 2.83 sec to fill. - if (n > 0 && mapInfo.size() % n == 0 && mapAddr.find(addr) == mapAddr.end()) { - // Add to the "tried" table (if the bucket slot is free). - const CAddrInfo dummy{addr, source}; - const int bucket = dummy.GetTriedBucket(nKey, m_asmap); - const int bucket_pos = dummy.GetBucketPosition(nKey, false, bucket); - if (vvTried[bucket][bucket_pos] == -1) { - int id; - CAddrInfo* addr_info = Create(addr, source, &id); - vvTried[bucket][bucket_pos] = id; - addr_info->fInTried = true; - ++nTried; - } - } else { - // Add to the "new" table. - Add_(addr, source, time_penalty); - } -#else - // 261.91 sec to fill. - Add_(addr, source, time_penalty); - if (n > 0 && mapInfo.size() % n == 0) { - Good_(addr, false, GetTime()); - } -#endif - // Add 10% of the addresses from more than one source. - if (insecure_rand.randrange(10) == 0 && prev_source.IsValid()) { - Add_(addr, prev_source, time_penalty); - } + return addr; +} + +/** Fill addrman with lots of addresses from lots of sources. */ +void FillAddrman(AddrMan& addrman, FuzzedDataProvider& fuzzed_data_provider) +{ + // Add a fraction of the addresses to the "tried" table. + // 0, 1, 2, 3 corresponding to 0%, 100%, 50%, 33% + const size_t n = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 3); + + const size_t num_sources = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 50); + CNetAddr prev_source; + // Generate a FastRandomContext seed to use inside the loops instead of + // fuzzed_data_provider. When fuzzed_data_provider is exhausted it + // just returns 0. + FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; + for (size_t i = 0; i < num_sources; ++i) { + const auto source = RandAddr(fuzzed_data_provider, fast_random_context); + const size_t num_addresses = fast_random_context.randrange(500) + 1; // [1..500] + + for (size_t j = 0; j < num_addresses; ++j) { + const auto addr = CAddress{CService{RandAddr(fuzzed_data_provider, fast_random_context), 8333}, NODE_NETWORK}; + const auto time_penalty = fast_random_context.randrange(100000001); + addrman.Add({addr}, source, time_penalty); + + if (n > 0 && addrman.size() % n == 0) { + addrman.Good(addr, GetTime()); + } + + // Add 10% of the addresses from more than one source. + if (fast_random_context.randrange(10) == 0 && prev_source.IsValid()) { + addrman.Add({addr}, prev_source, time_penalty); } - prev_source = source; } + prev_source = source; + } +} + +class AddrManDeterministic : public AddrMan +{ +public: + explicit AddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) + : AddrMan(std::move(asmap), /* deterministic */ true, /* consistency_check_ratio */ 0) + { + WITH_LOCK(m_impl->cs, m_impl->insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); } /** @@ -137,46 +125,51 @@ public: * - vvNew entries refer to the same addresses * - vvTried entries refer to the same addresses */ - bool operator==(const CAddrManDeterministic& other) + bool operator==(const AddrManDeterministic& other) { - LOCK2(cs, other.cs); + LOCK2(m_impl->cs, other.m_impl->cs); - if (mapInfo.size() != other.mapInfo.size() || nNew != other.nNew || - nTried != other.nTried) { + if (m_impl->mapInfo.size() != other.m_impl->mapInfo.size() || m_impl->nNew != other.m_impl->nNew || + m_impl->nTried != other.m_impl->nTried) { return false; } // Check that all values in `mapInfo` are equal to all values in `other.mapInfo`. // Keys may be different. - using CAddrInfoHasher = std::function<size_t(const CAddrInfo&)>; - using CAddrInfoEq = std::function<bool(const CAddrInfo&, const CAddrInfo&)>; - - CNetAddrHash netaddr_hasher; - - CAddrInfoHasher addrinfo_hasher = [&netaddr_hasher](const CAddrInfo& a) { - return netaddr_hasher(static_cast<CNetAddr>(a)) ^ netaddr_hasher(a.source) ^ - a.nLastSuccess ^ a.nAttempts ^ a.nRefCount ^ a.fInTried; + auto addrinfo_hasher = [](const AddrInfo& a) { + CSipHasher hasher(0, 0); + auto addr_key = a.GetKey(); + auto source_key = a.source.GetAddrBytes(); + hasher.Write(a.nLastSuccess); + hasher.Write(a.nAttempts); + hasher.Write(a.nRefCount); + hasher.Write(a.fInTried); + hasher.Write(a.GetNetwork()); + hasher.Write(a.source.GetNetwork()); + hasher.Write(addr_key.size()); + hasher.Write(source_key.size()); + hasher.Write(addr_key.data(), addr_key.size()); + hasher.Write(source_key.data(), source_key.size()); + return (size_t)hasher.Finalize(); }; - CAddrInfoEq addrinfo_eq = [](const CAddrInfo& lhs, const CAddrInfo& rhs) { - return static_cast<CNetAddr>(lhs) == static_cast<CNetAddr>(rhs) && - lhs.source == rhs.source && lhs.nLastSuccess == rhs.nLastSuccess && - lhs.nAttempts == rhs.nAttempts && lhs.nRefCount == rhs.nRefCount && - lhs.fInTried == rhs.fInTried; + auto addrinfo_eq = [](const AddrInfo& lhs, const AddrInfo& rhs) { + return std::tie(static_cast<const CService&>(lhs), lhs.source, lhs.nLastSuccess, lhs.nAttempts, lhs.nRefCount, lhs.fInTried) == + std::tie(static_cast<const CService&>(rhs), rhs.source, rhs.nLastSuccess, rhs.nAttempts, rhs.nRefCount, rhs.fInTried); }; - using Addresses = std::unordered_set<CAddrInfo, CAddrInfoHasher, CAddrInfoEq>; + using Addresses = std::unordered_set<AddrInfo, decltype(addrinfo_hasher), decltype(addrinfo_eq)>; - const size_t num_addresses{mapInfo.size()}; + const size_t num_addresses{m_impl->mapInfo.size()}; Addresses addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; - for (const auto& [id, addr] : mapInfo) { + for (const auto& [id, addr] : m_impl->mapInfo) { addresses.insert(addr); } Addresses other_addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; - for (const auto& [id, addr] : other.mapInfo) { + for (const auto& [id, addr] : other.m_impl->mapInfo) { other_addresses.insert(addr); } @@ -184,14 +177,14 @@ public: return false; } - auto IdsReferToSameAddress = [&](int id, int other_id) EXCLUSIVE_LOCKS_REQUIRED(cs, other.cs) { + auto IdsReferToSameAddress = [&](int id, int other_id) EXCLUSIVE_LOCKS_REQUIRED(m_impl->cs, other.m_impl->cs) { if (id == -1 && other_id == -1) { return true; } if ((id == -1 && other_id != -1) || (id != -1 && other_id == -1)) { return false; } - return mapInfo.at(id) == other.mapInfo.at(other_id); + return m_impl->mapInfo.at(id) == other.m_impl->mapInfo.at(other_id); }; // Check that `vvNew` contains the same addresses as `other.vvNew`. Notice - `vvNew[i][j]` @@ -199,7 +192,7 @@ public: // themselves may differ between `vvNew` and `other.vvNew`. for (size_t i = 0; i < ADDRMAN_NEW_BUCKET_COUNT; ++i) { for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { - if (!IdsReferToSameAddress(vvNew[i][j], other.vvNew[i][j])) { + if (!IdsReferToSameAddress(m_impl->vvNew[i][j], other.m_impl->vvNew[i][j])) { return false; } } @@ -208,7 +201,7 @@ public: // Same for `vvTried`. for (size_t i = 0; i < ADDRMAN_TRIED_BUCKET_COUNT; ++i) { for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { - if (!IdsReferToSameAddress(vvTried[i][j], other.vvTried[i][j])) { + if (!IdsReferToSameAddress(m_impl->vvTried[i][j], other.m_impl->vvTried[i][j])) { return false; } } @@ -230,7 +223,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); - auto addr_man_ptr = std::make_unique<CAddrManDeterministic>(asmap, fuzzed_data_provider); + auto addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); if (fuzzed_data_provider.ConsumeBool()) { const std::vector<uint8_t> serialized_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; CDataStream ds(serialized_data, SER_DISK, INIT_PROTO_VERSION); @@ -239,11 +232,11 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) try { ds >> *addr_man_ptr; } catch (const std::ios_base::failure&) { - addr_man_ptr = std::make_unique<CAddrManDeterministic>(asmap, fuzzed_data_provider); + addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); } } - CAddrManDeterministic& addr_man = *addr_man_ptr; - while (fuzzed_data_provider.ConsumeBool()) { + AddrManDeterministic& addr_man = *addr_man_ptr; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { @@ -254,7 +247,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) }, [&] { std::vector<CAddress> addresses; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<CAddress> opt_address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); if (!opt_address) { break; @@ -291,7 +284,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) } }); } - const CAddrMan& const_addr_man{addr_man}; + const AddrMan& const_addr_man{addr_man}; (void)const_addr_man.GetAddr( /* max_addresses */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), /* max_pct */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), @@ -309,12 +302,12 @@ FUZZ_TARGET_INIT(addrman_serdeser, initialize_addrman) SetMockTime(ConsumeTime(fuzzed_data_provider)); std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); - CAddrManDeterministic addr_man1{asmap, fuzzed_data_provider}; - CAddrManDeterministic addr_man2{asmap, fuzzed_data_provider}; + AddrManDeterministic addr_man1{asmap, fuzzed_data_provider}; + AddrManDeterministic addr_man2{asmap, fuzzed_data_provider}; CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); - addr_man1.Fill(); + FillAddrman(addr_man1, fuzzed_data_provider); data_stream << addr_man1; data_stream >> addr_man2; assert(addr_man1 == addr_man2); diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index 479342e4be..0cc2d12d29 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -19,7 +19,7 @@ FUZZ_TARGET(autofile) FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); CAutoFile auto_file = fuzzed_auto_file_provider.open(); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index 561cc83c72..fbba25c404 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -48,7 +48,7 @@ FUZZ_TARGET_INIT(banman, initialize_banman) const bool start_with_corrupted_banlist{fuzzed_data_provider.ConsumeBool()}; bool force_read_and_write_to_err{false}; if (start_with_corrupted_banlist) { - assert(WriteBinaryFile(banlist_file.string() + ".json", + assert(WriteBinaryFile(banlist_file + ".json", fuzzed_data_provider.ConsumeRandomLengthString())); } else { force_read_and_write_to_err = fuzzed_data_provider.ConsumeBool(); @@ -111,5 +111,5 @@ FUZZ_TARGET_INIT(banman, initialize_banman) assert(banmap == banmap_read); } } - fs::remove(banlist_file.string() + ".json"); + fs::remove(fs::PathToString(banlist_file + ".json")); } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index c5bb8744a4..3e303ecc0f 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <bloom.h> +#include <common/bloom.h> #include <primitives/transaction.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -24,7 +24,7 @@ FUZZ_TARGET(bloom_filter) 1.0 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max()), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index ed72260d10..c3c2e4050f 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -29,7 +29,7 @@ FUZZ_TARGET(buffered_file) } if (opt_buffered_file && fuzzed_file != nullptr) { bool setpos_fail = false; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/chain.cpp b/src/test/fuzz/chain.cpp index 9f7074b423..0e12a55408 100644 --- a/src/test/fuzz/chain.cpp +++ b/src/test/fuzz/chain.cpp @@ -35,7 +35,7 @@ FUZZ_TARGET(chain) (void)CDiskBlockIndex{*disk_block_index}; (void)disk_block_index->BuildSkip(); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const BlockStatus block_status = fuzzed_data_provider.PickValueInArray({ BlockStatus::BLOCK_VALID_UNKNOWN, BlockStatus::BLOCK_VALID_RESERVED, diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index bbdb2c6917..325a9a170e 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <chainparams.h> #include <chainparamsbase.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> @@ -51,7 +51,7 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) COutPoint random_out_point; Coin random_coin; CMutableTransaction random_mutable_transaction; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { @@ -114,7 +114,7 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) }, [&] { CCoinsMap coins_map; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CCoinsCacheEntry coins_cache_entry; coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); if (fuzzed_data_provider.ConsumeBool()) { diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 01741103e4..9e4718e603 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -25,13 +25,13 @@ FUZZ_TARGET_INIT(connman, initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - CAddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), addrman, fuzzed_data_provider.ConsumeBool()}; CNetAddr random_netaddr; CNode random_node = ConsumeNode(fuzzed_data_provider); CSubNet random_subnet; std::string random_string; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/crypto_aes256.cpp b/src/test/fuzz/crypto_aes256.cpp index ccabd1f7dc..0937026fdd 100644 --- a/src/test/fuzz/crypto_aes256.cpp +++ b/src/test/fuzz/crypto_aes256.cpp @@ -19,7 +19,7 @@ FUZZ_TARGET(crypto_aes256) AES256Encrypt encrypt{key.data()}; AES256Decrypt decrypt{key.data()}; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::vector<uint8_t> plaintext = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES_BLOCKSIZE); std::vector<uint8_t> ciphertext(AES_BLOCKSIZE); encrypt.Encrypt(ciphertext.data(), plaintext.data()); diff --git a/src/test/fuzz/crypto_aes256cbc.cpp b/src/test/fuzz/crypto_aes256cbc.cpp index 6d4138e546..5fe67bd4da 100644 --- a/src/test/fuzz/crypto_aes256cbc.cpp +++ b/src/test/fuzz/crypto_aes256cbc.cpp @@ -21,7 +21,7 @@ FUZZ_TARGET(crypto_aes256cbc) AES256CBCEncrypt encrypt{key.data(), iv.data(), pad}; AES256CBCDecrypt decrypt{key.data(), iv.data(), pad}; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::vector<uint8_t> plaintext = ConsumeRandomLengthByteVector(fuzzed_data_provider); std::vector<uint8_t> ciphertext(plaintext.size() + AES_BLOCKSIZE); const int encrypt_ret = encrypt.Encrypt(plaintext.data(), plaintext.size(), ciphertext.data()); diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp index 8adfa92420..3f552a8cda 100644 --- a/src/test/fuzz/crypto_chacha20.cpp +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -19,7 +19,7 @@ FUZZ_TARGET(crypto_chacha20) const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(16, 32)); chacha20 = ChaCha20{key.data(), key.size()}; } - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp index bb4ef22158..5e60b0f25b 100644 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -28,7 +28,7 @@ FUZZ_TARGET(crypto_chacha20_poly1305_aead) std::vector<uint8_t> in(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); std::vector<uint8_t> out(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); bool is_encrypt = fuzzed_data_provider.ConsumeBool(); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp index 8cb9c55283..24bcc03dfd 100644 --- a/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp +++ b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp @@ -18,7 +18,7 @@ FUZZ_TARGET(crypto_hkdf_hmac_sha256_l32) const std::vector<uint8_t> initial_key_material = ConsumeRandomLengthByteVector(fuzzed_data_provider); CHKDF_HMAC_SHA256_L32 hkdf_hmac_sha256_l32(initial_key_material.data(), initial_key_material.size(), fuzzed_data_provider.ConsumeRandomLengthString(1024)); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { std::vector<uint8_t> out(32); hkdf_hmac_sha256_l32.Expand32(fuzzed_data_provider.ConsumeRandomLengthString(128), out.data()); } diff --git a/src/test/fuzz/cuckoocache.cpp b/src/test/fuzz/cuckoocache.cpp index a522c837ef..019afe1c47 100644 --- a/src/test/fuzz/cuckoocache.cpp +++ b/src/test/fuzz/cuckoocache.cpp @@ -37,7 +37,7 @@ FUZZ_TARGET(cuckoocache) } else { cuckoo_cache.setup(fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, 4096)); } - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { if (fuzzed_data_provider.ConsumeBool()) { cuckoo_cache.insert(fuzzed_data_provider.ConsumeBool()); } else { diff --git a/src/test/fuzz/data_stream.cpp b/src/test/fuzz/data_stream.cpp deleted file mode 100644 index 323090e041..0000000000 --- a/src/test/fuzz/data_stream.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2020-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <addrdb.h> -#include <addrman.h> -#include <net.h> -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> -#include <test/util/setup_common.h> - -#include <cstdint> -#include <vector> - -void initialize_data_stream_addr_man() -{ - static const auto testing_setup = MakeNoLogFileContext<>(); -} - -FUZZ_TARGET_INIT(data_stream_addr_man, initialize_data_stream_addr_man) -{ - FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); - CAddrMan addr_man(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); - try { - ReadFromStream(addr_man, data_stream); - } catch (const std::exception&) { - } -} diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 83ae1680e3..a9325fa738 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -4,6 +4,7 @@ #include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> #include <blockencodings.h> #include <blockfilter.h> #include <chain.h> @@ -104,7 +105,7 @@ FUZZ_TARGET_DESERIALIZE(block_filter_deserialize, { DeserializeFromFuzzingInput(buffer, block_filter); }) FUZZ_TARGET_DESERIALIZE(addr_info_deserialize, { - CAddrInfo addr_info; + AddrInfo addr_info; DeserializeFromFuzzingInput(buffer, addr_info); }) FUZZ_TARGET_DESERIALIZE(block_file_info_deserialize, { @@ -188,7 +189,7 @@ FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { BlockMerkleRoot(block, &mutated); }) FUZZ_TARGET_DESERIALIZE(addrman_deserialize, { - CAddrMan am(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan am(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); DeserializeFromFuzzingInput(buffer, am); }) FUZZ_TARGET_DESERIALIZE(blockheader_deserialize, { diff --git a/src/test/fuzz/fee_rate.cpp b/src/test/fuzz/fee_rate.cpp index dff0e58000..a852f8fb60 100644 --- a/src/test/fuzz/fee_rate.cpp +++ b/src/test/fuzz/fee_rate.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index 61c7681bf9..bcab66842c 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/fees.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -18,7 +18,7 @@ FUZZ_TARGET(fees) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CFeeRate minimal_incremental_fee{ConsumeMoney(fuzzed_data_provider)}; FeeFilterRounder fee_filter_rounder{minimal_incremental_fee}; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const CAmount current_minimum_fee = ConsumeMoney(fuzzed_data_provider); const CAmount rounded_fee = fee_filter_rounder.round(current_minimum_fee); assert(MoneyRange(rounded_fee)); diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 5a732aeeff..b6c40809e3 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <arith_uint256.h> #include <compressor.h> +#include <consensus/amount.h> #include <consensus/merkle.h> #include <core_io.h> #include <crypto/common.h> diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 5b1acae57b..4ad8123554 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -50,8 +50,6 @@ FUZZ_TARGET(locale) const bool parseint32_without_locale = ParseInt32(random_string, &parseint32_out_without_locale); int64_t parseint64_out_without_locale; const bool parseint64_without_locale = ParseInt64(random_string, &parseint64_out_without_locale); - const int64_t atoi64_without_locale = atoi64(random_string); - const int atoi_without_locale = atoi(random_string); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const std::string tostring_without_locale = ToString(random_int64); // The variable `random_int32` is no longer used, but the harness still needs to @@ -77,10 +75,6 @@ FUZZ_TARGET(locale) if (parseint64_without_locale) { assert(parseint64_out_without_locale == parseint64_out_with_locale); } - const int64_t atoi64_with_locale = atoi64(random_string); - assert(atoi64_without_locale == atoi64_with_locale); - const int atoi_with_locale = atoi(random_string); - assert(atoi_without_locale == atoi_with_locale); const std::string tostring_with_locale = ToString(random_int64); assert(tostring_without_locale == tostring_with_locale); const std::string strprintf_int_with_locale = strprintf("%d", random_int64); diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index 1eefd4c521..6271367a9c 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -34,7 +34,7 @@ FUZZ_TARGET(merkleblock) if (fuzzed_data_provider.ConsumeBool()) { merkle_block = CMerkleBlock{*opt_block, bloom_filter}; } else if (fuzzed_data_provider.ConsumeBool()) { - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { txids.insert(ConsumeUInt256(fuzzed_data_provider)); } merkle_block = CMerkleBlock{*opt_block, txids}; diff --git a/src/test/fuzz/muhash.cpp b/src/test/fuzz/muhash.cpp index 4ea9511870..8304e6fdb8 100644 --- a/src/test/fuzz/muhash.cpp +++ b/src/test/fuzz/muhash.cpp @@ -12,52 +12,47 @@ FUZZ_TARGET(muhash) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - std::vector<uint8_t> data = ConsumeRandomLengthByteVector(fuzzed_data_provider); - std::vector<uint8_t> data2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); - if (data.empty()) { - data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); - } - if (data2.empty()) { - data2.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); - } - - data = ConsumeRandomLengthByteVector(fuzzed_data_provider); - data2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); + std::vector<uint8_t> data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; + std::vector<uint8_t> data2{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; MuHash3072 muhash; - // Test that MuHash result is consistent independent of order of operations muhash.Insert(data); muhash.Insert(data2); + const std::string initial_state_hash{"dd5ad2a105c2d29495f577245c357409002329b9f4d6182c0af3dc2f462555c8"}; uint256 out; - muhash.Finalize(out); - - muhash = MuHash3072(); - muhash.Insert(data2); - muhash.Insert(data); - uint256 out2; - muhash.Finalize(out2); - + CallOneOf( + fuzzed_data_provider, + [&] { + // Test that MuHash result is consistent independent of order of operations + muhash.Finalize(out); + + muhash = MuHash3072(); + muhash.Insert(data2); + muhash.Insert(data); + muhash.Finalize(out2); + }, + [&] { + // Test that multiplication with the initial state never changes the finalized result + muhash.Finalize(out); + MuHash3072 muhash3; + muhash3 *= muhash; + muhash3.Finalize(out2); + }, + [&] { + // Test that dividing a MuHash by itself brings it back to it's initial state + muhash /= muhash; + muhash.Finalize(out); + out2 = uint256S(initial_state_hash); + }, + [&] { + // Test that removing all added elements brings the object back to it's initial state + muhash.Remove(data); + muhash.Remove(data2); + muhash.Finalize(out); + out2 = uint256S(initial_state_hash); + }); assert(out == out2); - MuHash3072 muhash3; - muhash3 *= muhash; - uint256 out3; - muhash3.Finalize(out3); - assert(out == out3); - - // Test that removing all added elements brings the object back to it's initial state - muhash /= muhash; - muhash.Finalize(out); - - MuHash3072 muhash2; - muhash2.Finalize(out2); - - assert(out == out2); - - muhash3.Remove(data); - muhash3.Remove(data2); - muhash3.Finalize(out3); - assert(out == out3); } diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index bd1bb79d0e..fb11ea36ce 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -32,7 +32,7 @@ FUZZ_TARGET_INIT(net, initialize_net) SetMockTime(ConsumeTime(fuzzed_data_provider)); CNode node{ConsumeNode(fuzzed_data_provider)}; node.SetCommonVersion(fuzzed_data_provider.ConsumeIntegral<int>()); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/netbase_dns_lookup.cpp b/src/test/fuzz/netbase_dns_lookup.cpp index cf2fa33744..d01d413cff 100644 --- a/src/test/fuzz/netbase_dns_lookup.cpp +++ b/src/test/fuzz/netbase_dns_lookup.cpp @@ -22,7 +22,7 @@ FUZZ_TARGET(netbase_dns_lookup) auto fuzzed_dns_lookup_function = [&](const std::string&, bool) { std::vector<CNetAddr> resolved_addresses; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { resolved_addresses.push_back(ConsumeNetAddr(fuzzed_data_provider)); } return resolved_addresses; diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index a3f71426fa..2e3b51e753 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -18,7 +18,7 @@ FUZZ_TARGET(node_eviction) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; std::vector<NodeEvictionCandidate> eviction_candidates; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { eviction_candidates.push_back({ /* id */ fuzzed_data_provider.ConsumeIntegral<NodeId>(), /* nTimeConnected */ fuzzed_data_provider.ConsumeIntegral<int64_t>(), diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index edee5aeef7..29b7223c90 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -68,18 +68,16 @@ FUZZ_TARGET_INIT(p2p_transport_serialization, initialize_p2p_transport_serializa } if (deserializer.Complete()) { const std::chrono::microseconds m_time{std::numeric_limits<int64_t>::max()}; - uint32_t out_err_raw_size{0}; - std::optional<CNetMessage> result{deserializer.GetMessage(m_time, out_err_raw_size)}; - if (result) { - assert(result->m_command.size() <= CMessageHeader::COMMAND_SIZE); - assert(result->m_raw_message_size <= mutable_msg_bytes.size()); - assert(result->m_raw_message_size == CMessageHeader::HEADER_SIZE + result->m_message_size); - assert(result->m_time == m_time); + bool reject_message{false}; + CNetMessage msg = deserializer.GetMessage(m_time, reject_message); + assert(msg.m_command.size() <= CMessageHeader::COMMAND_SIZE); + assert(msg.m_raw_message_size <= mutable_msg_bytes.size()); + assert(msg.m_raw_message_size == CMessageHeader::HEADER_SIZE + msg.m_message_size); + assert(msg.m_time == m_time); - std::vector<unsigned char> header; - auto msg = CNetMsgMaker{result->m_recv.GetVersion()}.Make(result->m_command, MakeUCharSpan(result->m_recv)); - serializer.prepareForTransport(msg, header); - } + std::vector<unsigned char> header; + auto msg2 = CNetMsgMaker{msg.m_recv.GetVersion()}.Make(msg.m_command, MakeUCharSpan(msg.m_recv)); + serializer.prepareForTransport(msg2, header); } } } diff --git a/src/test/fuzz/parse_numbers.cpp b/src/test/fuzz/parse_numbers.cpp index 69e58c3f63..85fee062f0 100644 --- a/src/test/fuzz/parse_numbers.cpp +++ b/src/test/fuzz/parse_numbers.cpp @@ -14,9 +14,6 @@ FUZZ_TARGET(parse_numbers) (void)ParseMoney(random_string); - double d; - (void)ParseDouble(random_string, &d); - uint8_t u8; (void)ParseUInt8(random_string, &u8); @@ -25,13 +22,13 @@ FUZZ_TARGET(parse_numbers) int32_t i32; (void)ParseInt32(random_string, &i32); - (void)atoi(random_string); + (void)LocaleIndependentAtoi<int>(random_string); uint32_t u32; (void)ParseUInt32(random_string, &u32); int64_t i64; - (void)atoi64(random_string); + (void)LocaleIndependentAtoi<int64_t>(random_string); (void)ParseFixedPoint(random_string, 3, &i64); (void)ParseInt64(random_string, &i64); diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 116b7a71d9..63dc4ce1d9 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -24,7 +24,7 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CBlockPolicyEstimator block_policy_estimator; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { @@ -40,7 +40,7 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) }, [&] { std::vector<CTxMemPoolEntry> mempool_entries; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { break; diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 47b4323e81..1123c8c170 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -27,7 +27,7 @@ FUZZ_TARGET_INIT(pow, initialize_pow) std::vector<CBlockIndex> blocks; const uint32_t fixed_time = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const uint32_t fixed_bits = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { const std::optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!block_header) { continue; diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 11b236c9bd..91ec2aafde 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -55,7 +55,7 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) connman.AddTestNode(p2p_node); } - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; const auto mock_time = ConsumeTime(fuzzed_data_provider); diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index 26c89a70c3..990bce5f6c 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -24,7 +24,7 @@ FUZZ_TARGET(rbf) return; } CTxMemPool pool; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!another_mtx) { break; diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index b9ed497e68..9c18ad49cb 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <bloom.h> +#include <common/bloom.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 9195cc4873..251687104e 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -294,7 +294,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) { std::vector<std::string> scalar_arguments; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); } return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; @@ -348,7 +348,7 @@ FUZZ_TARGET_INIT(rpc, initialize_rpc) return; } std::vector<std::string> arguments; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); } try { diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 950ee45d1d..0979967384 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -41,9 +41,7 @@ void initialize_script() FUZZ_TARGET_INIT(script, initialize_script) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const std::optional<CScript> script_opt = ConsumeDeserializable<CScript>(fuzzed_data_provider); - if (!script_opt) return; - const CScript script{*script_opt}; + const CScript script{ConsumeScript(fuzzed_data_provider)}; CompressedScript compressed; if (CompressScript(script, compressed)) { @@ -56,46 +54,8 @@ FUZZ_TARGET_INIT(script, initialize_script) assert(script == decompressed_script); } - CTxDestination address; - TxoutType type_ret; - std::vector<CTxDestination> addresses; - int required_ret; - bool extract_destinations_ret = ExtractDestinations(script, type_ret, addresses, required_ret); - bool extract_destination_ret = ExtractDestination(script, address); - if (!extract_destinations_ret) { - assert(!extract_destination_ret); - if (type_ret == TxoutType::MULTISIG) { - assert(addresses.empty() && required_ret == 0); - } else { - assert(type_ret == TxoutType::PUBKEY || - type_ret == TxoutType::NONSTANDARD || - type_ret == TxoutType::NULL_DATA); - } - } else { - assert(required_ret >= 1 && required_ret <= 16); - assert((unsigned long)required_ret == addresses.size()); - assert(type_ret == TxoutType::MULTISIG || required_ret == 1); - } - if (type_ret == TxoutType::NONSTANDARD || type_ret == TxoutType::NULL_DATA) { - assert(!extract_destinations_ret); - } - if (!extract_destination_ret) { - assert(type_ret == TxoutType::PUBKEY || - type_ret == TxoutType::NONSTANDARD || - type_ret == TxoutType::NULL_DATA || - type_ret == TxoutType::MULTISIG); - } else { - assert(address == addresses[0]); - } - if (type_ret == TxoutType::NONSTANDARD || - type_ret == TxoutType::NULL_DATA || - type_ret == TxoutType::MULTISIG) { - assert(!extract_destination_ret); - } - TxoutType which_type; bool is_standard_ret = IsStandard(script, which_type); - assert(type_ret == which_type); if (!is_standard_ret) { assert(which_type == TxoutType::NONSTANDARD || which_type == TxoutType::NULL_DATA || @@ -112,6 +72,20 @@ FUZZ_TARGET_INIT(script, initialize_script) which_type == TxoutType::NONSTANDARD); } + CTxDestination address; + bool extract_destination_ret = ExtractDestination(script, address); + if (!extract_destination_ret) { + assert(which_type == TxoutType::PUBKEY || + which_type == TxoutType::NONSTANDARD || + which_type == TxoutType::NULL_DATA || + which_type == TxoutType::MULTISIG); + } + if (which_type == TxoutType::NONSTANDARD || + which_type == TxoutType::NULL_DATA || + which_type == TxoutType::MULTISIG) { + assert(!extract_destination_ret); + } + const FlatSigningProvider signing_provider; (void)InferDescriptor(script, signing_provider); (void)IsSegWitOutput(signing_provider, script); @@ -133,15 +107,11 @@ FUZZ_TARGET_INIT(script, initialize_script) (void)ScriptToAsmStr(script, true); UniValue o1(UniValue::VOBJ); - ScriptPubKeyToUniv(script, o1, true, true); - ScriptPubKeyToUniv(script, o1, true, false); + ScriptPubKeyToUniv(script, o1, true); UniValue o2(UniValue::VOBJ); - ScriptPubKeyToUniv(script, o2, false, true); - ScriptPubKeyToUniv(script, o2, false, false); + ScriptPubKeyToUniv(script, o2, false); UniValue o3(UniValue::VOBJ); - ScriptToUniv(script, o3, true); - UniValue o4(UniValue::VOBJ); - ScriptToUniv(script, o4, false); + ScriptToUniv(script, o3); { const std::vector<uint8_t> bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); diff --git a/src/test/fuzz/script_descriptor_cache.cpp b/src/test/fuzz/script_descriptor_cache.cpp index 6ce13d5679..a90ad5e8ed 100644 --- a/src/test/fuzz/script_descriptor_cache.cpp +++ b/src/test/fuzz/script_descriptor_cache.cpp @@ -17,7 +17,7 @@ FUZZ_TARGET(script_descriptor_cache) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); DescriptorCache descriptor_cache; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::vector<uint8_t> code = fuzzed_data_provider.ConsumeBytes<uint8_t>(BIP32_EXTKEY_SIZE); if (code.size() == BIP32_EXTKEY_SIZE) { CExtPubKey xpub; diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 1278dc87d4..43927772ae 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <pubkey.h> #include <script/interpreter.h> #include <streams.h> diff --git a/src/test/fuzz/script_ops.cpp b/src/test/fuzz/script_ops.cpp index 4bc709ed35..12247679f2 100644 --- a/src/test/fuzz/script_ops.cpp +++ b/src/test/fuzz/script_ops.cpp @@ -15,7 +15,7 @@ FUZZ_TARGET(script_ops) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CScript script_mut = ConsumeScript(fuzzed_data_provider); - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 1000000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index 684324c36e..79380bd9c9 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -48,7 +48,7 @@ FUZZ_TARGET_INIT(script_sign, initialize_script_sign) { std::map<CPubKey, KeyOriginInfo> hd_keypaths; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<CPubKey> pub_key = ConsumeDeserializable<CPubKey>(fuzzed_data_provider); if (!pub_key) { break; @@ -125,7 +125,7 @@ FUZZ_TARGET_INIT(script_sign, initialize_script_sign) (void)signature_creator.CreateSig(provider, vch_sig, address, ConsumeScript(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0})); } std::map<COutPoint, Coin> coins; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<COutPoint> outpoint = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!outpoint) { break; diff --git a/src/test/fuzz/scriptnum_ops.cpp b/src/test/fuzz/scriptnum_ops.cpp index 62ed50d13f..0681aaf949 100644 --- a/src/test/fuzz/scriptnum_ops.cpp +++ b/src/test/fuzz/scriptnum_ops.cpp @@ -28,7 +28,7 @@ FUZZ_TARGET(scriptnum_ops) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CScriptNum script_num = ConsumeScriptNum(fuzzed_data_provider); - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 1000000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index 6b86c8889d..c3f416632d 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -58,8 +58,8 @@ FUZZ_TARGET_INIT(signature_checker, initialize_signature_checker) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const SigVersion sig_version = fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0}); - const auto script_1 = ConsumeScript(fuzzed_data_provider, 65536); - const auto script_2 = ConsumeScript(fuzzed_data_provider, 65536); + const auto script_1{ConsumeScript(fuzzed_data_provider)}; + const auto script_2{ConsumeScript(fuzzed_data_provider)}; std::vector<std::vector<unsigned char>> stack; (void)EvalScript(stack, script_1, flags, FuzzedSignatureChecker(fuzzed_data_provider), sig_version, nullptr); if (!IsValidFlagCombination(flags)) { diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 0c1b45b86c..ab646c68fc 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -31,9 +31,105 @@ #include <version.h> #include <cstdint> +#include <cstdlib> #include <string> #include <vector> +namespace { +bool LegacyParsePrechecks(const std::string& str) +{ + if (str.empty()) // No empty string allowed + return false; + if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size() - 1]))) // No padding allowed + return false; + if (!ValidAsCString(str)) // No embedded NUL characters allowed + return false; + return true; +} + +bool LegacyParseInt32(const std::string& str, int32_t* out) +{ + if (!LegacyParsePrechecks(str)) + return false; + char* endp = nullptr; + errno = 0; // strtol will not set errno if valid + long int n = strtol(str.c_str(), &endp, 10); + if (out) *out = (int32_t)n; + // Note that strtol returns a *long int*, so even if strtol doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit + // platforms the size of these types may be different. + return endp && *endp == 0 && !errno && + n >= std::numeric_limits<int32_t>::min() && + n <= std::numeric_limits<int32_t>::max(); +} + +bool LegacyParseInt64(const std::string& str, int64_t* out) +{ + if (!LegacyParsePrechecks(str)) + return false; + char* endp = nullptr; + errno = 0; // strtoll will not set errno if valid + long long int n = strtoll(str.c_str(), &endp, 10); + if (out) *out = (int64_t)n; + // Note that strtoll returns a *long long int*, so even if strtol doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *int64_t*. + return endp && *endp == 0 && !errno && + n >= std::numeric_limits<int64_t>::min() && + n <= std::numeric_limits<int64_t>::max(); +} + +bool LegacyParseUInt32(const std::string& str, uint32_t* out) +{ + if (!LegacyParsePrechecks(str)) + return false; + if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoul accepts these by default if they fit in the range + return false; + char* endp = nullptr; + errno = 0; // strtoul will not set errno if valid + unsigned long int n = strtoul(str.c_str(), &endp, 10); + if (out) *out = (uint32_t)n; + // Note that strtoul returns a *unsigned long int*, so even if it doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *uint32_t*. On 64-bit + // platforms the size of these types may be different. + return endp && *endp == 0 && !errno && + n <= std::numeric_limits<uint32_t>::max(); +} + +bool LegacyParseUInt8(const std::string& str, uint8_t* out) +{ + uint32_t u32; + if (!LegacyParseUInt32(str, &u32) || u32 > std::numeric_limits<uint8_t>::max()) { + return false; + } + if (out != nullptr) { + *out = static_cast<uint8_t>(u32); + } + return true; +} + +bool LegacyParseUInt64(const std::string& str, uint64_t* out) +{ + if (!LegacyParsePrechecks(str)) + return false; + if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoull accepts these by default if they fit in the range + return false; + char* endp = nullptr; + errno = 0; // strtoull will not set errno if valid + unsigned long long int n = strtoull(str.c_str(), &endp, 10); + if (out) *out = (uint64_t)n; + // Note that strtoull returns a *unsigned long long int*, so even if it doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *uint64_t*. + return endp && *endp == 0 && !errno && + n <= std::numeric_limits<uint64_t>::max(); +} + +// For backwards compatibility checking. +int64_t atoi64_legacy(const std::string& str) +{ + return strtoll(str.c_str(), nullptr, 10); +} +}; // namespace + FUZZ_TARGET(string) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -133,4 +229,67 @@ FUZZ_TARGET(string) const bilingual_str bs2{random_string_2, random_string_1}; (void)(bs1 + bs2); } + { + int32_t i32; + int64_t i64; + uint32_t u32; + uint64_t u64; + uint8_t u8; + const bool ok_i32 = ParseInt32(random_string_1, &i32); + const bool ok_i64 = ParseInt64(random_string_1, &i64); + const bool ok_u32 = ParseUInt32(random_string_1, &u32); + const bool ok_u64 = ParseUInt64(random_string_1, &u64); + const bool ok_u8 = ParseUInt8(random_string_1, &u8); + + int32_t i32_legacy; + int64_t i64_legacy; + uint32_t u32_legacy; + uint64_t u64_legacy; + uint8_t u8_legacy; + const bool ok_i32_legacy = LegacyParseInt32(random_string_1, &i32_legacy); + const bool ok_i64_legacy = LegacyParseInt64(random_string_1, &i64_legacy); + const bool ok_u32_legacy = LegacyParseUInt32(random_string_1, &u32_legacy); + const bool ok_u64_legacy = LegacyParseUInt64(random_string_1, &u64_legacy); + const bool ok_u8_legacy = LegacyParseUInt8(random_string_1, &u8_legacy); + + assert(ok_i32 == ok_i32_legacy); + assert(ok_i64 == ok_i64_legacy); + assert(ok_u32 == ok_u32_legacy); + assert(ok_u64 == ok_u64_legacy); + assert(ok_u8 == ok_u8_legacy); + + if (ok_i32) { + assert(i32 == i32_legacy); + } + if (ok_i64) { + assert(i64 == i64_legacy); + } + if (ok_u32) { + assert(u32 == u32_legacy); + } + if (ok_u64) { + assert(u64 == u64_legacy); + } + if (ok_u8) { + assert(u8 == u8_legacy); + } + } + + { + const int atoi_result = atoi(random_string_1.c_str()); + const int locale_independent_atoi_result = LocaleIndependentAtoi<int>(random_string_1); + const int64_t atoi64_result = atoi64_legacy(random_string_1); + const bool out_of_range = atoi64_result < std::numeric_limits<int>::min() || atoi64_result > std::numeric_limits<int>::max(); + if (out_of_range) { + assert(locale_independent_atoi_result == 0); + } else { + assert(atoi_result == locale_independent_atoi_result); + } + } + + { + const int64_t atoi64_result = atoi64_legacy(random_string_1); + const int64_t locale_independent_atoi_result = LocaleIndependentAtoi<int64_t>(random_string_1); + assert(atoi64_result == locale_independent_atoi_result || locale_independent_atoi_result == 0); + } } diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 0f53939eac..dc3f9c8b8f 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -5,6 +5,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <cstdint> @@ -12,6 +13,11 @@ #include <vector> namespace { +void initialize_system() +{ + static const auto testing_setup = MakeNoLogFileContext<>(); +} + std::string GetArgumentName(const std::string& name) { size_t idx = name.find('='); @@ -20,9 +26,8 @@ std::string GetArgumentName(const std::string& name) } return name.substr(0, idx); } -} // namespace -FUZZ_TARGET(system) +FUZZ_TARGET_INIT(system, initialize_system) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ArgsManager args_manager{}; @@ -97,7 +102,7 @@ FUZZ_TARGET(system) const int64_t i64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const bool b = fuzzed_data_provider.ConsumeBool(); - (void)args_manager.GetArg(s1, i64); + (void)args_manager.GetIntArg(s1, i64); (void)args_manager.GetArg(s1, s2); (void)args_manager.GetArgFlags(s1); (void)args_manager.GetArgs(s1); @@ -114,3 +119,4 @@ FUZZ_TARGET(system) (void)HelpRequested(args_manager); } +} // namespace diff --git a/src/test/fuzz/torcontrol.cpp b/src/test/fuzz/torcontrol.cpp index a97d3962bf..a78715f769 100644 --- a/src/test/fuzz/torcontrol.cpp +++ b/src/test/fuzz/torcontrol.cpp @@ -44,7 +44,7 @@ FUZZ_TARGET_INIT(torcontrol, initialize_torcontrol) FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; TorController tor_controller; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { TorControlReply tor_control_reply; CallOneOf( fuzzed_data_provider, diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index ff34cc87b2..a21e5cea0c 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -103,6 +103,6 @@ FUZZ_TARGET_INIT(transaction, initialize_transaction) (void)IsWitnessStandard(tx, coins_view_cache); UniValue u(UniValue::VOBJ); - TxToUniv(tx, /* hashBlock */ uint256::ZERO, /* include_addresses */ true, u); - TxToUniv(tx, /* hashBlock */ uint256::ONE, /* include_addresses */ false, u); + TxToUniv(tx, /* hashBlock */ uint256::ZERO, u); + TxToUniv(tx, /* hashBlock */ uint256::ONE, u); } diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 6201cc813c..17b5ef88b9 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -81,7 +81,7 @@ void SetMempoolConstraints(ArgsManager& args, FuzzedDataProvider& fuzzed_data_pr void Finish(FuzzedDataProvider& fuzzed_data_provider, MockedTxPool& tx_pool, CChainState& chainstate) { - WITH_LOCK(::cs_main, tx_pool.check(chainstate)); + WITH_LOCK(::cs_main, tx_pool.check(chainstate.CoinsTip(), chainstate.m_chain.Height() + 1)); { BlockAssembler::Options options; options.nBlockMaxWeight = fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BLOCK_WEIGHT); @@ -97,7 +97,7 @@ void Finish(FuzzedDataProvider& fuzzed_data_provider, MockedTxPool& tx_pool, CCh std::vector<uint256> all_txids; tx_pool.queryHashes(all_txids); assert(all_txids.size() < info_all.size()); - WITH_LOCK(::cs_main, tx_pool.check(chainstate)); + WITH_LOCK(::cs_main, tx_pool.check(chainstate.CoinsTip(), chainstate.m_chain.Height() + 1)); } SyncWithValidationInterfaceQueue(); } diff --git a/src/test/fuzz/txrequest.cpp b/src/test/fuzz/txrequest.cpp index 72438ff2d7..a73bbcfc25 100644 --- a/src/test/fuzz/txrequest.cpp +++ b/src/test/fuzz/txrequest.cpp @@ -204,7 +204,7 @@ public: } // Call TxRequestTracker's implementation. - m_tracker.ReceivedInv(peer, GenTxid{is_wtxid, TXHASHES[txhash]}, preferred, reqtime); + m_tracker.ReceivedInv(peer, is_wtxid ? GenTxid::Wtxid(TXHASHES[txhash]) : GenTxid::Txid(TXHASHES[txhash]), preferred, reqtime); } void RequestedTx(int peer, int txhash, std::chrono::microseconds exptime) @@ -252,7 +252,7 @@ public: for (int peer2 = 0; peer2 < MAX_PEERS; ++peer2) { Announcement& ann2 = m_announcements[txhash][peer2]; if (ann2.m_state == State::REQUESTED && ann2.m_time <= m_now) { - expected_expired.emplace_back(peer2, GenTxid{ann2.m_is_wtxid, TXHASHES[txhash]}); + expected_expired.emplace_back(peer2, ann2.m_is_wtxid ? GenTxid::Wtxid(TXHASHES[txhash]) : GenTxid::Txid(TXHASHES[txhash])); ann2.m_state = State::COMPLETED; break; } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 0d87f687d3..ae5f7a379e 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <pubkey.h> #include <test/fuzz/util.h> #include <test/util/script.h> @@ -266,7 +267,7 @@ CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const auto amount = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-10, 50 * COIN + 10); const auto script_pk = p2wsh_op_true ? P2WSH_OP_TRUE : - ConsumeScript(fuzzed_data_provider, /* max_length */ 128, /* maybe_p2wsh */ true); + ConsumeScript(fuzzed_data_provider, /*maybe_p2wsh=*/true); tx_mut.vout.emplace_back(amount, script_pk); } return tx_mut; @@ -282,10 +283,63 @@ CScriptWitness ConsumeScriptWitness(FuzzedDataProvider& fuzzed_data_provider, co return ret; } -CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length, const bool maybe_p2wsh) noexcept +CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const bool maybe_p2wsh) noexcept { - const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); - CScript r_script{b.begin(), b.end()}; + CScript r_script{}; + { + // Keep a buffer of bytes to allow the fuzz engine to produce smaller + // inputs to generate CScripts with repeated data. + static constexpr unsigned MAX_BUFFER_SZ{128}; + std::vector<uint8_t> buffer(MAX_BUFFER_SZ, uint8_t{'a'}); + while (fuzzed_data_provider.ConsumeBool()) { + CallOneOf( + fuzzed_data_provider, + [&] { + // Insert byte vector directly to allow malformed or unparsable scripts + r_script.insert(r_script.end(), buffer.begin(), buffer.begin() + fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BUFFER_SZ)); + }, + [&] { + // Push a byte vector from the buffer + r_script << std::vector<uint8_t>{buffer.begin(), buffer.begin() + fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BUFFER_SZ)}; + }, + [&] { + // Push multisig + // There is a special case for this to aid the fuzz engine + // navigate the highly structured multisig format. + r_script << fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 22); + int num_data{fuzzed_data_provider.ConsumeIntegralInRange(1, 22)}; + std::vector<uint8_t> pubkey_comp{buffer.begin(), buffer.begin() + CPubKey::COMPRESSED_SIZE}; + pubkey_comp.front() = fuzzed_data_provider.ConsumeIntegralInRange(2, 3); // Set first byte for GetLen() to pass + std::vector<uint8_t> pubkey_uncomp{buffer.begin(), buffer.begin() + CPubKey::SIZE}; + pubkey_uncomp.front() = fuzzed_data_provider.ConsumeIntegralInRange(4, 7); // Set first byte for GetLen() to pass + while (num_data--) { + auto& pubkey{fuzzed_data_provider.ConsumeBool() ? pubkey_uncomp : pubkey_comp}; + if (fuzzed_data_provider.ConsumeBool()) { + pubkey.back() = num_data; // Make each pubkey different + } + r_script << pubkey; + } + r_script << fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 22); + }, + [&] { + // Mutate the buffer + const auto vec{ConsumeRandomLengthByteVector(fuzzed_data_provider, /*max_length=*/MAX_BUFFER_SZ)}; + std::copy(vec.begin(), vec.end(), buffer.begin()); + }, + [&] { + // Push an integral + r_script << fuzzed_data_provider.ConsumeIntegral<int64_t>(); + }, + [&] { + // Push an opcode + r_script << ConsumeOpcodeType(fuzzed_data_provider); + }, + [&] { + // Push a scriptnum + r_script << ConsumeScriptNum(fuzzed_data_provider); + }); + } + } if (maybe_p2wsh && fuzzed_data_provider.ConsumeBool()) { uint256 script_hash; CSHA256().Write(r_script.data(), r_script.size()).Finalize(script_hash.begin()); diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index bb017b3497..40aaeac63f 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -5,12 +5,12 @@ #ifndef BITCOIN_TEST_FUZZ_UTIL_H #define BITCOIN_TEST_FUZZ_UTIL_H -#include <amount.h> #include <arith_uint256.h> #include <attributes.h> #include <chainparamsbase.h> #include <coins.h> #include <compat.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <merkleblock.h> #include <net.h> @@ -132,7 +132,7 @@ template <typename WeakEnumType, size_t size> [[nodiscard]] CScriptWitness ConsumeScriptWitness(FuzzedDataProvider& fuzzed_data_provider, const size_t max_stack_elem_size = 32) noexcept; -[[nodiscard]] CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt, const bool maybe_p2wsh = false) noexcept; +[[nodiscard]] CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const bool maybe_p2wsh = false) noexcept; [[nodiscard]] uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept; diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index 6f2bc081c6..8d2a06f11a 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -4,6 +4,7 @@ #include <chainparams.h> #include <consensus/validation.h> +#include <node/utxo_snapshot.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/versionbits.cpp b/src/test/fuzz/versionbits.cpp index 73a7d24971..cf95c0b9bf 100644 --- a/src/test/fuzz/versionbits.cpp +++ b/src/test/fuzz/versionbits.cpp @@ -199,7 +199,7 @@ FUZZ_TARGET_INIT(versionbits, initialize) const uint32_t signalling_mask = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); // mine prior periods - while (fuzzed_data_provider.remaining_bytes() > 0) { + while (fuzzed_data_provider.remaining_bytes() > 0) { // early exit; no need for LIMITED_WHILE // all blocks in these periods either do or don't signal bool signal = fuzzed_data_provider.ConsumeBool(); for (int b = 0; b < period; ++b) { diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 2a217f3455..b0c8068ab9 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -137,20 +137,20 @@ BOOST_AUTO_TEST_CASE(intarg) const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs(""); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 11), 11); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 0), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 11), 11); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 0), 0); ResetArgs("-foo -bar"); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 11), 0); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-bar", 11), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 11), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 11), 0); ResetArgs("-foo=11 -bar=12"); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 0), 11); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-bar", 11), 12); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 0), 11); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 11), 12); ResetArgs("-foo=NaN -bar=NotANumber"); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 1), 0); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-bar", 11), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 1), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 11), 0); } BOOST_AUTO_TEST_CASE(doubledash) @@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(doubledash) ResetArgs("--foo=verbose --bar=1"); BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", ""), "verbose"); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-bar", 0), 1); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 0), 1); } BOOST_AUTO_TEST_CASE(boolargno) @@ -194,8 +194,8 @@ BOOST_AUTO_TEST_CASE(boolargno) BOOST_AUTO_TEST_CASE(logargs) { - const auto okaylog_bool = std::make_pair("-okaylog-bool", ArgsManager::ALLOW_BOOL); - const auto okaylog_negbool = std::make_pair("-okaylog-negbool", ArgsManager::ALLOW_BOOL); + const auto okaylog_bool = std::make_pair("-okaylog-bool", ArgsManager::ALLOW_ANY); + const auto okaylog_negbool = std::make_pair("-okaylog-negbool", ArgsManager::ALLOW_ANY); const auto okaylog = std::make_pair("-okaylog", ArgsManager::ALLOW_ANY); const auto dontlog = std::make_pair("-dontlog", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE); SetupArgs({okaylog_bool, okaylog_negbool, okaylog, dontlog}); diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index 8629d13840..0361618c82 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE(key_io_valid_parse) privkey = DecodeSecret(exp_base58string); BOOST_CHECK_MESSAGE(privkey.IsValid(), "!IsValid:" + strTest); BOOST_CHECK_MESSAGE(privkey.IsCompressed() == isCompressed, "compressed mismatch:" + strTest); - BOOST_CHECK_MESSAGE(privkey.size() == exp_payload.size() && std::equal(privkey.begin(), privkey.end(), exp_payload.begin()), "key mismatch:" + strTest); + BOOST_CHECK_MESSAGE(Span<const uint8_t>{privkey} == Span<const uint8_t>{exp_payload}, "key mismatch:" + strTest); // Private key must be invalid public key destination = DecodeDestination(exp_base58string); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index bf36f8a6c9..b3497b8ef8 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -444,12 +444,12 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(entry.Fee(5000LL).FromTx(tx2)); pool.TrimToSize(pool.DynamicMemoryUsage()); // should do nothing - BOOST_CHECK(pool.exists(tx1.GetHash())); - BOOST_CHECK(pool.exists(tx2.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx2.GetHash()))); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // should remove the lower-feerate transaction - BOOST_CHECK(pool.exists(tx1.GetHash())); - BOOST_CHECK(!pool.exists(tx2.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx2.GetHash()))); pool.addUnchecked(entry.FromTx(tx2)); CMutableTransaction tx3 = CMutableTransaction(); @@ -462,14 +462,14 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(entry.Fee(20000LL).FromTx(tx3)); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // tx3 should pay for tx2 (CPFP) - BOOST_CHECK(!pool.exists(tx1.GetHash())); - BOOST_CHECK(pool.exists(tx2.GetHash())); - BOOST_CHECK(pool.exists(tx3.GetHash())); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx2.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx3.GetHash()))); pool.TrimToSize(GetVirtualTransactionSize(CTransaction(tx1))); // mempool is limited to tx1's size in memory usage, so nothing fits - BOOST_CHECK(!pool.exists(tx1.GetHash())); - BOOST_CHECK(!pool.exists(tx2.GetHash())); - BOOST_CHECK(!pool.exists(tx3.GetHash())); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx2.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx3.GetHash()))); CFeeRate maxFeeRateRemoved(25000, GetVirtualTransactionSize(CTransaction(tx3)) + GetVirtualTransactionSize(CTransaction(tx2))); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); @@ -529,19 +529,19 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) // we only require this to remove, at max, 2 txn, because it's not clear what we're really optimizing for aside from that pool.TrimToSize(pool.DynamicMemoryUsage() - 1); - BOOST_CHECK(pool.exists(tx4.GetHash())); - BOOST_CHECK(pool.exists(tx6.GetHash())); - BOOST_CHECK(!pool.exists(tx7.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx4.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx6.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx7.GetHash()))); - if (!pool.exists(tx5.GetHash())) + if (!pool.exists(GenTxid::Txid(tx5.GetHash()))) pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); pool.TrimToSize(pool.DynamicMemoryUsage() / 2); // should maximize mempool size by only removing 5/7 - BOOST_CHECK(pool.exists(tx4.GetHash())); - BOOST_CHECK(!pool.exists(tx5.GetHash())); - BOOST_CHECK(pool.exists(tx6.GetHash())); - BOOST_CHECK(!pool.exists(tx7.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx4.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx5.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx6.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx7.GetHash()))); pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); diff --git a/src/test/minisketch_tests.cpp b/src/test/minisketch_tests.cpp new file mode 100644 index 0000000000..6798331936 --- /dev/null +++ b/src/test/minisketch_tests.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <minisketch.h> +#include <minisketchwrapper.h> +#include <random.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +#include <utility> + +BOOST_AUTO_TEST_SUITE(minisketch_tests) + +BOOST_AUTO_TEST_CASE(minisketch_test) +{ + for (int i = 0; i < 100; ++i) { + uint32_t errors = 0 + InsecureRandRange(11); + uint32_t start_a = 1 + InsecureRandRange(1000000000); + uint32_t a_not_b = InsecureRandRange(errors + 1); + uint32_t b_not_a = errors - a_not_b; + uint32_t both = InsecureRandRange(10000); + uint32_t end_a = start_a + a_not_b + both; + uint32_t start_b = start_a + a_not_b; + uint32_t end_b = start_b + both + b_not_a; + + Minisketch sketch_a = MakeMinisketch32(10); + for (uint32_t a = start_a; a < end_a; ++a) sketch_a.Add(a); + Minisketch sketch_b = MakeMinisketch32(10); + for (uint32_t b = start_b; b < end_b; ++b) sketch_b.Add(b); + + Minisketch sketch_ar = MakeMinisketch32(10); + Minisketch sketch_br = MakeMinisketch32(10); + sketch_ar.Deserialize(sketch_a.Serialize()); + sketch_br.Deserialize(sketch_b.Serialize()); + + Minisketch sketch_c = std::move(sketch_ar); + sketch_c.Merge(sketch_br); + auto dec = sketch_c.Decode(errors); + BOOST_CHECK(dec.has_value()); + auto sols = std::move(*dec); + std::sort(sols.begin(), sols.end()); + for (uint32_t i = 0; i < a_not_b; ++i) BOOST_CHECK_EQUAL(sols[i], start_a + i); + for (uint32_t i = 0; i < b_not_a; ++i) BOOST_CHECK_EQUAL(sols[i + a_not_b], start_b + both + i); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 687d2f6747..b6d7496cc7 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -339,11 +339,13 @@ BOOST_AUTO_TEST_CASE(netbase_parsenetwork) BOOST_CHECK_EQUAL(ParseNetwork("ipv6"), NET_IPV6); BOOST_CHECK_EQUAL(ParseNetwork("onion"), NET_ONION); BOOST_CHECK_EQUAL(ParseNetwork("tor"), NET_ONION); + BOOST_CHECK_EQUAL(ParseNetwork("cjdns"), NET_CJDNS); BOOST_CHECK_EQUAL(ParseNetwork("IPv4"), NET_IPV4); BOOST_CHECK_EQUAL(ParseNetwork("IPv6"), NET_IPV6); BOOST_CHECK_EQUAL(ParseNetwork("ONION"), NET_ONION); BOOST_CHECK_EQUAL(ParseNetwork("TOR"), NET_ONION); + BOOST_CHECK_EQUAL(ParseNetwork("CJDNS"), NET_CJDNS); BOOST_CHECK_EQUAL(ParseNetwork(":)"), NET_UNROUTABLE); BOOST_CHECK_EQUAL(ParseNetwork("tÖr"), NET_UNROUTABLE); diff --git a/src/test/policy_fee_tests.cpp b/src/test/policy_fee_tests.cpp index 4a15be6ca6..f9c7d04d6c 100644 --- a/src/test/policy_fee_tests.cpp +++ b/src/test/policy_fee_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/fees.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_parse_tests.cpp b/src/test/script_parse_tests.cpp new file mode 100644 index 0000000000..004c1a9a84 --- /dev/null +++ b/src/test/script_parse_tests.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <script/script.h> +#include <util/strencodings.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(script_parse_tests) +BOOST_AUTO_TEST_CASE(parse_script) +{ + const std::vector<std::pair<std::string,std::string>> IN_OUT{ + // {IN: script string , OUT: hex string } + {"", ""}, + {"0", "00"}, + {"1", "51"}, + {"2", "52"}, + {"3", "53"}, + {"4", "54"}, + {"5", "55"}, + {"6", "56"}, + {"7", "57"}, + {"8", "58"}, + {"9", "59"}, + {"10", "5a"}, + {"11", "5b"}, + {"12", "5c"}, + {"13", "5d"}, + {"14", "5e"}, + {"15", "5f"}, + {"16", "60"}, + {"17", "0111"}, + {"-9", "0189"}, + {"0x17", "17"}, + {"'17'", "023137"}, + {"ELSE", "67"}, + {"NOP10", "b9"}, + }; + std::string all_in; + std::string all_out; + for (const auto& [in, out] : IN_OUT) { + BOOST_CHECK_EQUAL(HexStr(ParseScript(in)), out); + all_in += " " + in + " "; + all_out += out; + } + BOOST_CHECK_EQUAL(HexStr(ParseScript(all_in)), all_out); + + BOOST_CHECK_EXCEPTION(ParseScript("11111111111111111111"), std::runtime_error, HasReason("script parse error: decimal numeric value only allowed in the range -0xFFFFFFFF...0xFFFFFFFF")); + BOOST_CHECK_EXCEPTION(ParseScript("11111111111"), std::runtime_error, HasReason("script parse error: decimal numeric value only allowed in the range -0xFFFFFFFF...0xFFFFFFFF")); + BOOST_CHECK_EXCEPTION(ParseScript("OP_CHECKSIGADD"), std::runtime_error, HasReason("script parse error: unknown opcode")); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index a01d3fa03a..bf8ff5f5e2 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -252,67 +252,6 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(std::get<WitnessUnknown>(address) == unk); } -BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) -{ - CKey keys[3]; - CPubKey pubkeys[3]; - for (int i = 0; i < 3; i++) { - keys[i].MakeNewKey(true); - pubkeys[i] = keys[i].GetPubKey(); - } - - CScript s; - TxoutType whichType; - std::vector<CTxDestination> addresses; - int nRequired; - - // TxoutType::PUBKEY - s.clear(); - s << ToByteVector(pubkeys[0]) << OP_CHECKSIG; - BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEY); - BOOST_CHECK_EQUAL(addresses.size(), 1U); - BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); - - // TxoutType::PUBKEYHASH - s.clear(); - s << OP_DUP << OP_HASH160 << ToByteVector(pubkeys[0].GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; - BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEYHASH); - BOOST_CHECK_EQUAL(addresses.size(), 1U); - BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); - - // TxoutType::SCRIPTHASH - CScript redeemScript(s); // initialize with leftover P2PKH script - s.clear(); - s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; - BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TxoutType::SCRIPTHASH); - BOOST_CHECK_EQUAL(addresses.size(), 1U); - BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get<ScriptHash>(addresses[0]) == ScriptHash(redeemScript)); - - // TxoutType::MULTISIG - s.clear(); - s << OP_2 << - ToByteVector(pubkeys[0]) << - ToByteVector(pubkeys[1]) << - OP_2 << OP_CHECKMULTISIG; - BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TxoutType::MULTISIG); - BOOST_CHECK_EQUAL(addresses.size(), 2U); - BOOST_CHECK_EQUAL(nRequired, 2); - BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); - BOOST_CHECK(std::get<PKHash>(addresses[1]) == PKHash(pubkeys[1])); - - // TxoutType::NULL_DATA - s.clear(); - s << OP_RETURN << std::vector<unsigned char>({75}); - BOOST_CHECK(!ExtractDestinations(s, whichType, addresses, nRequired)); -} - BOOST_AUTO_TEST_CASE(script_standard_GetScriptFor_) { CKey keys[3]; diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 340ce33d91..15cba9e3e5 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -80,19 +80,19 @@ BOOST_AUTO_TEST_CASE(ReadWrite) "dupe": "dupe" })"); BOOST_CHECK(!util::ReadSettings(path, values, errors)); - std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", path.string())}; + std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), dup_keys.begin(), dup_keys.end()); // Check non-kv json files not allowed WriteText(path, R"("non-kv")"); BOOST_CHECK(!util::ReadSettings(path, values, errors)); - std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", path.string())}; + std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), non_kv.begin(), non_kv.end()); // Check invalid json not allowed WriteText(path, R"(invalid json)"); BOOST_CHECK(!util::ReadSettings(path, values, errors)); - std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", path.string())}; + std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), fail_parse.begin(), fail_parse.end()); } diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index acd0151e1a..54f04d2e67 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -215,7 +215,9 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) BOOST_AUTO_TEST_CASE(streams_buffered_file) { - FILE* file = fsbridge::fopen("streams_test_tmp", "w+b"); + fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; + FILE* file = fsbridge::fopen(streams_test_filename, "w+b"); + // The value at each offset is the offset. for (uint8_t j = 0; j < 40; ++j) { fwrite(&j, 1, 1, file); @@ -343,7 +345,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) // We can explicitly close the file, or the destructor will do it. bf.fclose(); - fs::remove("streams_test_tmp"); + fs::remove(streams_test_filename); } BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) @@ -351,8 +353,9 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) // Make this test deterministic. SeedInsecureRand(SeedRand::ZEROS); + fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; for (int rep = 0; rep < 50; ++rep) { - FILE* file = fsbridge::fopen("streams_test_tmp", "w+b"); + FILE* file = fsbridge::fopen(streams_test_filename, "w+b"); size_t fileSize = InsecureRandRange(256); for (uint8_t i = 0; i < fileSize; ++i) { fwrite(&i, 1, 1, file); @@ -453,7 +456,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) maxPos = currentPos; } } - fs::remove("streams_test_tmp"); + fs::remove(streams_test_filename); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 24029ea02e..252a85c282 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -8,6 +8,7 @@ #include <checkqueue.h> #include <clientversion.h> +#include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <core_io.h> @@ -809,10 +810,10 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // nDustThreshold = 182 * 3702 / 1000 dustRelayFee = CFeeRate(3702); // dust: - t.vout[0].nValue = 673 - 1; + t.vout[0].nValue = 674 - 1; CheckIsNotStandard(t, "dust"); // not dust: - t.vout[0].nValue = 673; + t.vout[0].nValue = 674; CheckIsStandard(t); dustRelayFee = CFeeRate(DUST_RELAY_TX_FEE); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp new file mode 100644 index 0000000000..537a6ccea1 --- /dev/null +++ b/src/test/txpackage_tests.cpp @@ -0,0 +1,117 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/validation.h> +#include <key_io.h> +#include <policy/packages.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <script/standard.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(txpackage_tests) + +// Create placeholder transactions that have no meaning. +inline CTransactionRef create_placeholder_tx(size_t num_inputs, size_t num_outputs) +{ + CMutableTransaction mtx = CMutableTransaction(); + mtx.vin.resize(num_inputs); + mtx.vout.resize(num_outputs); + auto random_script = CScript() << ToByteVector(InsecureRand256()) << ToByteVector(InsecureRand256()); + for (size_t i{0}; i < num_inputs; ++i) { + mtx.vin[i].prevout.hash = InsecureRand256(); + mtx.vin[i].prevout.n = 0; + mtx.vin[i].scriptSig = random_script; + } + for (size_t o{0}; o < num_outputs; ++o) { + mtx.vout[o].nValue = 1 * CENT; + mtx.vout[o].scriptPubKey = random_script; + } + return MakeTransactionRef(mtx); +} + +BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) +{ + // Packages can't have more than 25 transactions. + Package package_too_many; + package_too_many.reserve(MAX_PACKAGE_COUNT + 1); + for (size_t i{0}; i < MAX_PACKAGE_COUNT + 1; ++i) { + package_too_many.emplace_back(create_placeholder_tx(1, 1)); + } + PackageValidationState state_too_many; + BOOST_CHECK(!CheckPackage(package_too_many, state_too_many)); + BOOST_CHECK_EQUAL(state_too_many.GetResult(), PackageValidationResult::PCKG_POLICY); + BOOST_CHECK_EQUAL(state_too_many.GetRejectReason(), "package-too-many-transactions"); + + // Packages can't have a total size of more than 101KvB. + CTransactionRef large_ptx = create_placeholder_tx(150, 150); + Package package_too_large; + auto size_large = GetVirtualTransactionSize(*large_ptx); + size_t total_size{0}; + while (total_size <= MAX_PACKAGE_SIZE * 1000) { + package_too_large.push_back(large_ptx); + total_size += size_large; + } + BOOST_CHECK(package_too_large.size() <= MAX_PACKAGE_COUNT); + PackageValidationState state_too_large; + BOOST_CHECK(!CheckPackage(package_too_large, state_too_large)); + BOOST_CHECK_EQUAL(state_too_large.GetResult(), PackageValidationResult::PCKG_POLICY); + BOOST_CHECK_EQUAL(state_too_large.GetRejectReason(), "package-too-large"); +} + +BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) +{ + LOCK(cs_main); + unsigned int initialPoolSize = m_node.mempool->size(); + + // Parent and Child Package + CKey parent_key; + parent_key.MakeNewKey(true); + CScript parent_locking_script = GetScriptForDestination(PKHash(parent_key.GetPubKey())); + auto mtx_parent = CreateValidMempoolTransaction(/* input_transaction */ m_coinbase_txns[0], /* vout */ 0, + /* input_height */ 0, /* input_signing_key */ coinbaseKey, + /* output_destination */ parent_locking_script, + /* output_amount */ CAmount(49 * COIN), /* submit */ false); + CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); + + CKey child_key; + child_key.MakeNewKey(true); + CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); + auto mtx_child = CreateValidMempoolTransaction(/* input_transaction */ tx_parent, /* vout */ 0, + /* input_height */ 101, /* input_signing_key */ parent_key, + /* output_destination */ child_locking_script, + /* output_amount */ CAmount(48 * COIN), /* submit */ false); + CTransactionRef tx_child = MakeTransactionRef(mtx_child); + const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /* test_accept */ true); + BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), + "Package validation unexpectedly failed: " << result_parent_child.m_state.GetRejectReason()); + auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent != result_parent_child.m_tx_results.end()); + BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), + "Package validation unexpectedly failed: " << it_parent->second.m_state.GetRejectReason()); + BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); + BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), + "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); + + + // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. + CTransactionRef giant_ptx = create_placeholder_tx(999, 999); + BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > MAX_PACKAGE_SIZE * 1000); + auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {giant_ptx}, /* test_accept */ true); + BOOST_CHECK(result_single_large.m_state.IsInvalid()); + BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); + auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); + BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); + BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + + // Check that mempool size hasn't changed. + BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txrequest_tests.cpp b/src/test/txrequest_tests.cpp index 1d137b03b1..99d41882c9 100644 --- a/src/test/txrequest_tests.cpp +++ b/src/test/txrequest_tests.cpp @@ -221,7 +221,7 @@ public: /** Generate a random GenTxid; the txhash follows NewTxHash; the is_wtxid flag is random. */ GenTxid NewGTxid(const std::vector<std::vector<NodeId>>& orders = {}) { - return {InsecureRandBool(), NewTxHash(orders)}; + return InsecureRandBool() ? GenTxid::Wtxid(NewTxHash(orders)) : GenTxid::Txid(NewTxHash(orders)); } /** Generate a new random NodeId to use as peer. The same NodeId is never returned twice @@ -494,8 +494,8 @@ void BuildWtxidTest(Scenario& scenario, int config) auto peerT = scenario.NewPeer(); auto peerW = scenario.NewPeer(); auto txhash = scenario.NewTxHash(); - GenTxid txid{false, txhash}; - GenTxid wtxid{true, txhash}; + auto txid{GenTxid::Txid(txhash)}; + auto wtxid{GenTxid::Wtxid(txhash)}; auto reqtimeT = InsecureRandBool() ? MIN_TIME : scenario.Now() + RandomTime8s(); auto reqtimeW = InsecureRandBool() ? MIN_TIME : scenario.Now() + RandomTime8s(); diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index ade9e210f2..c71ab01af4 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -37,8 +37,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) LOCK(cs_main); unsigned int initialPoolSize = m_node.mempool->size(); - const MempoolAcceptResult result = AcceptToMemoryPool(m_node.chainman->ActiveChainstate(), *m_node.mempool, MakeTransactionRef(coinbaseTx), - true /* bypass_limits */); + const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(MakeTransactionRef(coinbaseTx)); BOOST_CHECK(result.m_result_type == MempoolAcceptResult::ResultType::INVALID); @@ -50,99 +49,4 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) BOOST_CHECK_EQUAL(result.m_state.GetRejectReason(), "coinbase"); BOOST_CHECK(result.m_state.GetResult() == TxValidationResult::TX_CONSENSUS); } - -// Create placeholder transactions that have no meaning. -inline CTransactionRef create_placeholder_tx(size_t num_inputs, size_t num_outputs) -{ - CMutableTransaction mtx = CMutableTransaction(); - mtx.vin.resize(num_inputs); - mtx.vout.resize(num_outputs); - auto random_script = CScript() << ToByteVector(InsecureRand256()) << ToByteVector(InsecureRand256()); - for (size_t i{0}; i < num_inputs; ++i) { - mtx.vin[i].prevout.hash = InsecureRand256(); - mtx.vin[i].prevout.n = 0; - mtx.vin[i].scriptSig = random_script; - } - for (size_t o{0}; o < num_outputs; ++o) { - mtx.vout[o].nValue = 1 * CENT; - mtx.vout[o].scriptPubKey = random_script; - } - return MakeTransactionRef(mtx); -} - -BOOST_FIXTURE_TEST_CASE(package_tests, TestChain100Setup) -{ - LOCK(cs_main); - unsigned int initialPoolSize = m_node.mempool->size(); - - // Parent and Child Package - CKey parent_key; - parent_key.MakeNewKey(true); - CScript parent_locking_script = GetScriptForDestination(PKHash(parent_key.GetPubKey())); - auto mtx_parent = CreateValidMempoolTransaction(/* input_transaction */ m_coinbase_txns[0], /* vout */ 0, - /* input_height */ 0, /* input_signing_key */ coinbaseKey, - /* output_destination */ parent_locking_script, - /* output_amount */ CAmount(49 * COIN), /* submit */ false); - CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); - - CKey child_key; - child_key.MakeNewKey(true); - CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); - auto mtx_child = CreateValidMempoolTransaction(/* input_transaction */ tx_parent, /* vout */ 0, - /* input_height */ 101, /* input_signing_key */ parent_key, - /* output_destination */ child_locking_script, - /* output_amount */ CAmount(48 * COIN), /* submit */ false); - CTransactionRef tx_child = MakeTransactionRef(mtx_child); - const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /* test_accept */ true); - BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), - "Package validation unexpectedly failed: " << result_parent_child.m_state.GetRejectReason()); - auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_parent->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); - - // Packages can't have more than 25 transactions. - Package package_too_many; - package_too_many.reserve(MAX_PACKAGE_COUNT + 1); - for (size_t i{0}; i < MAX_PACKAGE_COUNT + 1; ++i) { - package_too_many.emplace_back(create_placeholder_tx(1, 1)); - } - auto result_too_many = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_too_many, /* test_accept */ true); - BOOST_CHECK(result_too_many.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_too_many.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); - BOOST_CHECK_EQUAL(result_too_many.m_state.GetRejectReason(), "package-too-many-transactions"); - - // Packages can't have a total size of more than 101KvB. - CTransactionRef large_ptx = create_placeholder_tx(150, 150); - Package package_too_large; - auto size_large = GetVirtualTransactionSize(*large_ptx); - size_t total_size{0}; - while (total_size <= MAX_PACKAGE_SIZE * 1000) { - package_too_large.push_back(large_ptx); - total_size += size_large; - } - BOOST_CHECK(package_too_large.size() <= MAX_PACKAGE_COUNT); - auto result_too_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_too_large, /* test_accept */ true); - BOOST_CHECK(result_too_large.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_too_large.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); - BOOST_CHECK_EQUAL(result_too_large.m_state.GetRejectReason(), "package-too-large"); - - // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. - CTransactionRef giant_ptx = create_placeholder_tx(999, 999); - BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > MAX_PACKAGE_SIZE * 1000); - auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {giant_ptx}, /* test_accept */ true); - BOOST_CHECK(result_single_large.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); - auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); - BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); - - // Check that mempool size hasn't changed. - BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); -} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 1924ea55b1..8be64531c4 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -13,6 +13,11 @@ #include <boost/test/unit_test.hpp> +struct Dersig100Setup : public TestChain100Setup { + Dersig100Setup() + : TestChain100Setup{{"-testactivationheight=dersig@102"}} {} +}; + bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, @@ -20,7 +25,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, BOOST_AUTO_TEST_SUITE(txvalidationcache_tests) -BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) +BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, Dersig100Setup) { // Make sure skipping validation of transactions that were // validated going into the memory pool does not allow @@ -31,8 +36,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) const auto ToMemPool = [this](const CMutableTransaction& tx) { LOCK(cs_main); - const MempoolAcceptResult result = AcceptToMemoryPool(m_node.chainman->ActiveChainstate(), *m_node.mempool, MakeTransactionRef(tx), - true /* bypass_limits */); + const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(MakeTransactionRef(tx)); return result.m_result_type == MempoolAcceptResult::ResultType::VALID; }; @@ -153,7 +157,7 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail } } -BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) +BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) { // Test that passing CheckInputScripts with one set of script flags doesn't imply // that we would pass again with a different set of flags. diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h new file mode 100644 index 0000000000..e95573022c --- /dev/null +++ b/src/test/util/chainstate.h @@ -0,0 +1,54 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#ifndef BITCOIN_TEST_UTIL_CHAINSTATE_H +#define BITCOIN_TEST_UTIL_CHAINSTATE_H + +#include <clientversion.h> +#include <fs.h> +#include <node/context.h> +#include <node/utxo_snapshot.h> +#include <rpc/blockchain.h> +#include <validation.h> + +#include <univalue.h> + +#include <boost/test/unit_test.hpp> + +const auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; + +/** + * Create and activate a UTXO snapshot, optionally providing a function to + * malleate the snapshot. + */ +template<typename F = decltype(NoMalleation)> +static bool +CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) +{ + // Write out a snapshot to the test's tempdir. + // + int height; + WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); + fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); + FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; + CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + + UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); + BOOST_TEST_MESSAGE( + "Wrote UTXO snapshot to " << fs::PathToString(snapshot_path.make_preferred()) << ": " << result.write()); + + // Read the written snapshot in and then activate it. + // + FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; + CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + SnapshotMetadata metadata; + auto_infile >> metadata; + + malleation(auto_infile, metadata); + + return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); +} + + +#endif // BITCOIN_TEST_UTIL_CHAINSTATE_H diff --git a/src/test/util/net.h b/src/test/util/net.h index 939ec322ed..d89fc34b75 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -17,6 +17,12 @@ struct ConnmanTestMsg : public CConnman { using CConnman::CConnman; + + void SetPeerConnectTimeout(int64_t timeout) + { + m_peer_connect_timeout = timeout; + } + void AddTestNode(CNode& node) { LOCK(cs_vNodes); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index cabc4b3b49..5a0c8e152a 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -91,8 +91,8 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve extra_args); util::ThreadRename("test"); fs::create_directories(m_path_root); - m_args.ForceSetArg("-datadir", m_path_root.string()); - gArgs.ForceSetArg("-datadir", m_path_root.string()); + m_args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); + gArgs.ForceSetArg("-datadir", fs::PathToString(m_path_root)); gArgs.ClearPathCache(); { SetupServerArgs(*m_node.args); @@ -192,7 +192,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } - m_node.addrman = std::make_unique<CAddrMan>(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + m_node.addrman = std::make_unique<AddrMan>(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); m_node.banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. m_node.peerman = PeerManager::make(chainparams, *m_node.connman, *m_node.addrman, @@ -205,7 +205,8 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } } -TestChain100Setup::TestChain100Setup() +TestChain100Setup::TestChain100Setup(const std::vector<const char*>& extra_args) + : TestingSetup{CBaseChainParams::REGTEST, extra_args} { SetMockTime(1598887952); constexpr std::array<unsigned char, 32> vchKey = { @@ -234,11 +235,14 @@ void TestChain100Setup::mineBlocks(int num_blocks) } } -CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) +CBlock TestChain100Setup::CreateBlock( + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey, + CChainState& chainstate) { const CChainParams& chainparams = Params(); CTxMemPool empty_pool; - CBlock block = BlockAssembler(m_node.chainman->ActiveChainstate(), empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; + CBlock block = BlockAssembler(chainstate, empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; Assert(block.vtx.size() == 1); for (const CMutableTransaction& tx : txns) { @@ -248,6 +252,20 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; + return block; +} + +CBlock TestChain100Setup::CreateAndProcessBlock( + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey, + CChainState* chainstate) +{ + if (!chainstate) { + chainstate = &Assert(m_node.chainman)->ActiveChainstate(); + } + + const CChainParams& chainparams = Params(); + const CBlock block = this->CreateBlock(txns, scriptPubKey, *chainstate); std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); @@ -297,18 +315,13 @@ CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactio // If submit=true, add transaction to the mempool. if (submit) { LOCK(cs_main); - const MempoolAcceptResult result = AcceptToMemoryPool(m_node.chainman->ActiveChainstate(), *m_node.mempool.get(), MakeTransactionRef(mempool_txn), /* bypass_limits */ false); + const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(MakeTransactionRef(mempool_txn)); assert(result.m_result_type == MempoolAcceptResult::ResultType::VALID); } return mempool_txn; } -TestChain100Setup::~TestChain100Setup() -{ - gArgs.ForceSetArg("-segwitheight", "0"); -} - CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const { return FromTx(MakeTransactionRef(tx)); diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 5d12dc2323..7518cdb042 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -113,15 +113,26 @@ class CScript; /** * Testing fixture that pre-creates a 100-block REGTEST-mode block chain */ -struct TestChain100Setup : public RegTestingSetup { - TestChain100Setup(); +struct TestChain100Setup : public TestingSetup { + TestChain100Setup(const std::vector<const char*>& extra_args = {}); /** * Create a new block with just given transactions, coinbase paying to * scriptPubKey, and try to add it to the current chain. + * If no chainstate is specified, default to the active. */ CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, - const CScript& scriptPubKey); + const CScript& scriptPubKey, + CChainState* chainstate = nullptr); + + /** + * Create a new block with just given transactions, coinbase paying to + * scriptPubKey. + */ + CBlock CreateBlock( + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey, + CChainState& chainstate); //! Mine a series of new blocks on the active chain. void mineBlocks(int num_blocks); @@ -145,8 +156,6 @@ struct TestChain100Setup : public RegTestingSetup { CAmount output_amount = CAmount(1 * COIN), bool submit = true); - ~TestChain100Setup(); - std::vector<CTransactionRef> m_coinbase_txns; // For convenience, coinbase transactions CKey coinbaseKey; // private/public key needed to spend coinbase transactions }; diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp index 061659818f..76c1bf93a5 100644 --- a/src/test/util/wallet.cpp +++ b/src/test/util/wallet.cpp @@ -25,16 +25,4 @@ std::string getnewaddress(CWallet& w) return EncodeDestination(dest); } -void importaddress(CWallet& wallet, const std::string& address) -{ - auto spk_man = wallet.GetLegacyScriptPubKeyMan(); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); - const auto dest = DecodeDestination(address); - assert(IsValidDestination(dest)); - const auto script = GetScriptForDestination(dest); - wallet.MarkDirty(); - assert(!spk_man->HaveWatchOnly(script)); - if (!spk_man->AddWatchOnly(script, 0 /* nCreateTime */)) assert(false); - wallet.SetAddressBook(dest, /* label */ "", "receive"); -} #endif // ENABLE_WALLET diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index a62abf9b9c..b1300d06ba 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -51,23 +51,23 @@ BOOST_AUTO_TEST_CASE(util_datadir) { // Use local args variable instead of m_args to avoid making assumptions about test setup ArgsManager args; - args.ForceSetArg("-datadir", m_path_root.string()); + args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); const fs::path dd_norm = args.GetDataDirBase(); - args.ForceSetArg("-datadir", dd_norm.string() + "/"); + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/"); args.ClearPathCache(); BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - args.ForceSetArg("-datadir", dd_norm.string() + "/."); + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/."); args.ClearPathCache(); BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - args.ForceSetArg("-datadir", dd_norm.string() + "/./"); + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/./"); args.ClearPathCache(); BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - args.ForceSetArg("-datadir", dd_norm.string() + "/.//"); + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/.//"); args.ClearPathCache(); BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); } @@ -300,9 +300,9 @@ public: } if (expect.default_int) { - BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), 99999); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), 99999); } else if (expect.int_value) { - BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), *expect.int_value); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), *expect.int_value); } else { BOOST_CHECK(!success); } @@ -432,8 +432,8 @@ static void TestParse(const std::string& str, bool expected_bool, int64_t expect BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), expected_bool); BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), expected_bool); - BOOST_CHECK_EQUAL(test.GetArg("-value", 99998), expected_int); - BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), expected_int); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99998), expected_int); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), expected_int); } // Test bool and int parsing. @@ -784,9 +784,9 @@ BOOST_AUTO_TEST_CASE(util_GetArg) BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); - BOOST_CHECK_EQUAL(testArgs.GetArg("inttest1", -1), 12345); - BOOST_CHECK_EQUAL(testArgs.GetArg("inttest2", -1), 81985529216486895LL); - BOOST_CHECK_EQUAL(testArgs.GetArg("inttest3", -1), -1); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest1", -1), 12345); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest2", -1), 81985529216486895LL); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest3", -1), -1); BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest1", false), true); BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest2", false), false); BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest3", false), false); @@ -1181,13 +1181,13 @@ BOOST_AUTO_TEST_CASE(util_ReadWriteSettings) { // Test writing setting. TestArgsManager args1; - args1.ForceSetArg("-datadir", m_path_root.string()); + args1.ForceSetArg("-datadir", fs::PathToString(m_path_root)); args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; }); args1.WriteSettingsFile(); // Test reading setting. TestArgsManager args2; - args2.ForceSetArg("-datadir", m_path_root.string()); + args2.ForceSetArg("-datadir", fs::PathToString(m_path_root)); args2.ReadSettingsFile(); args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); @@ -1239,6 +1239,11 @@ BOOST_AUTO_TEST_CASE(util_FormatMoney) BOOST_AUTO_TEST_CASE(util_ParseMoney) { BOOST_CHECK_EQUAL(ParseMoney("0.0").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney("0.").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".0").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".6789").value(), 6789'0000); + BOOST_CHECK_EQUAL(ParseMoney("12345.").value(), COIN * 12345); BOOST_CHECK_EQUAL(ParseMoney("12345.6789").value(), (COIN/10000)*123456789); @@ -1276,11 +1281,18 @@ BOOST_AUTO_TEST_CASE(util_ParseMoney) BOOST_CHECK(!ParseMoney(" ")); // Parsing two numbers should fail + BOOST_CHECK(!ParseMoney("..")); + BOOST_CHECK(!ParseMoney("0..0")); BOOST_CHECK(!ParseMoney("1 2")); BOOST_CHECK(!ParseMoney(" 1 2 ")); BOOST_CHECK(!ParseMoney(" 1.2 3 ")); BOOST_CHECK(!ParseMoney(" 1 2.3 ")); + // Embedded whitespace should fail + BOOST_CHECK(!ParseMoney(" -1 .2 ")); + BOOST_CHECK(!ParseMoney(" 1 .2 ")); + BOOST_CHECK(!ParseMoney(" +1 .2 ")); + // Attempted 63 bit overflow should fail BOOST_CHECK(!ParseMoney("92233720368.54775808")); @@ -1474,6 +1486,168 @@ BOOST_AUTO_TEST_CASE(test_ParseInt32) BOOST_CHECK(!ParseInt32("32482348723847471234", nullptr)); } +template <typename T> +static void RunToIntegralTests() +{ + BOOST_CHECK(!ToIntegral<T>(STRING_WITH_EMBEDDED_NULL_CHAR)); + BOOST_CHECK(!ToIntegral<T>(" 1")); + BOOST_CHECK(!ToIntegral<T>("1 ")); + BOOST_CHECK(!ToIntegral<T>("1a")); + BOOST_CHECK(!ToIntegral<T>("1.1")); + BOOST_CHECK(!ToIntegral<T>("1.9")); + BOOST_CHECK(!ToIntegral<T>("+01.9")); + BOOST_CHECK(!ToIntegral<T>("-")); + BOOST_CHECK(!ToIntegral<T>("+")); + BOOST_CHECK(!ToIntegral<T>(" -1")); + BOOST_CHECK(!ToIntegral<T>("-1 ")); + BOOST_CHECK(!ToIntegral<T>(" -1 ")); + BOOST_CHECK(!ToIntegral<T>("+1")); + BOOST_CHECK(!ToIntegral<T>(" +1")); + BOOST_CHECK(!ToIntegral<T>(" +1 ")); + BOOST_CHECK(!ToIntegral<T>("+-1")); + BOOST_CHECK(!ToIntegral<T>("-+1")); + BOOST_CHECK(!ToIntegral<T>("++1")); + BOOST_CHECK(!ToIntegral<T>("--1")); + BOOST_CHECK(!ToIntegral<T>("")); + BOOST_CHECK(!ToIntegral<T>("aap")); + BOOST_CHECK(!ToIntegral<T>("0x1")); + BOOST_CHECK(!ToIntegral<T>("-32482348723847471234")); + BOOST_CHECK(!ToIntegral<T>("32482348723847471234")); +} + +BOOST_AUTO_TEST_CASE(test_ToIntegral) +{ + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("1234").value(), 1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("0").value(), 0); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("01234").value(), 1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("00000000000000001234").value(), 1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-00000000000000001234").value(), -1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("00000000000000000000").value(), 0); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-00000000000000000000").value(), 0); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-1234").value(), -1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-1").value(), -1); + + RunToIntegralTests<uint64_t>(); + RunToIntegralTests<int64_t>(); + RunToIntegralTests<uint32_t>(); + RunToIntegralTests<int32_t>(); + RunToIntegralTests<uint16_t>(); + RunToIntegralTests<int16_t>(); + RunToIntegralTests<uint8_t>(); + RunToIntegralTests<int8_t>(); + + BOOST_CHECK(!ToIntegral<int64_t>("-9223372036854775809")); + BOOST_CHECK_EQUAL(ToIntegral<int64_t>("-9223372036854775808").value(), -9'223'372'036'854'775'807LL - 1LL); + BOOST_CHECK_EQUAL(ToIntegral<int64_t>("9223372036854775807").value(), 9'223'372'036'854'775'807); + BOOST_CHECK(!ToIntegral<int64_t>("9223372036854775808")); + + BOOST_CHECK(!ToIntegral<uint64_t>("-1")); + BOOST_CHECK_EQUAL(ToIntegral<uint64_t>("0").value(), 0U); + BOOST_CHECK_EQUAL(ToIntegral<uint64_t>("18446744073709551615").value(), 18'446'744'073'709'551'615ULL); + BOOST_CHECK(!ToIntegral<uint64_t>("18446744073709551616")); + + BOOST_CHECK(!ToIntegral<int32_t>("-2147483649")); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-2147483648").value(), -2'147'483'648LL); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("2147483647").value(), 2'147'483'647); + BOOST_CHECK(!ToIntegral<int32_t>("2147483648")); + + BOOST_CHECK(!ToIntegral<uint32_t>("-1")); + BOOST_CHECK_EQUAL(ToIntegral<uint32_t>("0").value(), 0U); + BOOST_CHECK_EQUAL(ToIntegral<uint32_t>("4294967295").value(), 4'294'967'295U); + BOOST_CHECK(!ToIntegral<uint32_t>("4294967296")); + + BOOST_CHECK(!ToIntegral<int16_t>("-32769")); + BOOST_CHECK_EQUAL(ToIntegral<int16_t>("-32768").value(), -32'768); + BOOST_CHECK_EQUAL(ToIntegral<int16_t>("32767").value(), 32'767); + BOOST_CHECK(!ToIntegral<int16_t>("32768")); + + BOOST_CHECK(!ToIntegral<uint16_t>("-1")); + BOOST_CHECK_EQUAL(ToIntegral<uint16_t>("0").value(), 0U); + BOOST_CHECK_EQUAL(ToIntegral<uint16_t>("65535").value(), 65'535U); + BOOST_CHECK(!ToIntegral<uint16_t>("65536")); + + BOOST_CHECK(!ToIntegral<int8_t>("-129")); + BOOST_CHECK_EQUAL(ToIntegral<int8_t>("-128").value(), -128); + BOOST_CHECK_EQUAL(ToIntegral<int8_t>("127").value(), 127); + BOOST_CHECK(!ToIntegral<int8_t>("128")); + + BOOST_CHECK(!ToIntegral<uint8_t>("-1")); + BOOST_CHECK_EQUAL(ToIntegral<uint8_t>("0").value(), 0U); + BOOST_CHECK_EQUAL(ToIntegral<uint8_t>("255").value(), 255U); + BOOST_CHECK(!ToIntegral<uint8_t>("256")); +} + +BOOST_AUTO_TEST_CASE(test_LocaleIndependentAtoi) +{ + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1234"), 1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("0"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("01234"), 1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1234"), -1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" 1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1 "), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1a"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1.1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1.9"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+01.9"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1"), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" -1"), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1 "), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" -1 "), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" +1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" +1 "), 1); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+-1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-+1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("++1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("--1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(""), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("aap"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("0x1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-32482348723847471234"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("32482348723847471234"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775809"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775808"), -9'223'372'036'854'775'807LL - 1LL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775807"), 9'223'372'036'854'775'807); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775808"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551615"), 18'446'744'073'709'551'615ULL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551616"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483649"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483648"), -2'147'483'648LL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483647"), 2'147'483'647); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483648"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967295"), 4'294'967'295U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967296"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32769"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32768"), -32'768); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32767"), 32'767); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32768"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65535"), 65'535U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65536"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-129"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-128"), -128); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("127"), 127); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("128"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("255"), 255U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("256"), 0U); +} + BOOST_AUTO_TEST_CASE(test_ParseInt64) { int64_t n; @@ -1639,32 +1813,6 @@ BOOST_AUTO_TEST_CASE(test_ParseUInt64) BOOST_CHECK(!ParseUInt64("-1234", &n)); } -BOOST_AUTO_TEST_CASE(test_ParseDouble) -{ - double n; - // Valid values - BOOST_CHECK(ParseDouble("1234", nullptr)); - BOOST_CHECK(ParseDouble("0", &n) && n == 0.0); - BOOST_CHECK(ParseDouble("1234", &n) && n == 1234.0); - BOOST_CHECK(ParseDouble("01234", &n) && n == 1234.0); // no octal - BOOST_CHECK(ParseDouble("2147483647", &n) && n == 2147483647.0); - BOOST_CHECK(ParseDouble("-2147483648", &n) && n == -2147483648.0); - BOOST_CHECK(ParseDouble("-1234", &n) && n == -1234.0); - BOOST_CHECK(ParseDouble("1e6", &n) && n == 1e6); - BOOST_CHECK(ParseDouble("-1e6", &n) && n == -1e6); - // Invalid values - BOOST_CHECK(!ParseDouble("", &n)); - BOOST_CHECK(!ParseDouble(" 1", &n)); // no padding inside - BOOST_CHECK(!ParseDouble("1 ", &n)); - BOOST_CHECK(!ParseDouble("1a", &n)); - BOOST_CHECK(!ParseDouble("aap", &n)); - BOOST_CHECK(!ParseDouble("0x1", &n)); // no hex - BOOST_CHECK(!ParseDouble(STRING_WITH_EMBEDDED_NULL_CHAR, &n)); - // Overflow and underflow - BOOST_CHECK(!ParseDouble("-1e10000", nullptr)); - BOOST_CHECK(!ParseDouble("1e10000", nullptr)); -} - BOOST_AUTO_TEST_CASE(test_FormatParagraph) { BOOST_CHECK_EQUAL(FormatParagraph("", 79, 0), ""); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 8f4ff6815b..8a48d539f8 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -273,7 +273,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) { LOCK(cs_main); for (const auto& tx : txs) { - const MempoolAcceptResult result = AcceptToMemoryPool(m_node.chainman->ActiveChainstate(), *m_node.mempool, tx, false /* bypass_limits */); + const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(tx); BOOST_REQUIRE(result.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 315ef22599..9bb08f774f 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -2,10 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // +#include <chainparams.h> #include <random.h> #include <uint256.h> #include <consensus/validation.h> #include <sync.h> +#include <rpc/blockchain.h> +#include <test/util/chainstate.h> #include <test/util/setup_common.h> #include <validation.h> @@ -73,4 +76,78 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) WITH_LOCK(::cs_main, manager.Unload()); } +//! Test UpdateTip behavior for both active and background chainstates. +//! +//! When run on the background chainstate, UpdateTip should do a subset +//! of what it does for the active chainstate. +BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) +{ + ChainstateManager& chainman = *Assert(m_node.chainman); + uint256 curr_tip = ::g_best_block; + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + mineBlocks(10); + + // After adding some blocks to the tip, best block should have changed. + BOOST_CHECK(::g_best_block != curr_tip); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(chainman.IsSnapshotActive()); + + curr_tip = ::g_best_block; + + // Mine a new block on top of the activated snapshot chainstate. + mineBlocks(1); // Defined in TestChain100Setup. + + // After adding some blocks to the snapshot tip, best block should have changed. + BOOST_CHECK(::g_best_block != curr_tip); + + curr_tip = ::g_best_block; + + BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2); + + CChainState& background_cs{*[&] { + for (CChainState* cs : chainman.GetAll()) { + if (cs != &chainman.ActiveChainstate()) { + return cs; + } + } + assert(false); + }()}; + + // Create a block to append to the validation chain. + std::vector<CMutableTransaction> noTxns; + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, background_cs); + auto pblock = std::make_shared<const CBlock>(validation_block); + BlockValidationState state; + CBlockIndex* pindex = nullptr; + const CChainParams& chainparams = Params(); + bool newblock = false; + + // TODO: much of this is inlined from ProcessNewBlock(); just reuse PNB() + // once it is changed to support multiple chainstates. + { + LOCK(::cs_main); + bool checked = CheckBlock(*pblock, state, chainparams.GetConsensus()); + BOOST_CHECK(checked); + bool accepted = background_cs.AcceptBlock( + pblock, state, &pindex, true, nullptr, &newblock); + BOOST_CHECK(accepted); + } + // UpdateTip is called here + bool block_added = background_cs.ActivateBestChain(state, pblock); + + // Ensure tip is as expected + BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); + + // g_best_block should be unchanged after adding a block to the background + // validation chain. + BOOST_CHECK(block_added); + BOOST_CHECK_EQUAL(curr_tip, ::g_best_block); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 0bd378631b..be9e05a65e 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -8,13 +8,13 @@ #include <random.h> #include <rpc/blockchain.h> #include <sync.h> +#include <test/util/chainstate.h> #include <test/util/setup_common.h> #include <uint256.h> #include <validation.h> #include <validationinterface.h> #include <tinyformat.h> -#include <univalue.h> #include <vector> @@ -44,7 +44,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); - BOOST_CHECK(!manager.IsBackgroundIBD(&c1)); auto all = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); @@ -57,9 +56,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto exp_tip = c1.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip, exp_tip); - auto& validated_cs = manager.ValidatedChainstate(); - BOOST_CHECK_EQUAL(&validated_cs, &c1); - BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); // Create a snapshot-based chainstate. @@ -81,8 +77,8 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); - BOOST_CHECK(manager.IsBackgroundIBD(&c1)); - BOOST_CHECK(!manager.IsBackgroundIBD(&c2)); + BOOST_CHECK_EQUAL(&c2, &manager.ActiveChainstate()); + BOOST_CHECK(&c1 != &manager.ActiveChainstate()); auto all2 = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); @@ -99,16 +95,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // CCoinsViewCache instances. BOOST_CHECK(exp_tip != exp_tip2); - auto& validated_cs2 = manager.ValidatedChainstate(); - BOOST_CHECK_EQUAL(&validated_cs2, &c1); - - auto& validated_chain = manager.ValidatedChain(); - BOOST_CHECK_EQUAL(&validated_chain, &c1.m_chain); - - auto validated_tip = manager.ValidatedTip(); - exp_tip = c1.m_chain.Tip(); - BOOST_CHECK_EQUAL(validated_tip, exp_tip); - // Let scheduler events finish running to avoid accessing memory that is going to be unloaded SyncWithValidationInterfaceQueue(); @@ -168,36 +154,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } -auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; - -template<typename F = decltype(NoMalleation)> -static bool -CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) -{ - // Write out a snapshot to the test's tempdir. - // - int height; - WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); - fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); - FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; - CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; - - UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); - BOOST_TEST_MESSAGE( - "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); - - // Read the written snapshot in and then activate it. - // - FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; - CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; - SnapshotMetadata metadata; - auto_infile >> metadata; - - malleation(auto_infile, metadata); - - return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); -} - //! Test basic snapshot activation. BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) { @@ -321,27 +277,27 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) { LOCK(::cs_main); size_t coins_in_active{0}; - size_t coins_in_ibd{0}; - size_t coins_missing_ibd{0}; + size_t coins_in_background{0}; + size_t coins_missing_from_background{0}; for (CChainState* chainstate : chainman.GetAll()) { BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); CCoinsViewCache& coinscache = chainstate->CoinsTip(); - bool is_ibd = chainman.IsBackgroundIBD(chainstate); + bool is_background = chainstate != &chainman.ActiveChainstate(); for (CTransactionRef& txn : m_coinbase_txns) { COutPoint op{txn->GetHash(), 0}; if (coinscache.HaveCoin(op)) { - (is_ibd ? coins_in_ibd : coins_in_active)++; - } else if (is_ibd) { - coins_missing_ibd++; + (is_background ? coins_in_background : coins_in_active)++; + } else if (is_background) { + coins_missing_from_background++; } } } BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); - BOOST_CHECK_EQUAL(coins_in_ibd, initial_total_coins); - BOOST_CHECK_EQUAL(coins_missing_ibd, new_coins); + BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); } // Snapshot should refuse to load after one has already loaded. diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index 22aafcaa6c..9136c497ea 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -9,7 +9,7 @@ #include <boost/test/unit_test.hpp> -BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, BasicTestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, ChainTestingSetup) //! Test utilities for detecting when we need to flush the coins cache based //! on estimated memory usage. @@ -20,7 +20,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { CTxMemPool mempool; BlockManager blockman{}; - CChainState chainstate{&mempool, blockman}; + CChainState chainstate{&mempool, blockman, *Assert(m_node.chainman)}; chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index a0c2e76f00..ca52ecba2f 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <consensus/amount.h> #include <net.h> #include <signet.h> #include <uint256.h> diff --git a/src/timedata.cpp b/src/timedata.cpp index 354092752d..69d0273e79 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -11,6 +11,7 @@ #include <netaddress.h> #include <node/ui_interface.h> #include <sync.h> +#include <tinyformat.h> #include <util/system.h> #include <util/translation.h> #include <warnings.h> @@ -74,7 +75,7 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) int64_t nMedian = vTimeOffsets.median(); std::vector<int64_t> vSorted = vTimeOffsets.sorted(); // Only let other nodes change our time by so much - int64_t max_adjustment = std::max<int64_t>(0, gArgs.GetArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT)); + int64_t max_adjustment = std::max<int64_t>(0, gArgs.GetIntArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT)); if (nMedian >= -max_adjustment && nMedian <= max_adjustment) { nTimeOffset = nMedian; } else { @@ -98,11 +99,12 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) } if (LogAcceptCategory(BCLog::NET)) { + std::string log_message{"time data samples: "}; for (const int64_t n : vSorted) { - LogPrint(BCLog::NET, "%+d ", n); /* Continued */ + log_message += strprintf("%+d ", n); } - LogPrint(BCLog::NET, "| "); /* Continued */ - LogPrint(BCLog::NET, "nTimeOffset = %+d (%+d minutes)\n", nTimeOffset, nTimeOffset / 60); + log_message += strprintf("| median offset = %+d (%+d minutes)", nTimeOffset, nTimeOffset / 60); + LogPrint(BCLog::NET, "%s\n", log_message); } } } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index bb296456ba..55618a5c57 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -14,6 +14,7 @@ #include <netbase.h> #include <util/readwritefile.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> #include <util/time.h> @@ -83,7 +84,7 @@ void TorControlConnection::readcb(struct bufferevent *bev, void *ctx) if (s.size() < 4) // Short line continue; // <status>(-|+| )<data><CRLF> - self->message.code = atoi(s.substr(0,3)); + self->message.code = LocaleIndependentAtoi<int>(s.substr(0,3)); self->message.lines.push_back(s.substr(4)); char ch = s[3]; // '-','+' or ' ' if (ch == ' ') { @@ -317,7 +318,7 @@ TorController::TorController(struct event_base* _base, const std::string& tor_co // Read service private key if cached std::pair<bool,std::string> pkf = ReadBinaryFile(GetPrivateKeyFile()); if (pkf.first) { - LogPrint(BCLog::TOR, "tor: Reading cached private key from %s\n", GetPrivateKeyFile().string()); + LogPrint(BCLog::TOR, "tor: Reading cached private key from %s\n", fs::PathToString(GetPrivateKeyFile())); private_key = pkf.second; } } @@ -355,9 +356,9 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe service = LookupNumeric(std::string(service_id+".onion"), Params().GetDefaultPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { - LogPrint(BCLog::TOR, "tor: Cached service private key to %s\n", GetPrivateKeyFile().string()); + LogPrint(BCLog::TOR, "tor: Cached service private key to %s\n", fs::PathToString(GetPrivateKeyFile())); } else { - LogPrintf("tor: Error writing service private key to %s\n", GetPrivateKeyFile().string()); + LogPrintf("tor: Error writing service private key to %s\n", fs::PathToString(GetPrivateKeyFile())); } AddLocal(service, LOCAL_MANUAL); // ... onion requested - keep connection open @@ -507,7 +508,7 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro } else if (methods.count("SAFECOOKIE")) { // Cookie: hexdump -e '32/1 "%02x""\n"' ~/.tor/control_auth_cookie LogPrint(BCLog::TOR, "tor: Using SAFECOOKIE authentication, reading cookie authentication from %s\n", cookiefile); - std::pair<bool,std::string> status_cookie = ReadBinaryFile(cookiefile, TOR_COOKIE_SIZE); + std::pair<bool,std::string> status_cookie = ReadBinaryFile(fs::PathFromString(cookiefile), TOR_COOKIE_SIZE); if (status_cookie.first && status_cookie.second.size() == TOR_COOKIE_SIZE) { // _conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), std::bind(&TorController::auth_cb, this, std::placeholders::_1, std::placeholders::_2)); cookie = std::vector<uint8_t>(status_cookie.second.begin(), status_cookie.second.end()); @@ -585,6 +586,7 @@ static std::thread torControlThread; static void TorControlThread(CService onion_service_target) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::TOR_CONTROL); TorController ctrl(gBase, gArgs.GetArg("-torcontrol", DEFAULT_TOR_CONTROL), onion_service_target); event_base_dispatch(gBase); diff --git a/src/txdb.cpp b/src/txdb.cpp index cfa864668a..3839c9083c 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -107,8 +107,8 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { CDBBatch batch(*m_db); size_t count = 0; size_t changed = 0; - size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize); - int crash_simulate = gArgs.GetArg("-dbcrashratio", 0); + size_t batch_size = (size_t)gArgs.GetIntArg("-dbbatchsize", nDefaultDbBatchSize); + int crash_simulate = gArgs.GetIntArg("-dbcrashratio", 0); assert(!hashBlock.IsNull()); uint256 old_tip = GetBestBlock(); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index d5a888ac67..502a27dc6b 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -5,6 +5,7 @@ #include <txmempool.h> +#include <coins.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> @@ -21,23 +22,75 @@ #include <cmath> #include <optional> -CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, unsigned int _entryHeight, - bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp) - : tx(_tx), nFee(_nFee), nTxWeight(GetTransactionWeight(*tx)), nUsageSize(RecursiveDynamicUsage(tx)), nTime(_nTime), entryHeight(_entryHeight), - spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) +// Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. +struct update_descendant_state { - nCountWithDescendants = 1; - nSizeWithDescendants = GetTxSize(); - nModFeesWithDescendants = nFee; + update_descendant_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount) : + modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount) + {} - feeDelta = 0; + void operator() (CTxMemPoolEntry &e) + { e.UpdateDescendantState(modifySize, modifyFee, modifyCount); } - nCountWithAncestors = 1; - nSizeWithAncestors = GetTxSize(); - nModFeesWithAncestors = nFee; - nSigOpCostWithAncestors = sigOpCost; -} + private: + int64_t modifySize; + CAmount modifyFee; + int64_t modifyCount; +}; + +struct update_ancestor_state +{ + update_ancestor_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount, int64_t _modifySigOpsCost) : + modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount), modifySigOpsCost(_modifySigOpsCost) + {} + + void operator() (CTxMemPoolEntry &e) + { e.UpdateAncestorState(modifySize, modifyFee, modifyCount, modifySigOpsCost); } + + private: + int64_t modifySize; + CAmount modifyFee; + int64_t modifyCount; + int64_t modifySigOpsCost; +}; + +struct update_fee_delta +{ + explicit update_fee_delta(int64_t _feeDelta) : feeDelta(_feeDelta) { } + + void operator() (CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } + +private: + int64_t feeDelta; +}; + +struct update_lock_points +{ + explicit update_lock_points(const LockPoints& _lp) : lp(_lp) { } + + void operator() (CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); } + +private: + const LockPoints& lp; +}; + +CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, + int64_t time, unsigned int entry_height, + bool spends_coinbase, int64_t sigops_cost, LockPoints lp) + : tx{tx}, + nFee{fee}, + nTxWeight(GetTransactionWeight(*tx)), + nUsageSize{RecursiveDynamicUsage(tx)}, + nTime{time}, + entryHeight{entry_height}, + spendsCoinbase{spends_coinbase}, + sigOpCost{sigops_cost}, + lockPoints{lp}, + nSizeWithDescendants{GetTxSize()}, + nModFeesWithDescendants{nFee}, + nSizeWithAncestors{GetTxSize()}, + nModFeesWithAncestors{nFee}, + nSigOpCostWithAncestors{sigOpCost} {} void CTxMemPoolEntry::UpdateFeeDelta(int64_t newFeeDelta) { @@ -276,7 +329,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors) { - CTxMemPoolEntry::Parents parents = it->GetMemPoolParents(); + const CTxMemPoolEntry::Parents& parents = it->GetMemPoolParentsConst(); // add or remove this tx as a child of each parent for (const CTxMemPoolEntry& parent : parents) { UpdateChild(mapTx.iterator_to(parent), it, add); @@ -671,16 +724,7 @@ void CTxMemPool::clear() _clear(); } -static void CheckInputsAndUpdateCoins(const CTransaction& tx, CCoinsViewCache& mempoolDuplicate, const int64_t spendheight) -{ - TxValidationState dummy_state; // Not used. CheckTxInputs() should always pass - CAmount txfee = 0; - bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee); - assert(fCheckResult); - UpdateCoins(tx, mempoolDuplicate, std::numeric_limits<int>::max()); -} - -void CTxMemPool::check(CChainState& active_chainstate) const +void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendheight) const { if (m_check_ratio == 0) return; @@ -693,20 +737,16 @@ void CTxMemPool::check(CChainState& active_chainstate) const uint64_t checkTotal = 0; CAmount check_total_fee{0}; uint64_t innerUsage = 0; + uint64_t prev_ancestor_count{0}; - CCoinsViewCache& active_coins_tip = active_chainstate.CoinsTip(); CCoinsViewCache mempoolDuplicate(const_cast<CCoinsViewCache*>(&active_coins_tip)); - const int64_t spendheight = active_chainstate.m_chain.Height() + 1; - std::list<const CTxMemPoolEntry*> waitingOnDependants; - for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { - unsigned int i = 0; + for (const auto& it : GetSortedDepthAndScore()) { checkTotal += it->GetTxSize(); check_total_fee += it->GetFee(); innerUsage += it->DynamicMemoryUsage(); const CTransaction& tx = it->GetTx(); innerUsage += memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); - bool fDependsWait = false; CTxMemPoolEntry::Parents setParentCheck; for (const CTxIn &txin : tx.vin) { // Check that every mempool transaction's inputs refer to available coins, or other mempool tx's. @@ -714,17 +754,17 @@ void CTxMemPool::check(CChainState& active_chainstate) const if (it2 != mapTx.end()) { const CTransaction& tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull()); - fDependsWait = true; setParentCheck.insert(*it2); - } else { - assert(active_coins_tip.HaveCoin(txin.prevout)); } + // We are iterating through the mempool entries sorted in order by ancestor count. + // All parents must have been checked before their children and their coins added to + // the mempoolDuplicate coins cache. + assert(mempoolDuplicate.HaveCoin(txin.prevout)); // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); assert(it3 != mapNextTx.end()); assert(it3->first == &txin.prevout); assert(it3->second == &tx); - i++; } auto comp = [](const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) -> bool { return a.GetTx().GetHash() == b.GetTx().GetHash(); @@ -751,6 +791,9 @@ void CTxMemPool::check(CChainState& active_chainstate) const assert(it->GetSizeWithAncestors() == nSizeCheck); assert(it->GetSigOpCostWithAncestors() == nSigOpCheck); assert(it->GetModFeesWithAncestors() == nFeesCheck); + // Sanity check: we are walking in ascending ancestor count order. + assert(prev_ancestor_count <= it->GetCountWithAncestors()); + prev_ancestor_count = it->GetCountWithAncestors(); // Check children against mapNextTx CTxMemPoolEntry::Children setChildrenCheck; @@ -769,24 +812,12 @@ void CTxMemPool::check(CChainState& active_chainstate) const // just a sanity check, not definitive that this calc is correct... assert(it->GetSizeWithDescendants() >= child_sizes + it->GetTxSize()); - if (fDependsWait) - waitingOnDependants.push_back(&(*it)); - else { - CheckInputsAndUpdateCoins(tx, mempoolDuplicate, spendheight); - } - } - unsigned int stepsSinceLastRemove = 0; - while (!waitingOnDependants.empty()) { - const CTxMemPoolEntry* entry = waitingOnDependants.front(); - waitingOnDependants.pop_front(); - if (!mempoolDuplicate.HaveInputs(entry->GetTx())) { - waitingOnDependants.push_back(entry); - stepsSinceLastRemove++; - assert(stepsSinceLastRemove < waitingOnDependants.size()); - } else { - CheckInputsAndUpdateCoins(entry->GetTx(), mempoolDuplicate, spendheight); - stepsSinceLastRemove = 0; - } + TxValidationState dummy_state; // Not used. CheckTxInputs() should always pass + CAmount txfee = 0; + assert(!tx.IsCoinBase()); + assert(Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee)); + for (const auto& input: tx.vin) mempoolDuplicate.SpendCoin(input.prevout); + AddCoins(mempoolDuplicate, tx, std::numeric_limits<int>::max()); } for (auto it = mapNextTx.cbegin(); it != mapNextTx.cend(); it++) { uint256 hash = it->second->GetHash(); @@ -895,8 +926,6 @@ TxMempoolInfo CTxMemPool::info(const GenTxid& gtxid) const return GetInfo(i); } -TxMempoolInfo CTxMemPool::info(const uint256& txid) const { return info(GenTxid{false, txid}); } - void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { @@ -924,7 +953,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD ++nTransactionsUpdated; } } - LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); + LogPrintf("PrioritiseTransaction: %s fee += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); } void CTxMemPool::ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const @@ -969,7 +998,7 @@ CTxMemPool::setEntries CTxMemPool::GetIterSet(const std::set<uint256>& hashes) c bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const { for (unsigned int i = 0; i < tx.vin.size(); i++) - if (exists(tx.vin[i].prevout.hash)) + if (exists(GenTxid::Txid(tx.vin[i].prevout.hash))) return false; return true; } @@ -1140,7 +1169,7 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpends if (pvNoSpendsRemaining) { for (const CTransaction& tx : txn) { for (const CTxIn& txin : tx.vin) { - if (exists(txin.prevout.hash)) continue; + if (exists(GenTxid::Txid(txin.prevout.hash))) continue; pvNoSpendsRemaining->push_back(txin.prevout); } } @@ -1174,12 +1203,14 @@ uint64_t CTxMemPool::CalculateDescendantMaximum(txiter entry) const { return maximum; } -void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) const { +void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* const ancestorsize, CAmount* const ancestorfees) const { LOCK(cs); auto it = mapTx.find(txid); ancestors = descendants = 0; if (it != mapTx.end()) { ancestors = it->GetCountWithAncestors(); + if (ancestorsize) *ancestorsize = it->GetSizeWithAncestors(); + if (ancestorfees) *ancestorfees = it->GetModFeesWithAncestors(); descendants = CalculateDescendantMaximum(it); } } diff --git a/src/txmempool.h b/src/txmempool.h index 0a84a6e6b1..85417ac3fc 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -14,8 +14,8 @@ #include <utility> #include <vector> -#include <amount.h> #include <coins.h> +#include <consensus/amount.h> #include <indirectmap.h> #include <policy/feerate.h> #include <policy/packages.h> @@ -37,19 +37,16 @@ extern RecursiveMutex cs_main; /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; -struct LockPoints -{ +struct LockPoints { // Will be set to the blockchain height and median time past // values that would be necessary to satisfy all relative locktime // constraints (BIP68) of this tx given our view of block chain history - int height; - int64_t time; + int height{0}; + int64_t time{0}; // As long as the current chain descends from the highest height block // containing one of the inputs used in the calculation, then the cached // values are still valid even after a reorg. - CBlockIndex* maxInputBlock; - - LockPoints() : height(0), time(0), maxInputBlock(nullptr) { } + CBlockIndex* maxInputBlock{nullptr}; }; struct CompareIteratorByHash { @@ -98,27 +95,27 @@ private: const unsigned int entryHeight; //!< Chain height when entering the mempool const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost - int64_t feeDelta; //!< Used for determining the priority of the transaction for mining in a block + int64_t feeDelta{0}; //!< Used for determining the priority of the transaction for mining in a block LockPoints lockPoints; //!< Track the height and time at which tx was final // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these // descendants as well. - uint64_t nCountWithDescendants; //!< number of descendant transactions + uint64_t nCountWithDescendants{1}; //!< number of descendant transactions uint64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) // Analogous statistics for ancestor transactions - uint64_t nCountWithAncestors; + uint64_t nCountWithAncestors{1}; uint64_t nSizeWithAncestors; CAmount nModFeesWithAncestors; int64_t nSigOpCostWithAncestors; public: - CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, unsigned int _entryHeight, - bool spendsCoinbase, - int64_t nSigOpsCost, LockPoints lp); + CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, + int64_t time, unsigned int entry_height, + bool spends_coinbase, + int64_t sigops_cost, LockPoints lp); const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } @@ -162,58 +159,6 @@ public: mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms }; -// Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. -struct update_descendant_state -{ - update_descendant_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount) : - modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount) - {} - - void operator() (CTxMemPoolEntry &e) - { e.UpdateDescendantState(modifySize, modifyFee, modifyCount); } - - private: - int64_t modifySize; - CAmount modifyFee; - int64_t modifyCount; -}; - -struct update_ancestor_state -{ - update_ancestor_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount, int64_t _modifySigOpsCost) : - modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount), modifySigOpsCost(_modifySigOpsCost) - {} - - void operator() (CTxMemPoolEntry &e) - { e.UpdateAncestorState(modifySize, modifyFee, modifyCount, modifySigOpsCost); } - - private: - int64_t modifySize; - CAmount modifyFee; - int64_t modifyCount; - int64_t modifySigOpsCost; -}; - -struct update_fee_delta -{ - explicit update_fee_delta(int64_t _feeDelta) : feeDelta(_feeDelta) { } - - void operator() (CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } - -private: - int64_t feeDelta; -}; - -struct update_lock_points -{ - explicit update_lock_points(const LockPoints& _lp) : lp(_lp) { } - - void operator() (CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); } - -private: - const LockPoints& lp; -}; - // extracts a transaction hash from CTxMemPoolEntry or CTransactionRef struct mempoolentry_txid { @@ -545,7 +490,7 @@ public: * By design, it is guaranteed that: * * 1. Locking both `cs_main` and `mempool.cs` will give a view of mempool - * that is consistent with current chain tip (`::ChainActive()` and + * that is consistent with current chain tip (`ActiveChain()` and * `CoinsTip()`) and is fully populated. Fully populated means that if the * current active chain is missing transactions that were present in a * previously active chain, all the missing transactions will have been @@ -625,7 +570,7 @@ public: * all inputs are in the mapNextTx array). If sanity-checking is turned off, * check does nothing. */ - void check(CChainState& active_chainstate) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void check(const CCoinsViewCache& active_coins_tip, int64_t spendheight) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); // addUnchecked must updated state for all ancestors of a given transaction, // to track size/count of descendant transactions. First version of @@ -748,8 +693,10 @@ public: /** * Calculate the ancestor and descendant count for the given transaction. * The counts include the transaction itself. + * When ancestors is non-zero (ie, the transaction itself is in the mempool), + * ancestorsize and ancestorfees will also be set to the appropriate values. */ - void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) const; + void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) const; /** @returns true if the mempool is fully loaded */ bool IsLoaded() const; @@ -783,7 +730,6 @@ public: } return (mapTx.count(gtxid.GetHash()) != 0); } - bool exists(const uint256& txid) const { return exists(GenTxid{false, txid}); } CTransactionRef get(const uint256& hash) const; txiter get_iter_from_wtxid(const uint256& wtxid) const EXCLUSIVE_LOCKS_REQUIRED(cs) @@ -791,7 +737,6 @@ public: AssertLockHeld(cs); return mapTx.project<0>(mapTx.get<index_by_wtxid>().find(wtxid)); } - TxMempoolInfo info(const uint256& hash) const; TxMempoolInfo info(const GenTxid& gtxid) const; std::vector<TxMempoolInfo> infoAll() const; @@ -803,7 +748,7 @@ public: LOCK(cs); // Sanity check the transaction is in the mempool & insert into // unbroadcast set. - if (exists(txid)) m_unbroadcast_txids.insert(txid); + if (exists(GenTxid::Txid(txid))) m_unbroadcast_txids.insert(txid); }; /** Removes a transaction from the unbroadcast set */ diff --git a/src/txrequest.cpp b/src/txrequest.cpp index f8d7a1ece8..7d478a5b26 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -300,7 +300,7 @@ std::map<uint256, TxHashInfo> ComputeTxHashInfo(const Index& index, const Priori GenTxid ToGenTxid(const Announcement& ann) { - return {ann.m_is_wtxid, ann.m_txhash}; + return ann.m_is_wtxid ? GenTxid::Wtxid(ann.m_txhash) : GenTxid::Txid(ann.m_txhash); } } // namespace diff --git a/src/univalue/.cirrus.yml b/src/univalue/.cirrus.yml new file mode 100644 index 0000000000..f140fee12b --- /dev/null +++ b/src/univalue/.cirrus.yml @@ -0,0 +1,44 @@ +env: + MAKEJOBS: "-j4" + RUN_TESTS: "true" + BASE_OUTDIR: "$CIRRUS_WORKING_DIR/out_dir_base" + DEBIAN_FRONTEND: "noninteractive" + +task: + container: + image: ubuntu:focal + cpu: 1 + memory: 1G + greedy: true # https://medium.com/cirruslabs/introducing-greedy-container-instances-29aad06dc2b4 + + matrix: + - name: "gcc" + env: + CC: "gcc" + CXX: "g++" + APT_PKGS: "gcc" + - name: "clang" + env: + CC: "clang" + CXX: "clang++" + APT_PKGS: "clang" + - name: "mingw" + env: + CC: "" + CXX: "" + UNIVALUE_CONFIG: "--host=x86_64-w64-mingw32" + APT_PKGS: "g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64" + RUN_TESTS: "false" + + install_script: + - apt update + - apt install -y pkg-config build-essential libtool autotools-dev automake bsdmainutils + - apt install -y $APT_PKGS + autogen_script: + - ./autogen.sh + configure_script: + - ./configure --cache-file=config.cache --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib $UNIVALUE_CONFIG + make_script: + - make $MAKEJOBS V=1 + test_script: + - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS distcheck; fi diff --git a/src/univalue/.travis.yml b/src/univalue/.travis.yml deleted file mode 100644 index 43a1ed362e..0000000000 --- a/src/univalue/.travis.yml +++ /dev/null @@ -1,51 +0,0 @@ -language: cpp - -compiler: - - clang - - gcc - -os: - - linux - - osx - -sudo: false - -env: - global: - - MAKEJOBS=-j3 - - RUN_TESTS=true - - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - -cache: - apt: true - -addons: - apt: - packages: - - pkg-config - -before_script: - - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh - -script: - - if [ -n "$UNIVALUE_CONFIG" ]; then unset CC; unset CXX; fi - - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - - UNIVALUE_CONFIG_ALL="--prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" - - ./configure --cache-file=config.cache $UNIVALUE_CONFIG_ALL $UNIVALUE_CONFIG || ( cat config.log && false) - - make -s $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL ; false ) - - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS distcheck; fi - -matrix: - fast_finish: true - include: - - os: linux - compiler: gcc - env: UNIVALUE_CONFIG=--host=x86_64-w64-mingw32 RUN_TESTS=false - addons: - apt: - packages: - - g++-mingw-w64-x86-64 - - gcc-mingw-w64-x86-64 - - binutils-mingw-w64-x86-64 diff --git a/src/univalue/Makefile.am b/src/univalue/Makefile.am index 0f5ba59954..476f14b922 100644 --- a/src/univalue/Makefile.am +++ b/src/univalue/Makefile.am @@ -1,20 +1,17 @@ +include sources.mk ACLOCAL_AMFLAGS = -I build-aux/m4 -.PHONY: gen +.PHONY: gen FORCE .INTERMEDIATE: $(GENBIN) -include_HEADERS = include/univalue.h -noinst_HEADERS = lib/univalue_escapes.h lib/univalue_utffilter.h +include_HEADERS = $(UNIVALUE_DIST_HEADERS_INT) +noinst_HEADERS = $(UNIVALUE_LIB_HEADERS_INT) lib_LTLIBRARIES = libunivalue.la pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = pc/libunivalue.pc -libunivalue_la_SOURCES = \ - lib/univalue.cpp \ - lib/univalue_get.cpp \ - lib/univalue_read.cpp \ - lib/univalue_write.cpp +libunivalue_la_SOURCES = $(UNIVALUE_LIB_SOURCES_INT) libunivalue_la_LDFLAGS = \ -version-info $(LIBUNIVALUE_CURRENT):$(LIBUNIVALUE_REVISION):$(LIBUNIVALUE_AGE) \ @@ -30,89 +27,32 @@ $(GENBIN): $(GEN_SRCS) @echo Building $@ $(AM_V_at)c++ -I$(top_srcdir)/include -o $@ $< -gen: lib/univalue_escapes.h $(GENBIN) - @echo Updating $< +gen: $(GENBIN) FORCE + @echo Updating lib/univalue_escapes.h $(AM_V_at)$(GENBIN) > lib/univalue_escapes.h noinst_PROGRAMS = $(TESTS) test/test_json -TEST_DATA_DIR=test - -test_unitester_SOURCES = test/unitester.cpp +test_unitester_SOURCES = $(UNIVALUE_TEST_UNITESTER_INT) test_unitester_LDADD = libunivalue.la -test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(TEST_DATA_DIR)\" +test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_test_json_SOURCES = test/test_json.cpp +test_test_json_SOURCES = $(UNIVALUE_TEST_JSON_INT) test_test_json_LDADD = libunivalue.la test_test_json_CXXFLAGS = -I$(top_srcdir)/include test_test_json_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_no_nul_SOURCES = test/no_nul.cpp +test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) test_no_nul_LDADD = libunivalue.la test_no_nul_CXXFLAGS = -I$(top_srcdir)/include test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_object_SOURCES = test/object.cpp +test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) test_object_LDADD = libunivalue.la test_object_CXXFLAGS = -I$(top_srcdir)/include test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -TEST_FILES = \ - $(TEST_DATA_DIR)/fail10.json \ - $(TEST_DATA_DIR)/fail11.json \ - $(TEST_DATA_DIR)/fail12.json \ - $(TEST_DATA_DIR)/fail13.json \ - $(TEST_DATA_DIR)/fail14.json \ - $(TEST_DATA_DIR)/fail15.json \ - $(TEST_DATA_DIR)/fail16.json \ - $(TEST_DATA_DIR)/fail17.json \ - $(TEST_DATA_DIR)/fail18.json \ - $(TEST_DATA_DIR)/fail19.json \ - $(TEST_DATA_DIR)/fail1.json \ - $(TEST_DATA_DIR)/fail20.json \ - $(TEST_DATA_DIR)/fail21.json \ - $(TEST_DATA_DIR)/fail22.json \ - $(TEST_DATA_DIR)/fail23.json \ - $(TEST_DATA_DIR)/fail24.json \ - $(TEST_DATA_DIR)/fail25.json \ - $(TEST_DATA_DIR)/fail26.json \ - $(TEST_DATA_DIR)/fail27.json \ - $(TEST_DATA_DIR)/fail28.json \ - $(TEST_DATA_DIR)/fail29.json \ - $(TEST_DATA_DIR)/fail2.json \ - $(TEST_DATA_DIR)/fail30.json \ - $(TEST_DATA_DIR)/fail31.json \ - $(TEST_DATA_DIR)/fail32.json \ - $(TEST_DATA_DIR)/fail33.json \ - $(TEST_DATA_DIR)/fail34.json \ - $(TEST_DATA_DIR)/fail35.json \ - $(TEST_DATA_DIR)/fail36.json \ - $(TEST_DATA_DIR)/fail37.json \ - $(TEST_DATA_DIR)/fail38.json \ - $(TEST_DATA_DIR)/fail39.json \ - $(TEST_DATA_DIR)/fail40.json \ - $(TEST_DATA_DIR)/fail41.json \ - $(TEST_DATA_DIR)/fail42.json \ - $(TEST_DATA_DIR)/fail44.json \ - $(TEST_DATA_DIR)/fail45.json \ - $(TEST_DATA_DIR)/fail3.json \ - $(TEST_DATA_DIR)/fail4.json \ - $(TEST_DATA_DIR)/fail5.json \ - $(TEST_DATA_DIR)/fail6.json \ - $(TEST_DATA_DIR)/fail7.json \ - $(TEST_DATA_DIR)/fail8.json \ - $(TEST_DATA_DIR)/fail9.json \ - $(TEST_DATA_DIR)/pass1.json \ - $(TEST_DATA_DIR)/pass2.json \ - $(TEST_DATA_DIR)/pass3.json \ - $(TEST_DATA_DIR)/pass4.json \ - $(TEST_DATA_DIR)/round1.json \ - $(TEST_DATA_DIR)/round2.json \ - $(TEST_DATA_DIR)/round3.json \ - $(TEST_DATA_DIR)/round4.json \ - $(TEST_DATA_DIR)/round5.json \ - $(TEST_DATA_DIR)/round6.json \ - $(TEST_DATA_DIR)/round7.json - -EXTRA_DIST=$(TEST_FILES) $(GEN_SRCS) +TEST_FILES = $(UNIVALUE_TEST_FILES_INT) + +EXTRA_DIST=$(UNIVALUE_TEST_FILES_INT) $(GEN_SRCS) diff --git a/src/univalue/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/src/univalue/build-aux/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 0000000000..f7e5137003 --- /dev/null +++ b/src/univalue/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,962 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for no added switch, and then for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com> +# Copyright (c) 2012 Zack Weinberg <zackw@panix.com> +# Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu> +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com> +# Copyright (c) 2015 Paul Norman <penorman@mac.com> +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> +# Copyright (c) 2020 Jason Merrill <jason@redhat.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 12 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [], [dnl + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi]) + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template <typename T> + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check<void> single_type; + typedef check<check<void>> double_type; + typedef check<check<check<void>>> triple_type; + typedef check<check<check<check<void>>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same<int, decltype(0)>::value == true, ""); + static_assert(is_same<int, decltype(c)>::value == false, ""); + static_assert(is_same<int, decltype(v)>::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same<int, decltype(ac)>::value == true, ""); + static_assert(is_same<int, decltype(av)>::value == true, ""); + static_assert(is_same<int, decltype(sumi)>::value == true, ""); + static_assert(is_same<int, decltype(sumf)>::value == false, ""); + static_assert(is_same<int, decltype(add(c, v))>::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template <int...> + struct sum; + + template <int N0, int... N1toN> + struct sum<N0, N1toN...> + { + static constexpr auto value = N0 + sum<N1toN...>::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template<typename T> + using member = typename T::member_type; + + template<typename T> + void func(...) {} + + template<typename T> + void func(member<T>*) {} + + void test(); + + void test() { func<foo>(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same<int, decltype(f(x))>::value, ""); + static_assert(is_same<int&, decltype(g(x))>::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include <initializer_list> +#include <utility> +#include <type_traits> + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template<typename... Args> + int multiply(Args... args) + { + return (args * ... * 1); + } + + template<typename... Args> + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value); + static_assert(std::is_same<int, decltype(bar)>::value); + } + + namespace test_typename_in_template_template_parameter + { + + template<template<typename> typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template <bool cond> + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template <typename T1, typename T2> + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template <auto n> + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair<int, int> pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair<int, int>& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template<typename T> + Bad + f(T*, T*); + + template<typename T1, typename T2> + Good + f(T1*, T2*); + + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>); + + } + + namespace test_inline_variables + { + + template<class T> void f(T) + {} + + template<class T> inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/src/univalue/configure.ac b/src/univalue/configure.ac index 8298332ac1..495b25a53d 100644 --- a/src/univalue/configure.ac +++ b/src/univalue/configure.ac @@ -1,7 +1,7 @@ m4_define([libunivalue_major_version], [1]) m4_define([libunivalue_minor_version], [1]) -m4_define([libunivalue_micro_version], [3]) -m4_define([libunivalue_interface_age], [3]) +m4_define([libunivalue_micro_version], [4]) +m4_define([libunivalue_interface_age], [4]) # If you need a modifier for the version number. # Normally empty, but can be used to make "fixup" releases. m4_define([libunivalue_extraversion], []) @@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()]) -AC_INIT([univalue], [1.0.3], +AC_INIT([univalue], [1.0.4], [http://github.com/jgarzik/univalue/]) dnl make the compilation flags quiet unless V=1 is used @@ -45,6 +45,9 @@ AC_SUBST(LIBUNIVALUE_AGE) LT_INIT LT_LANG([C++]) +dnl Require C++11 compiler (no GNU extensions) +AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) + case $host in *mingw*) LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static" diff --git a/src/univalue/gen/gen.cpp b/src/univalue/gen/gen.cpp index 85fe20924a..b8a6c73f4e 100644 --- a/src/univalue/gen/gen.cpp +++ b/src/univalue/gen/gen.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. // // To re-create univalue_escapes.h: @@ -45,7 +45,7 @@ static void outputEscape() for (unsigned int i = 0; i < 256; i++) { if (escapes[i].empty()) { - printf("\tNULL,\n"); + printf("\tnullptr,\n"); } else { printf("\t\""); diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 048e162f7d..fc5cf402be 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #ifndef __UNIVALUE_H__ #define __UNIVALUE_H__ @@ -14,8 +14,6 @@ #include <map> #include <cassert> -#include <sstream> // .get_int64() - class UniValue { public: enum VType { VNULL, VOBJ, VARR, VSTR, VNUM, VBOOL, }; diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 4c9c15d63e..c4e59fae74 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <stdint.h> #include <iomanip> @@ -178,17 +178,19 @@ bool UniValue::findKey(const std::string& key, size_t& retIdx) const bool UniValue::checkObject(const std::map<std::string,UniValue::VType>& t) const { - if (typ != VOBJ) + if (typ != VOBJ) { return false; + } - for (std::map<std::string,UniValue::VType>::const_iterator it = t.begin(); - it != t.end(); ++it) { + for (const auto& object: t) { size_t idx = 0; - if (!findKey(it->first, idx)) + if (!findKey(object.first, idx)) { return false; + } - if (values.at(idx).getType() != it->second) + if (values.at(idx).getType() != object.second) { return false; + } } return true; @@ -228,7 +230,7 @@ const char *uvTypeName(UniValue::VType t) } // not reached - return NULL; + return nullptr; } const UniValue& find_value(const UniValue& obj, const std::string& name) diff --git a/src/univalue/lib/univalue_escapes.h b/src/univalue/lib/univalue_escapes.h index 74596aab6d..3f714f8e5b 100644 --- a/src/univalue/lib/univalue_escapes.h +++ b/src/univalue/lib/univalue_escapes.h @@ -34,229 +34,229 @@ static const char *escapes[256] = { "\\u001d", "\\u001e", "\\u001f", - NULL, - NULL, + nullptr, + nullptr, "\\\"", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, "\\\\", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, "\\u007f", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, }; #endif // BITCOIN_UNIVALUE_UNIVALUE_ESCAPES_H diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp index 0ad6146545..5af89a3561 100644 --- a/src/univalue/lib/univalue_get.cpp +++ b/src/univalue/lib/univalue_get.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <stdint.h> #include <errno.h> @@ -11,6 +11,7 @@ #include <vector> #include <limits> #include <string> +#include <sstream> #include "univalue.h" @@ -31,7 +32,7 @@ bool ParseInt32(const std::string& str, int32_t *out) { if (!ParsePrechecks(str)) return false; - char *endp = NULL; + char *endp = nullptr; errno = 0; // strtol will not set errno if valid long int n = strtol(str.c_str(), &endp, 10); if(out) *out = (int32_t)n; @@ -47,7 +48,7 @@ bool ParseInt64(const std::string& str, int64_t *out) { if (!ParsePrechecks(str)) return false; - char *endp = NULL; + char *endp = nullptr; errno = 0; // strtoll will not set errno if valid long long int n = strtoll(str.c_str(), &endp, 10); if(out) *out = (int64_t)n; diff --git a/src/univalue/lib/univalue_read.cpp b/src/univalue/lib/univalue_read.cpp index 5c6a1acf75..be39bfe57a 100644 --- a/src/univalue/lib/univalue_read.cpp +++ b/src/univalue/lib/univalue_read.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <string.h> #include <vector> @@ -227,7 +227,7 @@ enum jtokentype getJsonToken(std::string& tokenVal, unsigned int& consumed, } else { - writer.push_back(*raw); + writer.push_back(static_cast<unsigned char>(*raw)); raw++; } } @@ -244,7 +244,7 @@ enum jtokentype getJsonToken(std::string& tokenVal, unsigned int& consumed, } } -enum expect_bits { +enum expect_bits : unsigned { EXP_OBJ_NAME = (1U << 0), EXP_COLON = (1U << 1), EXP_ARR_VALUE = (1U << 2), diff --git a/src/univalue/lib/univalue_utffilter.h b/src/univalue/lib/univalue_utffilter.h index 20d4043009..c24ac58eaf 100644 --- a/src/univalue/lib/univalue_utffilter.h +++ b/src/univalue/lib/univalue_utffilter.h @@ -1,6 +1,6 @@ // Copyright 2016 Wladimir J. van der Laan // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #ifndef UNIVALUE_UTFFILTER_H #define UNIVALUE_UTFFILTER_H diff --git a/src/univalue/lib/univalue_write.cpp b/src/univalue/lib/univalue_write.cpp index 827eb9b271..3a2c580c7f 100644 --- a/src/univalue/lib/univalue_write.cpp +++ b/src/univalue/lib/univalue_write.cpp @@ -1,9 +1,8 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <iomanip> -#include <sstream> #include <stdio.h> #include "univalue.h" #include "univalue_escapes.h" @@ -14,13 +13,13 @@ static std::string json_escape(const std::string& inS) outS.reserve(inS.size() * 2); for (unsigned int i = 0; i < inS.size(); i++) { - unsigned char ch = inS[i]; + unsigned char ch = static_cast<unsigned char>(inS[i]); const char *escStr = escapes[ch]; if (escStr) outS += escStr; else - outS += ch; + outS += static_cast<char>(ch); } return outS; diff --git a/src/univalue/sources.mk b/src/univalue/sources.mk new file mode 100644 index 0000000000..efab6d277f --- /dev/null +++ b/src/univalue/sources.mk @@ -0,0 +1,95 @@ +# - All variables are namespaced with UNIVALUE_ to avoid colliding with +# downstream makefiles. +# - All Variables ending in _HEADERS or _SOURCES confuse automake, so the +# _INT postfix is applied. +# - Convenience variables, for example a UNIVALUE_TEST_DIR should not be used +# as they interfere with automatic dependency generation +# - The %reldir% is the relative path from the Makefile.am. This allows +# downstreams to use these variables without having to manually account for +# the path change. + +UNIVALUE_INCLUDE_DIR_INT = %reldir%/include + +UNIVALUE_DIST_HEADERS_INT = +UNIVALUE_DIST_HEADERS_INT += %reldir%/include/univalue.h + +UNIVALUE_LIB_HEADERS_INT = +UNIVALUE_LIB_HEADERS_INT += %reldir%/lib/univalue_utffilter.h +UNIVALUE_LIB_HEADERS_INT += %reldir%/lib/univalue_escapes.h + +UNIVALUE_LIB_SOURCES_INT = +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_get.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_read.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_write.cpp + +UNIVALUE_TEST_DATA_DIR_INT = %reldir%/test + +UNIVALUE_TEST_UNITESTER_INT = +UNIVALUE_TEST_UNITESTER_INT += %reldir%/test/unitester.cpp + +UNIVALUE_TEST_JSON_INT = +UNIVALUE_TEST_JSON_INT += %reldir%/test/test_json.cpp + +UNIVALUE_TEST_NO_NUL_INT = +UNIVALUE_TEST_NO_NUL_INT += %reldir%/test/no_nul.cpp + +UNIVALUE_TEST_OBJECT_INT = +UNIVALUE_TEST_OBJECT_INT += %reldir%/test/object.cpp + +UNIVALUE_TEST_FILES_INT = +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail5.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail6.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail7.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail8.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail9.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail10.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail11.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail12.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail13.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail14.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail15.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail16.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail17.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail18.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail19.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail20.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail21.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail22.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail23.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail24.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail25.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail26.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail27.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail28.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail29.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail30.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail31.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail32.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail33.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail34.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail35.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail36.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail37.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail38.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail39.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail40.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail41.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail42.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail44.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail45.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round5.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round6.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round7.json diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index ccc1344836..c2f52f83ac 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -1,7 +1,7 @@ // Copyright (c) 2014 BitPay Inc. // Copyright (c) 2014-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <stdint.h> #include <vector> diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 2308afbcdf..02e1a83c6d 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <stdlib.h> #include <stdio.h> @@ -58,7 +58,7 @@ static void runtest_file(const char *filename_) std::string basename(filename_); std::string filename = srcdir + "/" + basename; FILE *f = fopen(filename.c_str(), "r"); - assert(f != NULL); + assert(f != nullptr); std::string jdata; diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index 5695c62012..b696c65e9d 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -201,7 +201,7 @@ std::vector<bool> DecodeAsmap(fs::path path) } fseek(filestr, 0, SEEK_END); int length = ftell(filestr); - LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length); + LogPrintf("Opened asmap file %s (%d bytes) from disk\n", fs::quoted(fs::PathToString(path)), length); fseek(filestr, 0, SEEK_SET); uint8_t cur_byte; for (int i = 0; i < length; ++i) { @@ -211,7 +211,7 @@ std::vector<bool> DecodeAsmap(fs::path path) } } if (!SanityCheckASMap(bits, 128)) { - LogPrintf("Sanity check of asmap file %s failed\n", path); + LogPrintf("Sanity check of asmap file %s failed\n", fs::quoted(fs::PathToString(path))); return {}; } return bits; diff --git a/src/util/error.cpp b/src/util/error.cpp index 48c81693f3..d019ba018d 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -20,9 +20,9 @@ bilingual_str TransactionErrorString(const TransactionError err) case TransactionError::P2P_DISABLED: return Untranslated("Peer-to-peer functionality missing or disabled"); case TransactionError::MEMPOOL_REJECTED: - return Untranslated("Transaction rejected by AcceptToMemoryPool"); + return Untranslated("Transaction rejected by mempool"); case TransactionError::MEMPOOL_ERROR: - return Untranslated("AcceptToMemoryPool failed"); + return Untranslated("Mempool internal error"); case TransactionError::INVALID_PSBT: return Untranslated("PSBT is not well-formed"); case TransactionError::PSBT_MISMATCH: diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index d3f4029607..1aed7daacf 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -5,7 +5,7 @@ #include <util/moneystr.h> -#include <amount.h> +#include <consensus/amount.h> #include <tinyformat.h> #include <util/strencodings.h> #include <util/string.h> @@ -77,8 +77,7 @@ std::optional<CAmount> ParseMoney(const std::string& money_string) return std::nullopt; if (nUnits < 0 || nUnits > COIN) return std::nullopt; - int64_t nWhole = atoi64(strWhole); - + int64_t nWhole = LocaleIndependentAtoi<int64_t>(strWhole); CAmount value = nWhole * COIN + nUnits; if (!MoneyRange(value)) { diff --git a/src/util/moneystr.h b/src/util/moneystr.h index b71dffd0db..f37dc1cffd 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -9,8 +9,8 @@ #ifndef BITCOIN_UTIL_MONEYSTR_H #define BITCOIN_UTIL_MONEYSTR_H -#include <amount.h> #include <attributes.h> +#include <consensus/amount.h> #include <optional> #include <string> diff --git a/src/util/rbf.h b/src/util/rbf.h index 6d44a2cb83..aa522d8bfb 100644 --- a/src/util/rbf.h +++ b/src/util/rbf.h @@ -9,7 +9,7 @@ class CTransaction; -static const uint32_t MAX_BIP125_RBF_SEQUENCE = 0xfffffffd; +static constexpr uint32_t MAX_BIP125_RBF_SEQUENCE{0xfffffffd}; /** Check whether the sequence numbers on this transaction are signaling opt-in to replace-by-fee, * according to BIP 125. Allow opt-out of transaction replacement by setting nSequence > @@ -17,7 +17,7 @@ static const uint32_t MAX_BIP125_RBF_SEQUENCE = 0xfffffffd; * * SEQUENCE_FINAL-1 is picked to still allow use of nLockTime by non-replaceable transactions. All * inputs rather than just one is for the sake of multi-party protocols, where we don't want a single -* party to be able to disable replacement. */ -bool SignalsOptInRBF(const CTransaction &tx); +* party to be able to disable replacement by opting out in their own input. */ +bool SignalsOptInRBF(const CTransaction& tx); #endif // BITCOIN_UTIL_RBF_H diff --git a/src/util/settings.cpp b/src/util/settings.cpp index 846b34089d..7fb35c073e 100644 --- a/src/util/settings.cpp +++ b/src/util/settings.cpp @@ -66,24 +66,24 @@ bool ReadSettings(const fs::path& path, std::map<std::string, SettingsValue>& va fsbridge::ifstream file; file.open(path); if (!file.is_open()) { - errors.emplace_back(strprintf("%s. Please check permissions.", path.string())); + errors.emplace_back(strprintf("%s. Please check permissions.", fs::PathToString(path))); return false; } SettingsValue in; if (!in.read(std::string{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()})) { - errors.emplace_back(strprintf("Unable to parse settings file %s", path.string())); + errors.emplace_back(strprintf("Unable to parse settings file %s", fs::PathToString(path))); return false; } if (file.fail()) { - errors.emplace_back(strprintf("Failed reading settings file %s", path.string())); + errors.emplace_back(strprintf("Failed reading settings file %s", fs::PathToString(path))); return false; } file.close(); // Done with file descriptor. Release while copying data. if (!in.isObject()) { - errors.emplace_back(strprintf("Found non-object value %s in settings file %s", in.write(), path.string())); + errors.emplace_back(strprintf("Found non-object value %s in settings file %s", in.write(), fs::PathToString(path))); return false; } @@ -92,7 +92,7 @@ bool ReadSettings(const fs::path& path, std::map<std::string, SettingsValue>& va for (size_t i = 0; i < in_keys.size(); ++i) { auto inserted = values.emplace(in_keys[i], in_values[i]); if (!inserted.second) { - errors.emplace_back(strprintf("Found duplicate key %s in settings file %s", in_keys[i], path.string())); + errors.emplace_back(strprintf("Found duplicate key %s in settings file %s", in_keys[i], fs::PathToString(path))); } } return errors.empty(); @@ -109,7 +109,7 @@ bool WriteSettings(const fs::path& path, fsbridge::ofstream file; file.open(path); if (file.fail()) { - errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", path.string())); + errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", fs::PathToString(path))); return false; } file << out.write(/* prettyIndent= */ 1, /* indentLevel= */ 4) << std::endl; diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index f514613f0d..15bd07b374 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -11,8 +11,7 @@ #include <algorithm> #include <cstdlib> #include <cstring> -#include <errno.h> -#include <limits> +#include <optional> static const std::string CHARS_ALPHA_NUM = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; @@ -282,118 +281,55 @@ std::string DecodeBase32(const std::string& str, bool* pf_invalid) return std::string((const char*)vchRet.data(), vchRet.size()); } -[[nodiscard]] static bool ParsePrechecks(const std::string& str) +namespace { +template <typename T> +bool ParseIntegral(const std::string& str, T* out) { - if (str.empty()) // No empty string allowed + static_assert(std::is_integral<T>::value); + // Replicate the exact behavior of strtol/strtoll/strtoul/strtoull when + // handling leading +/- for backwards compatibility. + if (str.length() >= 2 && str[0] == '+' && str[1] == '-') { return false; - if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size()-1]))) // No padding allowed - return false; - if (!ValidAsCString(str)) // No embedded NUL characters allowed + } + const std::optional<T> opt_int = ToIntegral<T>((!str.empty() && str[0] == '+') ? str.substr(1) : str); + if (!opt_int) { return false; + } + if (out != nullptr) { + *out = *opt_int; + } return true; } +}; // namespace -bool ParseInt32(const std::string& str, int32_t *out) +bool ParseInt32(const std::string& str, int32_t* out) { - if (!ParsePrechecks(str)) - return false; - char *endp = nullptr; - errno = 0; // strtol will not set errno if valid - long int n = strtol(str.c_str(), &endp, 10); - if(out) *out = (int32_t)n; - // Note that strtol returns a *long int*, so even if strtol doesn't report an over/underflow - // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit - // platforms the size of these types may be different. - return endp && *endp == 0 && !errno && - n >= std::numeric_limits<int32_t>::min() && - n <= std::numeric_limits<int32_t>::max(); + return ParseIntegral<int32_t>(str, out); } -bool ParseInt64(const std::string& str, int64_t *out) +bool ParseInt64(const std::string& str, int64_t* out) { - if (!ParsePrechecks(str)) - return false; - char *endp = nullptr; - errno = 0; // strtoll will not set errno if valid - long long int n = strtoll(str.c_str(), &endp, 10); - if(out) *out = (int64_t)n; - // Note that strtoll returns a *long long int*, so even if strtol doesn't report an over/underflow - // we still have to check that the returned value is within the range of an *int64_t*. - return endp && *endp == 0 && !errno && - n >= std::numeric_limits<int64_t>::min() && - n <= std::numeric_limits<int64_t>::max(); + return ParseIntegral<int64_t>(str, out); } -bool ParseUInt8(const std::string& str, uint8_t *out) +bool ParseUInt8(const std::string& str, uint8_t* out) { - uint32_t u32; - if (!ParseUInt32(str, &u32) || u32 > std::numeric_limits<uint8_t>::max()) { - return false; - } - if (out != nullptr) { - *out = static_cast<uint8_t>(u32); - } - return true; + return ParseIntegral<uint8_t>(str, out); } bool ParseUInt16(const std::string& str, uint16_t* out) { - uint32_t u32; - if (!ParseUInt32(str, &u32) || u32 > std::numeric_limits<uint16_t>::max()) { - return false; - } - if (out != nullptr) { - *out = static_cast<uint16_t>(u32); - } - return true; + return ParseIntegral<uint16_t>(str, out); } -bool ParseUInt32(const std::string& str, uint32_t *out) +bool ParseUInt32(const std::string& str, uint32_t* out) { - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoul accepts these by default if they fit in the range - return false; - char *endp = nullptr; - errno = 0; // strtoul will not set errno if valid - unsigned long int n = strtoul(str.c_str(), &endp, 10); - if(out) *out = (uint32_t)n; - // Note that strtoul returns a *unsigned long int*, so even if it doesn't report an over/underflow - // we still have to check that the returned value is within the range of an *uint32_t*. On 64-bit - // platforms the size of these types may be different. - return endp && *endp == 0 && !errno && - n <= std::numeric_limits<uint32_t>::max(); -} - -bool ParseUInt64(const std::string& str, uint64_t *out) -{ - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoull accepts these by default if they fit in the range - return false; - char *endp = nullptr; - errno = 0; // strtoull will not set errno if valid - unsigned long long int n = strtoull(str.c_str(), &endp, 10); - if(out) *out = (uint64_t)n; - // Note that strtoull returns a *unsigned long long int*, so even if it doesn't report an over/underflow - // we still have to check that the returned value is within the range of an *uint64_t*. - return endp && *endp == 0 && !errno && - n <= std::numeric_limits<uint64_t>::max(); + return ParseIntegral<uint32_t>(str, out); } - -bool ParseDouble(const std::string& str, double *out) +bool ParseUInt64(const std::string& str, uint64_t* out) { - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed - return false; - std::istringstream text(str); - text.imbue(std::locale::classic()); - double result; - text >> result; - if(out) *out = result; - return text.eof() && !text.fail(); + return ParseIntegral<uint64_t>(str, out); } std::string FormatParagraph(const std::string& in, size_t width, size_t indent) @@ -437,20 +373,6 @@ std::string FormatParagraph(const std::string& in, size_t width, size_t indent) return out.str(); } -int64_t atoi64(const std::string& str) -{ -#ifdef _MSC_VER - return _atoi64(str.c_str()); -#else - return strtoll(str.c_str(), nullptr, 10); -#endif -} - -int atoi(const std::string& str) -{ - return atoi(str.c_str()); -} - /** Upper bound for mantissa. * 10^18-1 is the largest arbitrary decimal that will fit in a signed 64-bit integer. * Larger integers cannot consist of arbitrary combinations of 0-9: diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 26dc0a0ce3..eedb5ec2f8 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -11,9 +11,12 @@ #include <attributes.h> #include <span.h> +#include <util/string.h> +#include <charconv> #include <cstdint> #include <iterator> +#include <optional> #include <string> #include <vector> @@ -66,8 +69,33 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad = true); std::string EncodeBase32(const std::string& str, bool pad = true); void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut); -int64_t atoi64(const std::string& str); -int atoi(const std::string& str); + +// LocaleIndependentAtoi is provided for backwards compatibility reasons. +// +// New code should use ToIntegral or the ParseInt* functions +// which provide parse error feedback. +// +// The goal of LocaleIndependentAtoi is to replicate the exact defined behaviour +// of atoi and atoi64 as they behave under the "C" locale. +template <typename T> +T LocaleIndependentAtoi(const std::string& str) +{ + static_assert(std::is_integral<T>::value); + T result; + // Emulate atoi(...) handling of white space and leading +/-. + std::string s = TrimString(str); + if (!s.empty() && s[0] == '+') { + if (s.length() >= 2 && s[1] == '-') { + return 0; + } + s = s.substr(1); + } + auto [_, error_condition] = std::from_chars(s.data(), s.data() + s.size(), result); + if (error_condition != std::errc{}) { + return 0; + } + return result; +} /** * Tests if the given character is a decimal digit. @@ -95,6 +123,26 @@ constexpr inline bool IsSpace(char c) noexcept { } /** + * Convert string to integral type T. Leading whitespace, a leading +, or any + * trailing character fail the parsing. The required format expressed as regex + * is `-?[0-9]+`. The minus sign is only permitted for signed integer types. + * + * @returns std::nullopt if the entire string could not be parsed, or if the + * parsed value is not in the range representable by the type T. + */ +template <typename T> +std::optional<T> ToIntegral(const std::string& str) +{ + static_assert(std::is_integral<T>::value); + T result; + const auto [first_nonmatching, error_condition] = std::from_chars(str.data(), str.data() + str.size(), result); + if (first_nonmatching != str.data() + str.size() || error_condition != std::errc{}) { + return std::nullopt; + } + return result; +} + +/** * Convert string to signed 32-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. @@ -137,13 +185,6 @@ constexpr inline bool IsSpace(char c) noexcept { [[nodiscard]] bool ParseUInt64(const std::string& str, uint64_t *out); /** - * Convert string to double with strict parse error feedback. - * @returns true if the entire string could be parsed as valid double, - * false if not the entire string could be parsed or when overflow or underflow occurred. - */ -[[nodiscard]] bool ParseDouble(const std::string& str, double *out); - -/** * Convert a span of bytes to a lower-case hexadecimal string. */ std::string HexStr(const Span<const uint8_t> s); diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp new file mode 100644 index 0000000000..bc69df44f4 --- /dev/null +++ b/src/util/syscall_sandbox.cpp @@ -0,0 +1,919 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif // defined(HAVE_CONFIG_H) + +#include <util/syscall_sandbox.h> + +#if defined(USE_SYSCALL_SANDBOX) +#include <array> +#include <cassert> +#include <cstdint> +#include <exception> +#include <map> +#include <new> +#include <set> +#include <string> +#include <vector> + +#include <logging.h> +#include <tinyformat.h> +#include <util/threadnames.h> + +#include <linux/audit.h> +#include <linux/filter.h> +#include <linux/seccomp.h> +#include <linux/unistd.h> +#include <signal.h> +#include <sys/prctl.h> +#include <sys/types.h> +#include <unistd.h> + +namespace { +bool g_syscall_sandbox_enabled{false}; +bool g_syscall_sandbox_log_violation_before_terminating{false}; + +#if !defined(__x86_64__) +#error Syscall sandbox is an experimental feature currently available only under Linux x86-64. +#endif // defined(__x86_64__) + +#ifndef SECCOMP_RET_KILL_PROCESS +#define SECCOMP_RET_KILL_PROCESS 0x80000000U +#endif + +// Define system call numbers for x86_64 that are referenced in the system call profile +// but not provided by the kernel headers used in the GUIX build. +// Usually, they can be found via "grep name /usr/include/x86_64-linux-gnu/asm/unistd_64.h" + +#ifndef __NR_clone3 +#define __NR_clone3 435 +#endif + +#ifndef __NR_statx +#define __NR_statx 332 +#endif + +#ifndef __NR_getrandom +#define __NR_getrandom 318 +#endif + +#ifndef __NR_membarrier +#define __NR_membarrier 324 +#endif + +#ifndef __NR_copy_file_range +#define __NR_copy_file_range 326 +#endif + +// This list of syscalls in LINUX_SYSCALLS is only used to map syscall numbers to syscall names in +// order to be able to print user friendly error messages which include the syscall name in addition +// to the syscall number. +// +// Example output in case of a syscall violation where the syscall is present in LINUX_SYSCALLS: +// +// ``` +// 2021-06-09T12:34:56Z ERROR: The syscall "execve" (syscall number 59) is not allowed by the syscall sandbox in thread "msghand". Please report. +// ``` +// +// Example output in case of a syscall violation where the syscall is not present in LINUX_SYSCALLS: +// +// ``` +// 2021-06-09T12:34:56Z ERROR: The syscall "*unknown*" (syscall number 314) is not allowed by the syscall sandbox in thread "msghand". Please report. +// `` +// +// LINUX_SYSCALLS contains two types of syscalls: +// 1.) Syscalls that are present under all architectures or relevant Linux kernel versions for which +// we support the syscall sandbox feature (currently only Linux x86-64). Examples include read, +// write, open, close, etc. +// 2.) Syscalls that are present under a subset of architectures or relevant Linux kernel versions +// for which we support the syscall sandbox feature. This type of syscalls should be added to +// LINUX_SYSCALLS conditional on availability like in the following example: +// ... +// #if defined(__NR_arch_dependent_syscall) +// {__NR_arch_dependent_syscall, "arch_dependent_syscall"}, +// #endif // defined(__NR_arch_dependent_syscall) +// ... +const std::map<uint32_t, std::string> LINUX_SYSCALLS{ + {__NR_accept, "accept"}, + {__NR_accept4, "accept4"}, + {__NR_access, "access"}, + {__NR_acct, "acct"}, + {__NR_add_key, "add_key"}, + {__NR_adjtimex, "adjtimex"}, + {__NR_afs_syscall, "afs_syscall"}, + {__NR_alarm, "alarm"}, + {__NR_arch_prctl, "arch_prctl"}, + {__NR_bind, "bind"}, + {__NR_bpf, "bpf"}, + {__NR_brk, "brk"}, + {__NR_capget, "capget"}, + {__NR_capset, "capset"}, + {__NR_chdir, "chdir"}, + {__NR_chmod, "chmod"}, + {__NR_chown, "chown"}, + {__NR_chroot, "chroot"}, + {__NR_clock_adjtime, "clock_adjtime"}, + {__NR_clock_getres, "clock_getres"}, + {__NR_clock_gettime, "clock_gettime"}, + {__NR_clock_nanosleep, "clock_nanosleep"}, + {__NR_clock_settime, "clock_settime"}, + {__NR_clone, "clone"}, + {__NR_clone3, "clone3"}, + {__NR_close, "close"}, + {__NR_connect, "connect"}, + {__NR_copy_file_range, "copy_file_range"}, + {__NR_creat, "creat"}, + {__NR_create_module, "create_module"}, + {__NR_delete_module, "delete_module"}, + {__NR_dup, "dup"}, + {__NR_dup2, "dup2"}, + {__NR_dup3, "dup3"}, + {__NR_epoll_create, "epoll_create"}, + {__NR_epoll_create1, "epoll_create1"}, + {__NR_epoll_ctl, "epoll_ctl"}, + {__NR_epoll_ctl_old, "epoll_ctl_old"}, + {__NR_epoll_pwait, "epoll_pwait"}, + {__NR_epoll_wait, "epoll_wait"}, + {__NR_epoll_wait_old, "epoll_wait_old"}, + {__NR_eventfd, "eventfd"}, + {__NR_eventfd2, "eventfd2"}, + {__NR_execve, "execve"}, + {__NR_execveat, "execveat"}, + {__NR_exit, "exit"}, + {__NR_exit_group, "exit_group"}, + {__NR_faccessat, "faccessat"}, + {__NR_fadvise64, "fadvise64"}, + {__NR_fallocate, "fallocate"}, + {__NR_fanotify_init, "fanotify_init"}, + {__NR_fanotify_mark, "fanotify_mark"}, + {__NR_fchdir, "fchdir"}, + {__NR_fchmod, "fchmod"}, + {__NR_fchmodat, "fchmodat"}, + {__NR_fchown, "fchown"}, + {__NR_fchownat, "fchownat"}, + {__NR_fcntl, "fcntl"}, + {__NR_fdatasync, "fdatasync"}, + {__NR_fgetxattr, "fgetxattr"}, + {__NR_finit_module, "finit_module"}, + {__NR_flistxattr, "flistxattr"}, + {__NR_flock, "flock"}, + {__NR_fork, "fork"}, + {__NR_fremovexattr, "fremovexattr"}, + {__NR_fsetxattr, "fsetxattr"}, + {__NR_fstat, "fstat"}, + {__NR_fstatfs, "fstatfs"}, + {__NR_fsync, "fsync"}, + {__NR_ftruncate, "ftruncate"}, + {__NR_futex, "futex"}, + {__NR_futimesat, "futimesat"}, + {__NR_get_kernel_syms, "get_kernel_syms"}, + {__NR_get_mempolicy, "get_mempolicy"}, + {__NR_get_robust_list, "get_robust_list"}, + {__NR_get_thread_area, "get_thread_area"}, + {__NR_getcpu, "getcpu"}, + {__NR_getcwd, "getcwd"}, + {__NR_getdents, "getdents"}, + {__NR_getdents64, "getdents64"}, + {__NR_getegid, "getegid"}, + {__NR_geteuid, "geteuid"}, + {__NR_getgid, "getgid"}, + {__NR_getgroups, "getgroups"}, + {__NR_getitimer, "getitimer"}, + {__NR_getpeername, "getpeername"}, + {__NR_getpgid, "getpgid"}, + {__NR_getpgrp, "getpgrp"}, + {__NR_getpid, "getpid"}, + {__NR_getpmsg, "getpmsg"}, + {__NR_getppid, "getppid"}, + {__NR_getpriority, "getpriority"}, + {__NR_getrandom, "getrandom"}, + {__NR_getresgid, "getresgid"}, + {__NR_getresuid, "getresuid"}, + {__NR_getrlimit, "getrlimit"}, + {__NR_getrusage, "getrusage"}, + {__NR_getsid, "getsid"}, + {__NR_getsockname, "getsockname"}, + {__NR_getsockopt, "getsockopt"}, + {__NR_gettid, "gettid"}, + {__NR_gettimeofday, "gettimeofday"}, + {__NR_getuid, "getuid"}, + {__NR_getxattr, "getxattr"}, + {__NR_init_module, "init_module"}, + {__NR_inotify_add_watch, "inotify_add_watch"}, + {__NR_inotify_init, "inotify_init"}, + {__NR_inotify_init1, "inotify_init1"}, + {__NR_inotify_rm_watch, "inotify_rm_watch"}, + {__NR_io_cancel, "io_cancel"}, + {__NR_io_destroy, "io_destroy"}, + {__NR_io_getevents, "io_getevents"}, + {__NR_io_setup, "io_setup"}, + {__NR_io_submit, "io_submit"}, + {__NR_ioctl, "ioctl"}, + {__NR_ioperm, "ioperm"}, + {__NR_iopl, "iopl"}, + {__NR_ioprio_get, "ioprio_get"}, + {__NR_ioprio_set, "ioprio_set"}, + {__NR_kcmp, "kcmp"}, + {__NR_kexec_file_load, "kexec_file_load"}, + {__NR_kexec_load, "kexec_load"}, + {__NR_keyctl, "keyctl"}, + {__NR_kill, "kill"}, + {__NR_lchown, "lchown"}, + {__NR_lgetxattr, "lgetxattr"}, + {__NR_link, "link"}, + {__NR_linkat, "linkat"}, + {__NR_listen, "listen"}, + {__NR_listxattr, "listxattr"}, + {__NR_llistxattr, "llistxattr"}, + {__NR_lookup_dcookie, "lookup_dcookie"}, + {__NR_lremovexattr, "lremovexattr"}, + {__NR_lseek, "lseek"}, + {__NR_lsetxattr, "lsetxattr"}, + {__NR_lstat, "lstat"}, + {__NR_madvise, "madvise"}, + {__NR_mbind, "mbind"}, + {__NR_membarrier, "membarrier"}, + {__NR_memfd_create, "memfd_create"}, + {__NR_migrate_pages, "migrate_pages"}, + {__NR_mincore, "mincore"}, + {__NR_mkdir, "mkdir"}, + {__NR_mkdirat, "mkdirat"}, + {__NR_mknod, "mknod"}, + {__NR_mknodat, "mknodat"}, + {__NR_mlock, "mlock"}, + {__NR_mlock2, "mlock2"}, + {__NR_mlockall, "mlockall"}, + {__NR_mmap, "mmap"}, + {__NR_modify_ldt, "modify_ldt"}, + {__NR_mount, "mount"}, + {__NR_move_pages, "move_pages"}, + {__NR_mprotect, "mprotect"}, + {__NR_mq_getsetattr, "mq_getsetattr"}, + {__NR_mq_notify, "mq_notify"}, + {__NR_mq_open, "mq_open"}, + {__NR_mq_timedreceive, "mq_timedreceive"}, + {__NR_mq_timedsend, "mq_timedsend"}, + {__NR_mq_unlink, "mq_unlink"}, + {__NR_mremap, "mremap"}, + {__NR_msgctl, "msgctl"}, + {__NR_msgget, "msgget"}, + {__NR_msgrcv, "msgrcv"}, + {__NR_msgsnd, "msgsnd"}, + {__NR_msync, "msync"}, + {__NR_munlock, "munlock"}, + {__NR_munlockall, "munlockall"}, + {__NR_munmap, "munmap"}, + {__NR_name_to_handle_at, "name_to_handle_at"}, + {__NR_nanosleep, "nanosleep"}, + {__NR_newfstatat, "newfstatat"}, + {__NR_nfsservctl, "nfsservctl"}, + {__NR_open, "open"}, + {__NR_open_by_handle_at, "open_by_handle_at"}, + {__NR_openat, "openat"}, + {__NR_pause, "pause"}, + {__NR_perf_event_open, "perf_event_open"}, + {__NR_personality, "personality"}, + {__NR_pipe, "pipe"}, + {__NR_pipe2, "pipe2"}, + {__NR_pivot_root, "pivot_root"}, +#ifdef __NR_pkey_alloc + {__NR_pkey_alloc, "pkey_alloc"}, +#endif +#ifdef __NR_pkey_free + {__NR_pkey_free, "pkey_free"}, +#endif +#ifdef __NR_pkey_mprotect + {__NR_pkey_mprotect, "pkey_mprotect"}, +#endif + {__NR_poll, "poll"}, + {__NR_ppoll, "ppoll"}, + {__NR_prctl, "prctl"}, + {__NR_pread64, "pread64"}, + {__NR_preadv, "preadv"}, +#ifdef __NR_preadv2 + {__NR_preadv2, "preadv2"}, +#endif + {__NR_prlimit64, "prlimit64"}, + {__NR_process_vm_readv, "process_vm_readv"}, + {__NR_process_vm_writev, "process_vm_writev"}, + {__NR_pselect6, "pselect6"}, + {__NR_ptrace, "ptrace"}, + {__NR_putpmsg, "putpmsg"}, + {__NR_pwrite64, "pwrite64"}, + {__NR_pwritev, "pwritev"}, +#ifdef __NR_pwritev2 + {__NR_pwritev2, "pwritev2"}, +#endif + {__NR__sysctl, "_sysctl"}, + {__NR_query_module, "query_module"}, + {__NR_quotactl, "quotactl"}, + {__NR_read, "read"}, + {__NR_readahead, "readahead"}, + {__NR_readlink, "readlink"}, + {__NR_readlinkat, "readlinkat"}, + {__NR_readv, "readv"}, + {__NR_reboot, "reboot"}, + {__NR_recvfrom, "recvfrom"}, + {__NR_recvmmsg, "recvmmsg"}, + {__NR_recvmsg, "recvmsg"}, + {__NR_remap_file_pages, "remap_file_pages"}, + {__NR_removexattr, "removexattr"}, + {__NR_rename, "rename"}, + {__NR_renameat, "renameat"}, + {__NR_renameat2, "renameat2"}, + {__NR_request_key, "request_key"}, + {__NR_restart_syscall, "restart_syscall"}, + {__NR_rmdir, "rmdir"}, + {__NR_rt_sigaction, "rt_sigaction"}, + {__NR_rt_sigpending, "rt_sigpending"}, + {__NR_rt_sigprocmask, "rt_sigprocmask"}, + {__NR_rt_sigqueueinfo, "rt_sigqueueinfo"}, + {__NR_rt_sigreturn, "rt_sigreturn"}, + {__NR_rt_sigsuspend, "rt_sigsuspend"}, + {__NR_rt_sigtimedwait, "rt_sigtimedwait"}, + {__NR_rt_tgsigqueueinfo, "rt_tgsigqueueinfo"}, + {__NR_sched_get_priority_max, "sched_get_priority_max"}, + {__NR_sched_get_priority_min, "sched_get_priority_min"}, + {__NR_sched_getaffinity, "sched_getaffinity"}, + {__NR_sched_getattr, "sched_getattr"}, + {__NR_sched_getparam, "sched_getparam"}, + {__NR_sched_getscheduler, "sched_getscheduler"}, + {__NR_sched_rr_get_interval, "sched_rr_get_interval"}, + {__NR_sched_setaffinity, "sched_setaffinity"}, + {__NR_sched_setattr, "sched_setattr"}, + {__NR_sched_setparam, "sched_setparam"}, + {__NR_sched_setscheduler, "sched_setscheduler"}, + {__NR_sched_yield, "sched_yield"}, + {__NR_seccomp, "seccomp"}, + {__NR_security, "security"}, + {__NR_select, "select"}, + {__NR_semctl, "semctl"}, + {__NR_semget, "semget"}, + {__NR_semop, "semop"}, + {__NR_semtimedop, "semtimedop"}, + {__NR_sendfile, "sendfile"}, + {__NR_sendmmsg, "sendmmsg"}, + {__NR_sendmsg, "sendmsg"}, + {__NR_sendto, "sendto"}, + {__NR_set_mempolicy, "set_mempolicy"}, + {__NR_set_robust_list, "set_robust_list"}, + {__NR_set_thread_area, "set_thread_area"}, + {__NR_set_tid_address, "set_tid_address"}, + {__NR_setdomainname, "setdomainname"}, + {__NR_setfsgid, "setfsgid"}, + {__NR_setfsuid, "setfsuid"}, + {__NR_setgid, "setgid"}, + {__NR_setgroups, "setgroups"}, + {__NR_sethostname, "sethostname"}, + {__NR_setitimer, "setitimer"}, + {__NR_setns, "setns"}, + {__NR_setpgid, "setpgid"}, + {__NR_setpriority, "setpriority"}, + {__NR_setregid, "setregid"}, + {__NR_setresgid, "setresgid"}, + {__NR_setresuid, "setresuid"}, + {__NR_setreuid, "setreuid"}, + {__NR_setrlimit, "setrlimit"}, + {__NR_setsid, "setsid"}, + {__NR_setsockopt, "setsockopt"}, + {__NR_settimeofday, "settimeofday"}, + {__NR_setuid, "setuid"}, + {__NR_setxattr, "setxattr"}, + {__NR_shmat, "shmat"}, + {__NR_shmctl, "shmctl"}, + {__NR_shmdt, "shmdt"}, + {__NR_shmget, "shmget"}, + {__NR_shutdown, "shutdown"}, + {__NR_sigaltstack, "sigaltstack"}, + {__NR_signalfd, "signalfd"}, + {__NR_signalfd4, "signalfd4"}, + {__NR_socket, "socket"}, + {__NR_socketpair, "socketpair"}, + {__NR_splice, "splice"}, + {__NR_stat, "stat"}, + {__NR_statfs, "statfs"}, + {__NR_statx, "statx"}, + {__NR_swapoff, "swapoff"}, + {__NR_swapon, "swapon"}, + {__NR_symlink, "symlink"}, + {__NR_symlinkat, "symlinkat"}, + {__NR_sync, "sync"}, + {__NR_sync_file_range, "sync_file_range"}, + {__NR_syncfs, "syncfs"}, + {__NR_sysfs, "sysfs"}, + {__NR_sysinfo, "sysinfo"}, + {__NR_syslog, "syslog"}, + {__NR_tee, "tee"}, + {__NR_tgkill, "tgkill"}, + {__NR_time, "time"}, + {__NR_timer_create, "timer_create"}, + {__NR_timer_delete, "timer_delete"}, + {__NR_timer_getoverrun, "timer_getoverrun"}, + {__NR_timer_gettime, "timer_gettime"}, + {__NR_timer_settime, "timer_settime"}, + {__NR_timerfd_create, "timerfd_create"}, + {__NR_timerfd_gettime, "timerfd_gettime"}, + {__NR_timerfd_settime, "timerfd_settime"}, + {__NR_times, "times"}, + {__NR_tkill, "tkill"}, + {__NR_truncate, "truncate"}, + {__NR_tuxcall, "tuxcall"}, + {__NR_umask, "umask"}, + {__NR_umount2, "umount2"}, + {__NR_uname, "uname"}, + {__NR_unlink, "unlink"}, + {__NR_unlinkat, "unlinkat"}, + {__NR_unshare, "unshare"}, + {__NR_uselib, "uselib"}, + {__NR_userfaultfd, "userfaultfd"}, + {__NR_ustat, "ustat"}, + {__NR_utime, "utime"}, + {__NR_utimensat, "utimensat"}, + {__NR_utimes, "utimes"}, + {__NR_vfork, "vfork"}, + {__NR_vhangup, "vhangup"}, + {__NR_vmsplice, "vmsplice"}, + {__NR_vserver, "vserver"}, + {__NR_wait4, "wait4"}, + {__NR_waitid, "waitid"}, + {__NR_write, "write"}, + {__NR_writev, "writev"}, +}; + +std::string GetLinuxSyscallName(uint32_t syscall_number) +{ + const auto element = LINUX_SYSCALLS.find(syscall_number); + if (element != LINUX_SYSCALLS.end()) { + return element->second; + } + return "*unknown*"; +} + +// See Linux kernel developer Kees Cook's seccomp guide at <https://outflux.net/teach-seccomp/> for +// an accessible introduction to using seccomp. +// +// This function largely follows <https://outflux.net/teach-seccomp/step-3/syscall-reporter.c> and +// <https://outflux.net/teach-seccomp/step-3/seccomp-bpf.h>. +// +// Seccomp BPF resources: +// * Seccomp BPF documentation: <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> +// * seccomp(2) manual page: <https://www.kernel.org/doc/man-pages/online/pages/man2/seccomp.2.html> +// * Seccomp BPF demo code samples: <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/samples/seccomp> +void SyscallSandboxDebugSignalHandler(int, siginfo_t* signal_info, void* void_signal_context) +{ + // The si_code field inside the siginfo_t argument that is passed to a SA_SIGINFO signal handler + // is a value indicating why the signal was sent. + // + // The following value can be placed in si_code for a SIGSYS signal: + // * SYS_SECCOMP (since Linux 3.5): Triggered by a seccomp(2) filter rule. + constexpr int32_t SYS_SECCOMP_SI_CODE{1}; + assert(signal_info->si_code == SYS_SECCOMP_SI_CODE); + + // The ucontext_t structure contains signal context information that was saved on the user-space + // stack by the kernel. + const ucontext_t* signal_context = static_cast<ucontext_t*>(void_signal_context); + assert(signal_context != nullptr); + + std::set_new_handler(std::terminate); + // Portability note: REG_RAX is Linux x86_64 specific. + const uint32_t syscall_number = static_cast<uint32_t>(signal_context->uc_mcontext.gregs[REG_RAX]); + const std::string syscall_name = GetLinuxSyscallName(syscall_number); + const std::string thread_name = !util::ThreadGetInternalName().empty() ? util::ThreadGetInternalName() : "*unnamed*"; + const std::string error_message = strprintf("ERROR: The syscall \"%s\" (syscall number %d) is not allowed by the syscall sandbox in thread \"%s\". Please report.", syscall_name, syscall_number, thread_name); + tfm::format(std::cerr, "%s\n", error_message); + LogPrintf("%s\n", error_message); + std::terminate(); +} + +// This function largely follows install_syscall_reporter from Kees Cook's seccomp guide: +// <https://outflux.net/teach-seccomp/step-3/syscall-reporter.c> +bool SetupSyscallSandboxDebugHandler() +{ + struct sigaction action = {}; + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGSYS); + action.sa_sigaction = &SyscallSandboxDebugSignalHandler; + action.sa_flags = SA_SIGINFO; + if (sigaction(SIGSYS, &action, nullptr) < 0) { + return false; + } + if (sigprocmask(SIG_UNBLOCK, &mask, nullptr)) { + return false; + } + return true; +} + +enum class SyscallSandboxAction { + KILL_PROCESS, + INVOKE_SIGNAL_HANDLER, +}; + +class SeccompPolicyBuilder +{ + std::set<uint32_t> allowed_syscalls; + +public: + SeccompPolicyBuilder() + { + // Allowed by default. + AllowAddressSpaceAccess(); + AllowEpoll(); + AllowEventFd(); + AllowFutex(); + AllowGeneralIo(); + AllowGetRandom(); + AllowGetSimpleId(); + AllowGetTime(); + AllowGlobalProcessEnvironment(); + AllowGlobalSystemStatus(); + AllowKernelInternalApi(); + AllowNetworkSocketInformation(); + AllowOperationOnExistingFileDescriptor(); + AllowPipe(); + AllowPrctl(); + AllowProcessStartOrDeath(); + AllowScheduling(); + AllowSignalHandling(); + AllowSleep(); + AllowUmask(); + } + + void AllowAddressSpaceAccess() + { + allowed_syscalls.insert(__NR_brk); // change data segment size + allowed_syscalls.insert(__NR_madvise); // give advice about use of memory + allowed_syscalls.insert(__NR_membarrier); // issue memory barriers on a set of threads + allowed_syscalls.insert(__NR_mincore); // check if virtual memory is in RAM + allowed_syscalls.insert(__NR_mlock); // lock memory + allowed_syscalls.insert(__NR_mmap); // map files or devices into memory + allowed_syscalls.insert(__NR_mprotect); // set protection on a region of memory + allowed_syscalls.insert(__NR_mremap); // remap a file in memory + allowed_syscalls.insert(__NR_munlock); // unlock memory + allowed_syscalls.insert(__NR_munmap); // unmap files or devices into memory + } + + void AllowEpoll() + { + allowed_syscalls.insert(__NR_epoll_create1); // open an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_ctl); // control interface for an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_pwait); // wait for an I/O event on an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_wait); // wait for an I/O event on an epoll file descriptor + } + + void AllowEventFd() + { + allowed_syscalls.insert(__NR_eventfd2); // create a file descriptor for event notification + } + + void AllowFileSystem() + { + allowed_syscalls.insert(__NR_access); // check user's permissions for a file + allowed_syscalls.insert(__NR_chdir); // change working directory + allowed_syscalls.insert(__NR_chmod); // change permissions of a file + allowed_syscalls.insert(__NR_copy_file_range); // copy a range of data from one file to another + allowed_syscalls.insert(__NR_fallocate); // manipulate file space + allowed_syscalls.insert(__NR_fchmod); // change permissions of a file + allowed_syscalls.insert(__NR_fchown); // change ownership of a file + allowed_syscalls.insert(__NR_fdatasync); // synchronize a file's in-core state with storage device + allowed_syscalls.insert(__NR_flock); // apply or remove an advisory lock on an open file + allowed_syscalls.insert(__NR_fstat); // get file status + allowed_syscalls.insert(__NR_newfstatat); // get file status + allowed_syscalls.insert(__NR_fsync); // synchronize a file's in-core state with storage device + allowed_syscalls.insert(__NR_ftruncate); // truncate a file to a specified length + allowed_syscalls.insert(__NR_getcwd); // get current working directory + allowed_syscalls.insert(__NR_getdents); // get directory entries + allowed_syscalls.insert(__NR_getdents64); // get directory entries + allowed_syscalls.insert(__NR_lstat); // get file status + allowed_syscalls.insert(__NR_mkdir); // create a directory + allowed_syscalls.insert(__NR_open); // open and possibly create a file + allowed_syscalls.insert(__NR_openat); // open and possibly create a file + allowed_syscalls.insert(__NR_readlink); // read value of a symbolic link + allowed_syscalls.insert(__NR_rename); // change the name or location of a file + allowed_syscalls.insert(__NR_rmdir); // delete a directory + allowed_syscalls.insert(__NR_stat); // get file status + allowed_syscalls.insert(__NR_statfs); // get filesystem statistics + allowed_syscalls.insert(__NR_statx); // get file status (extended) + allowed_syscalls.insert(__NR_unlink); // delete a name and possibly the file it refers to + } + + void AllowFutex() + { + allowed_syscalls.insert(__NR_futex); // fast user-space locking + allowed_syscalls.insert(__NR_set_robust_list); // set list of robust futexes + } + + void AllowGeneralIo() + { + allowed_syscalls.insert(__NR_ioctl); // control device + allowed_syscalls.insert(__NR_lseek); // reposition read/write file offset + allowed_syscalls.insert(__NR_poll); // wait for some event on a file descriptor + allowed_syscalls.insert(__NR_ppoll); // wait for some event on a file descriptor + allowed_syscalls.insert(__NR_pread64); // read from a file descriptor at a given offset + allowed_syscalls.insert(__NR_pwrite64); // write to a file descriptor at a given offset + allowed_syscalls.insert(__NR_read); // read from a file descriptor + allowed_syscalls.insert(__NR_readv); // read data into multiple buffers + allowed_syscalls.insert(__NR_recvfrom); // receive a message from a socket + allowed_syscalls.insert(__NR_recvmsg); // receive a message from a socket + allowed_syscalls.insert(__NR_select); // synchronous I/O multiplexing + allowed_syscalls.insert(__NR_sendmmsg); // send multiple messages on a socket + allowed_syscalls.insert(__NR_sendmsg); // send a message on a socket + allowed_syscalls.insert(__NR_sendto); // send a message on a socket + allowed_syscalls.insert(__NR_write); // write to a file descriptor + allowed_syscalls.insert(__NR_writev); // write data into multiple buffers + } + + void AllowGetRandom() + { + allowed_syscalls.insert(__NR_getrandom); // obtain a series of random bytes + } + + void AllowGetSimpleId() + { + allowed_syscalls.insert(__NR_getegid); // get group identity + allowed_syscalls.insert(__NR_geteuid); // get user identity + allowed_syscalls.insert(__NR_getgid); // get group identity + allowed_syscalls.insert(__NR_getpgid); // get process group + allowed_syscalls.insert(__NR_getpid); // get process identification + allowed_syscalls.insert(__NR_getppid); // get process identification + allowed_syscalls.insert(__NR_getresgid); // get real, effective and saved group IDs + allowed_syscalls.insert(__NR_getresuid); // get real, effective and saved user IDs + allowed_syscalls.insert(__NR_getsid); // get session ID + allowed_syscalls.insert(__NR_gettid); // get thread identification + allowed_syscalls.insert(__NR_getuid); // get user identity + } + + void AllowGetTime() + { + allowed_syscalls.insert(__NR_clock_getres); // find the resolution (precision) of the specified clock + allowed_syscalls.insert(__NR_clock_gettime); // retrieve the time of the specified clock + allowed_syscalls.insert(__NR_gettimeofday); // get timeval + } + + void AllowGlobalProcessEnvironment() + { + allowed_syscalls.insert(__NR_getrlimit); // get resource limits + allowed_syscalls.insert(__NR_getrusage); // get resource usage + allowed_syscalls.insert(__NR_prlimit64); // get/set resource limits + } + + void AllowGlobalSystemStatus() + { + allowed_syscalls.insert(__NR_sysinfo); // return system information + allowed_syscalls.insert(__NR_uname); // get name and information about current kernel + } + + void AllowKernelInternalApi() + { + allowed_syscalls.insert(__NR_restart_syscall); // restart a system call after interruption by a stop signal + } + + void AllowNetwork() + { + allowed_syscalls.insert(__NR_accept); // accept a connection on a socket + allowed_syscalls.insert(__NR_accept4); // accept a connection on a socket + allowed_syscalls.insert(__NR_bind); // bind a name to a socket + allowed_syscalls.insert(__NR_connect); // initiate a connection on a socket + allowed_syscalls.insert(__NR_listen); // listen for connections on a socket + allowed_syscalls.insert(__NR_setsockopt); // set options on sockets + allowed_syscalls.insert(__NR_socket); // create an endpoint for communication + allowed_syscalls.insert(__NR_socketpair); // create a pair of connected sockets + } + + void AllowNetworkSocketInformation() + { + allowed_syscalls.insert(__NR_getpeername); // get name of connected peer socket + allowed_syscalls.insert(__NR_getsockname); // get socket name + allowed_syscalls.insert(__NR_getsockopt); // get options on sockets + } + + void AllowOperationOnExistingFileDescriptor() + { + allowed_syscalls.insert(__NR_close); // close a file descriptor + allowed_syscalls.insert(__NR_dup); // duplicate a file descriptor + allowed_syscalls.insert(__NR_dup2); // duplicate a file descriptor + allowed_syscalls.insert(__NR_fcntl); // manipulate file descriptor + allowed_syscalls.insert(__NR_shutdown); // shut down part of a full-duplex connection + } + + void AllowPipe() + { + allowed_syscalls.insert(__NR_pipe); // create pipe + allowed_syscalls.insert(__NR_pipe2); // create pipe + } + + void AllowPrctl() + { + allowed_syscalls.insert(__NR_arch_prctl); // set architecture-specific thread state + allowed_syscalls.insert(__NR_prctl); // operations on a process + } + + void AllowProcessStartOrDeath() + { + allowed_syscalls.insert(__NR_clone); // create a child process + allowed_syscalls.insert(__NR_clone3); // create a child process + allowed_syscalls.insert(__NR_exit); // terminate the calling process + allowed_syscalls.insert(__NR_exit_group); // exit all threads in a process + allowed_syscalls.insert(__NR_fork); // create a child process + allowed_syscalls.insert(__NR_tgkill); // send a signal to a thread + allowed_syscalls.insert(__NR_wait4); // wait for process to change state, BSD style + } + + void AllowScheduling() + { + allowed_syscalls.insert(__NR_sched_getaffinity); // set a thread's CPU affinity mask + allowed_syscalls.insert(__NR_sched_getparam); // get scheduling parameters + allowed_syscalls.insert(__NR_sched_getscheduler); // get scheduling policy/parameters + allowed_syscalls.insert(__NR_sched_setscheduler); // set scheduling policy/parameters + allowed_syscalls.insert(__NR_sched_yield); // yield the processor + } + + void AllowSignalHandling() + { + allowed_syscalls.insert(__NR_rt_sigaction); // examine and change a signal action + allowed_syscalls.insert(__NR_rt_sigprocmask); // examine and change blocked signals + allowed_syscalls.insert(__NR_rt_sigreturn); // return from signal handler and cleanup stack frame + allowed_syscalls.insert(__NR_sigaltstack); // set and/or get signal stack context + } + + void AllowSleep() + { + allowed_syscalls.insert(__NR_clock_nanosleep); // high-resolution sleep with specifiable clock + allowed_syscalls.insert(__NR_nanosleep); // high-resolution sleep + } + + void AllowUmask() + { + allowed_syscalls.insert(__NR_umask); // set file mode creation mask + } + + // See Linux kernel developer Kees Cook's seccomp guide at <https://outflux.net/teach-seccomp/> + // for an accessible introduction to using seccomp. + // + // This function largely follows <https://outflux.net/teach-seccomp/step-3/seccomp-bpf.h>. + std::vector<sock_filter> BuildFilter(SyscallSandboxAction default_action) + { + std::vector<sock_filter> bpf_policy; + // See VALIDATE_ARCHITECTURE in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_STMT(BPF_LD + BPF_W + BPF_ABS, offsetof(struct seccomp_data, arch))); + // Portability note: AUDIT_ARCH_X86_64 is Linux x86_64 specific. + bpf_policy.push_back(BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, AUDIT_ARCH_X86_64, 1, 0)); + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_KILL_PROCESS)); + // See EXAMINE_SYSCALL in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_STMT(BPF_LD + BPF_W + BPF_ABS, offsetof(struct seccomp_data, nr))); + for (const uint32_t allowed_syscall : allowed_syscalls) { + // See ALLOW_SYSCALL in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, allowed_syscall, 0, 1)); + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW)); + } + switch (default_action) { + case SyscallSandboxAction::KILL_PROCESS: + // Disallow syscall and kill the process. + // + // See KILL_PROCESS in seccomp-bpf.h referenced above. + // + // Note that we're using SECCOMP_RET_KILL_PROCESS (kill the process) instead + // of SECCOMP_RET_KILL_THREAD (kill the thread). The SECCOMP_RET_KILL_PROCESS + // action was introduced in Linux 4.14. + // + // SECCOMP_RET_KILL_PROCESS: Results in the entire process exiting immediately without + // executing the system call. + // + // SECCOMP_RET_KILL_PROCESS documentation: + // <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_KILL_PROCESS)); + break; + case SyscallSandboxAction::INVOKE_SIGNAL_HANDLER: + // Disallow syscall and force a SIGSYS to trigger syscall debug reporter. + // + // SECCOMP_RET_TRAP: Results in the kernel sending a SIGSYS signal to the triggering + // task without executing the system call. + // + // SECCOMP_RET_TRAP documentation: + // <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_TRAP)); + break; + } + return bpf_policy; + } +}; +} // namespace + +bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating) +{ + assert(!g_syscall_sandbox_enabled && "SetupSyscallSandbox(...) should only be called once."); + g_syscall_sandbox_enabled = true; + g_syscall_sandbox_log_violation_before_terminating = log_syscall_violation_before_terminating; + if (log_syscall_violation_before_terminating) { + if (!SetupSyscallSandboxDebugHandler()) { + return false; + } + } + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION); + return true; +} + +void TestDisallowedSandboxCall() +{ + // The getgroups syscall is assumed NOT to be allowed by the syscall sandbox policy. + std::array<gid_t, 1> groups; + [[maybe_unused]] int32_t ignored = getgroups(groups.size(), groups.data()); +} +#endif // defined(USE_SYSCALL_SANDBOX) + +void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy) +{ +#if defined(USE_SYSCALL_SANDBOX) + if (!g_syscall_sandbox_enabled) { + return; + } + SeccompPolicyBuilder seccomp_policy_builder; + switch (syscall_policy) { + case SyscallSandboxPolicy::INITIALIZATION: // Thread: main thread (state: init) + // SyscallSandboxPolicy::INITIALIZATION is the first policy loaded. + // + // Subsequently loaded policies can reduce the abilities further, but + // abilities can never be regained. + // + // SyscallSandboxPolicy::INITIALIZATION must thus be a superset of all + // other policies. + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::INITIALIZATION_DNS_SEED: // Thread: dnsseed + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS: // Thread: loadblk + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::INITIALIZATION_MAP_PORT: // Thread: mapport + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::MESSAGE_HANDLER: // Thread: msghand + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::NET: // Thread: net + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_ADD_CONNECTION: // Thread: addcon + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_HTTP_SERVER: // Thread: http + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_HTTP_SERVER_WORKER: // Thread: httpworker.<N> + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_OPEN_CONNECTION: // Thread: opencon + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::SCHEDULER: // Thread: scheduler + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::TOR_CONTROL: // Thread: torcontrol + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::TX_INDEX: // Thread: txindex + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::VALIDATION_SCRIPT_CHECK: // Thread: scriptch.<N> + break; + case SyscallSandboxPolicy::SHUTOFF: // Thread: main thread (state: shutoff) + seccomp_policy_builder.AllowFileSystem(); + break; + } + + const SyscallSandboxAction default_action = g_syscall_sandbox_log_violation_before_terminating ? SyscallSandboxAction::INVOKE_SIGNAL_HANDLER : SyscallSandboxAction::KILL_PROCESS; + std::vector<sock_filter> filter = seccomp_policy_builder.BuildFilter(default_action); + const sock_fprog prog = { + .len = static_cast<uint16_t>(filter.size()), + .filter = filter.data(), + }; + // Do not allow abilities to be regained after being dropped. + // + // PR_SET_NO_NEW_PRIVS documentation: <https://www.kernel.org/doc/html/latest/userspace-api/no_new_privs.html> + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { + throw std::runtime_error("Syscall sandbox enforcement failed: prctl(PR_SET_NO_NEW_PRIVS)"); + } + // Install seccomp-bpf syscall filter. + // + // PR_SET_SECCOMP documentation: <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog) != 0) { + throw std::runtime_error("Syscall sandbox enforcement failed: prctl(PR_SET_SECCOMP)"); + } + + const std::string thread_name = !util::ThreadGetInternalName().empty() ? util::ThreadGetInternalName() : "*unnamed*"; + LogPrint(BCLog::UTIL, "Syscall filter installed for thread \"%s\"\n", thread_name); +#endif // defined(USE_SYSCALL_SANDBOX) +} diff --git a/src/util/syscall_sandbox.h b/src/util/syscall_sandbox.h new file mode 100644 index 0000000000..0a0c964f94 --- /dev/null +++ b/src/util/syscall_sandbox.h @@ -0,0 +1,57 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SYSCALL_SANDBOX_H +#define BITCOIN_UTIL_SYSCALL_SANDBOX_H + +enum class SyscallSandboxPolicy { + // 1. Initialization + INITIALIZATION, + INITIALIZATION_DNS_SEED, + INITIALIZATION_LOAD_BLOCKS, + INITIALIZATION_MAP_PORT, + + // 2. Steady state (non-initialization, non-shutdown) + MESSAGE_HANDLER, + NET, + NET_ADD_CONNECTION, + NET_HTTP_SERVER, + NET_HTTP_SERVER_WORKER, + NET_OPEN_CONNECTION, + SCHEDULER, + TOR_CONTROL, + TX_INDEX, + VALIDATION_SCRIPT_CHECK, + + // 3. Shutdown + SHUTOFF, +}; + +//! Force the current thread (and threads created from the current thread) into a restricted-service +//! operating mode where only a subset of all syscalls are available. +//! +//! Subsequent calls to this function can reduce the abilities further, but abilities can never be +//! regained. +//! +//! This function is a no-op unless SetupSyscallSandbox(...) has been called. +//! +//! SetupSyscallSandbox(...) is called during bitcoind initialization if Bitcoin Core was compiled +//! with seccomp-bpf support (--with-seccomp) *and* the parameter -sandbox=<mode> was passed to +//! bitcoind. +//! +//! This experimental feature is available under Linux x86_64 only. +void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy); + +#if defined(USE_SYSCALL_SANDBOX) +//! Setup and enable the experimental syscall sandbox for the running process. +//! +//! SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION) is called as part of +//! SetupSyscallSandbox(...). +[[nodiscard]] bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating); + +//! Invoke a disallowed syscall. Use for testing purposes. +void TestDisallowedSandboxCall(); +#endif // defined(USE_SYSCALL_SANDBOX) + +#endif // BITCOIN_UTIL_SYSCALL_SANDBOX_H diff --git a/src/util/system.cpp b/src/util/system.cpp index 08f62f1da7..99d111b066 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -71,6 +71,7 @@ #endif #include <boost/algorithm/string/replace.hpp> +#include <optional> #include <thread> #include <typeinfo> #include <univalue.h> @@ -98,7 +99,7 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b fs::path pathLockFile = directory / lockfile_name; // If a lock for this directory already exists in the map, don't try to re-lock it - if (dir_locks.count(pathLockFile.string())) { + if (dir_locks.count(fs::PathToString(pathLockFile))) { return true; } @@ -107,11 +108,11 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b if (file) fclose(file); auto lock = std::make_unique<fsbridge::FileLock>(pathLockFile); if (!lock->TryLock()) { - return error("Error while attempting to lock directory %s: %s", directory.string(), lock->GetReason()); + return error("Error while attempting to lock directory %s: %s", fs::PathToString(directory), lock->GetReason()); } if (!probe_only) { // Lock successful and we're not just probing, put it into the map - dir_locks.emplace(pathLockFile.string(), std::move(lock)); + dir_locks.emplace(fs::PathToString(pathLockFile), std::move(lock)); } return true; } @@ -119,7 +120,7 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name) { LOCK(cs_dir_locks); - dir_locks.erase((directory / lockfile_name).string()); + dir_locks.erase(fs::PathToString(directory / lockfile_name)); } void ReleaseDirectoryLocks() @@ -158,16 +159,14 @@ std::streampos GetFileSize(const char* path, std::streamsize max) { /** * Interpret a string argument as a boolean. * - * The definition of atoi() requires that non-numeric string values like "foo", - * return 0. This means that if a user unintentionally supplies a non-integer - * argument here, the return value is always false. This means that -foo=false - * does what the user probably expects, but -foo=true is well defined but does - * not do what they probably expected. + * The definition of LocaleIndependentAtoi<int>() requires that non-numeric string values + * like "foo", return 0. This means that if a user unintentionally supplies a + * non-integer argument here, the return value is always false. This means that + * -foo=false does what the user probably expects, but -foo=true is well defined + * but does not do what they probably expected. * - * The return value of atoi() is undefined when given input not representable as - * an int. On most systems this means string value between "-2147483648" and - * "2147483647" are well defined (this method will return true). Setting - * -txindex=2147483648 on most systems, however, is probably undefined. + * The return value of LocaleIndependentAtoi<int>(...) is zero when given input not + * representable as an int. * * For a more extensive discussion of this topic (and a wide range of opinions * on the Right Way to change this code), see PR12713. @@ -176,7 +175,7 @@ static bool InterpretBool(const std::string& strValue) { if (strValue.empty()) return true; - return (atoi(strValue) != 0); + return (LocaleIndependentAtoi<int>(strValue) != 0); } static std::string SettingName(const std::string& arg) @@ -184,67 +183,72 @@ static std::string SettingName(const std::string& arg) return arg.size() > 0 && arg[0] == '-' ? arg.substr(1) : arg; } +struct KeyInfo { + std::string name; + std::string section; + bool negated{false}; +}; + /** - * Interpret -nofoo as if the user supplied -foo=0. - * - * This method also tracks when the -no form was supplied, and if so, - * checks whether there was a double-negative (-nofoo=0 -> -foo=1). - * - * If there was not a double negative, it removes the "no" from the key - * and returns false. + * Parse "name", "section.name", "noname", "section.noname" settings keys. * - * If there was a double negative, it removes "no" from the key, and - * returns true. - * - * If there was no "no", it returns the string value untouched. - * - * Where an option was negated can be later checked using the + * @note Where an option was negated can be later checked using the * IsArgNegated() method. One use case for this is to have a way to disable * options that are not normally boolean (e.g. using -nodebuglogfile to request * that debug log output is not sent to any file at all). */ - -static util::SettingsValue InterpretOption(std::string& section, std::string& key, const std::string& value) +KeyInfo InterpretKey(std::string key) { + KeyInfo result; // Split section name from key name for keys like "testnet.foo" or "regtest.bar" size_t option_index = key.find('.'); if (option_index != std::string::npos) { - section = key.substr(0, option_index); + result.section = key.substr(0, option_index); key.erase(0, option_index + 1); } if (key.substr(0, 2) == "no") { key.erase(0, 2); - // Double negatives like -nofoo=0 are supported (but discouraged) - if (!InterpretBool(value)) { - LogPrintf("Warning: parsed potentially confusing double-negative -%s=%s\n", key, value); - return true; - } - return false; + result.negated = true; } - return value; + result.name = key; + return result; } /** - * Check settings value validity according to flags. + * Interpret settings value based on registered flags. + * + * @param[in] key key information to know if key was negated + * @param[in] value string value of setting to be parsed + * @param[in] flags ArgsManager registered argument flags + * @param[out] error Error description if settings value is not valid * - * TODO: Add more meaningful error checks here in the future - * See "here's how the flags are meant to behave" in - * https://github.com/bitcoin/bitcoin/pull/16097#issuecomment-514627823 + * @return parsed settings value if it is valid, otherwise nullopt accompanied + * by a descriptive error string */ -static bool CheckValid(const std::string& key, const util::SettingsValue& val, unsigned int flags, std::string& error) -{ - if (val.isBool() && !(flags & ArgsManager::ALLOW_BOOL)) { - error = strprintf("Negating of -%s is meaningless and therefore forbidden", key); +static std::optional<util::SettingsValue> InterpretValue(const KeyInfo& key, const std::string& value, + unsigned int flags, std::string& error) +{ + // Return negated settings as false values. + if (key.negated) { + if (flags & ArgsManager::DISALLOW_NEGATION) { + error = strprintf("Negating of -%s is meaningless and therefore forbidden", key.name); + return std::nullopt; + } + // Double negatives like -nofoo=0 are supported (but discouraged) + if (!InterpretBool(value)) { + LogPrintf("Warning: parsed potentially confusing double-negative -%s=%s\n", key.name, value); + return true; + } return false; } - return true; + return value; } namespace { fs::path StripRedundantLastElementsOfPath(const fs::path& path) { auto result = path; - while (result.filename().string() == ".") { + while (fs::PathToString(result.filename()) == ".") { result = result.parent_path(); } @@ -353,21 +357,21 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin // Transform -foo to foo key.erase(0, 1); - std::string section; - util::SettingsValue value = InterpretOption(section, key, val); - std::optional<unsigned int> flags = GetArgFlags('-' + key); + KeyInfo keyinfo = InterpretKey(key); + std::optional<unsigned int> flags = GetArgFlags('-' + keyinfo.name); // Unknown command line options and command line options with dot - // characters (which are returned from InterpretOption with nonempty + // characters (which are returned from InterpretKey with nonempty // section strings) are not valid. - if (!flags || !section.empty()) { + if (!flags || !keyinfo.section.empty()) { error = strprintf("Invalid parameter %s", argv[i]); return false; } - if (!CheckValid(key, value, *flags, error)) return false; + std::optional<util::SettingsValue> value = InterpretValue(keyinfo, val, *flags, error); + if (!value) return false; - m_settings.command_line_options[key].push_back(value); + m_settings.command_line_options[keyinfo.name].push_back(*value); } // we do not allow -includeconf from command line, only -noincludeconf @@ -404,7 +408,7 @@ const fs::path& ArgsManager::GetBlocksDirPath() const if (!path.empty()) return path; if (IsArgSet("-blocksdir")) { - path = fs::system_complete(GetArg("-blocksdir", "")); + path = fs::system_complete(fs::PathFromString(GetArg("-blocksdir", ""))); if (!fs::is_directory(path)) { path = ""; return path; @@ -413,7 +417,7 @@ const fs::path& ArgsManager::GetBlocksDirPath() const path = GetDataDirBase(); } - path /= BaseParams().DataDir(); + path /= fs::PathFromString(BaseParams().DataDir()); path /= "blocks"; fs::create_directories(path); path = StripRedundantLastElementsOfPath(path); @@ -431,7 +435,7 @@ const fs::path& ArgsManager::GetDataDir(bool net_specific) const std::string datadir = GetArg("-datadir", ""); if (!datadir.empty()) { - path = fs::system_complete(datadir); + path = fs::system_complete(fs::PathFromString(datadir)); if (!fs::is_directory(path)) { path = ""; return path; @@ -440,7 +444,7 @@ const fs::path& ArgsManager::GetDataDir(bool net_specific) const path = GetDefaultDataDir(); } if (net_specific) - path /= BaseParams().DataDir(); + path /= fs::PathFromString(BaseParams().DataDir()); if (fs::create_directories(path)) { // This is the first run, create wallets subdirectory too @@ -519,7 +523,7 @@ bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const } if (filepath) { std::string settings = GetArg("-settings", BITCOIN_SETTINGS_FILENAME); - *filepath = fsbridge::AbsPathJoin(GetDataDirNet(), temp ? settings + ".tmp" : settings); + *filepath = fsbridge::AbsPathJoin(GetDataDirNet(), fs::PathFromString(temp ? settings + ".tmp" : settings)); } return true; } @@ -550,10 +554,8 @@ bool ArgsManager::ReadSettingsFile(std::vector<std::string>* errors) return false; } for (const auto& setting : m_settings.rw_settings) { - std::string section; - std::string key = setting.first; - (void)InterpretOption(section, key, /* value */ {}); // Split setting key into section and argname - if (!GetArgFlags('-' + key)) { + KeyInfo key = InterpretKey(setting.first); // Split setting key into section and argname + if (!GetArgFlags('-' + key.name)) { LogPrintf("Ignoring unknown rw_settings value %s\n", setting.first); } } @@ -574,7 +576,7 @@ bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors) const return false; } if (!RenameOver(path_tmp, path)) { - SaveErrors({strprintf("Failed renaming settings file %s to %s\n", path_tmp.string(), path.string())}, errors); + SaveErrors({strprintf("Failed renaming settings file %s to %s\n", fs::PathToString(path_tmp), fs::PathToString(path))}, errors); return false; } return true; @@ -591,10 +593,10 @@ std::string ArgsManager::GetArg(const std::string& strArg, const std::string& st return value.isNull() ? strDefault : value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str(); } -int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) const +int64_t ArgsManager::GetIntArg(const std::string& strArg, int64_t nDefault) const { const util::SettingsValue value = GetSetting(strArg); - return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : atoi64(value.get_str()); + return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : LocaleIndependentAtoi<int64_t>(value.get_str()); } bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const @@ -811,12 +813,12 @@ fs::path GetDefaultDataDir() bool CheckDataDirOption() { std::string datadir = gArgs.GetArg("-datadir", ""); - return datadir.empty() || fs::is_directory(fs::system_complete(datadir)); + return datadir.empty() || fs::is_directory(fs::system_complete(fs::PathFromString(datadir))); } fs::path GetConfigFile(const std::string& confPath) { - return AbsPathForConfigVal(fs::path(confPath), false); + return AbsPathForConfigVal(fs::PathFromString(confPath), false); } static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) @@ -872,15 +874,14 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file return false; } for (const std::pair<std::string, std::string>& option : options) { - std::string section; - std::string key = option.first; - util::SettingsValue value = InterpretOption(section, key, option.second); - std::optional<unsigned int> flags = GetArgFlags('-' + key); + KeyInfo key = InterpretKey(option.first); + std::optional<unsigned int> flags = GetArgFlags('-' + key.name); if (flags) { - if (!CheckValid(key, value, *flags, error)) { + std::optional<util::SettingsValue> value = InterpretValue(key, option.second, *flags, error); + if (!value) { return false; } - m_settings.ro_config[section][key].push_back(value); + m_settings.ro_config[key.section][key.name].push_back(*value); } else { if (ignore_invalid_keys) { LogPrintf("Ignoring unknown configuration value %s\n", option.first); @@ -1067,7 +1068,7 @@ bool RenameOver(fs::path src, fs::path dest) return MoveFileExW(src.wstring().c_str(), dest.wstring().c_str(), MOVEFILE_REPLACE_EXISTING) != 0; #else - int rc = std::rename(src.string().c_str(), dest.string().c_str()); + int rc = std::rename(src.c_str(), dest.c_str()); return (rc == 0); #endif /* WIN32 */ } diff --git a/src/util/system.h b/src/util/system.h index 3c1399629c..37d976221b 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -158,12 +158,18 @@ struct SectionInfo class ArgsManager { public: + /** + * Flags controlling how config and command line arguments are validated and + * interpreted. + */ enum Flags : uint32_t { - // Boolean options can accept negation syntax -noOPTION or -noOPTION=1 - ALLOW_BOOL = 0x01, - ALLOW_INT = 0x02, - ALLOW_STRING = 0x04, - ALLOW_ANY = ALLOW_BOOL | ALLOW_INT | ALLOW_STRING, + ALLOW_ANY = 0x01, //!< disable validation + // ALLOW_BOOL = 0x02, //!< unimplemented, draft implementation in #16545 + // ALLOW_INT = 0x04, //!< unimplemented, draft implementation in #16545 + // ALLOW_STRING = 0x08, //!< unimplemented, draft implementation in #16545 + // ALLOW_LIST = 0x10, //!< unimplemented, draft implementation in #16545 + DISALLOW_NEGATION = 0x20, //!< disallow -nofoo syntax + DEBUG_ONLY = 0x100, /* Some options would cause cross-contamination if values for * mainnet were used while running on regtest/testnet (or vice-versa). @@ -327,7 +333,7 @@ protected: * @param nDefault (e.g. 1) * @return command-line argument (0 if invalid number) or default value */ - int64_t GetArg(const std::string& strArg, int64_t nDefault) const; + int64_t GetIntArg(const std::string& strArg, int64_t nDefault) const; /** * Return boolean argument or default value diff --git a/src/validation.cpp b/src/validation.cpp index cc87f98913..f163130a18 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -9,6 +9,7 @@ #include <chain.h> #include <chainparams.h> #include <checkqueue.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_check.h> @@ -24,6 +25,7 @@ #include <node/blockstorage.h> #include <node/coinstats.h> #include <node/ui_interface.h> +#include <node/utxo_snapshot.h> #include <policy/policy.h> #include <policy/rbf.h> #include <policy/settings.h> @@ -217,7 +219,7 @@ bool TestLockPointValidity(CChain& active_chain, const LockPoints* lp) // If there are relative lock times then the maxInputBlock will be set // If there are no relative lock times, the LockPoints don't depend on the chain if (lp->maxInputBlock) { - // Check whether ::ChainActive() is an extension of the block at which the LockPoints + // Check whether active_chain is an extension of the block at which the LockPoints // calculation was valid. If not LockPoints are no longer valid if (!active_chain.Contains(lp->maxInputBlock)) { return false; @@ -353,7 +355,7 @@ void CChainState::MaybeUpdateMempoolForReorg( // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); - } else if (m_mempool->exists((*it)->GetHash())) { + } else if (m_mempool->exists(GenTxid::Txid((*it)->GetHash()))) { vHashUpdate.push_back((*it)->GetHash()); } ++it; @@ -372,8 +374,8 @@ void CChainState::MaybeUpdateMempoolForReorg( LimitMempoolSize( *m_mempool, this->CoinsTip(), - gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, - std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, + std::chrono::hours{gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); } /** @@ -424,10 +426,10 @@ class MemPoolAccept { public: explicit MemPoolAccept(CTxMemPool& mempool, CChainState& active_chainstate) : m_pool(mempool), m_view(&m_dummy), m_viewmempool(&active_chainstate.CoinsTip(), m_pool), m_active_chainstate(active_chainstate), - m_limit_ancestors(gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT)), - m_limit_ancestor_size(gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000), - m_limit_descendants(gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)), - m_limit_descendant_size(gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000) { + m_limit_ancestors(gArgs.GetIntArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT)), + m_limit_ancestor_size(gArgs.GetIntArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000), + m_limit_descendants(gArgs.GetIntArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)), + m_limit_descendant_size(gArgs.GetIntArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000) { } // We put the arguments we're handed into a struct, so we can pass them @@ -448,7 +450,36 @@ public: /** Whether we allow transactions to replace mempool transactions by BIP125 rules. If false, * any transaction spending the same inputs as a transaction in the mempool is considered * a conflict. */ - const bool m_allow_bip125_replacement{true}; + const bool m_allow_bip125_replacement; + + /** Parameters for single transaction mempool validation. */ + static ATMPArgs SingleAccept(const CChainParams& chainparams, int64_t accept_time, + bool bypass_limits, std::vector<COutPoint>& coins_to_uncache, + bool test_accept) { + return ATMPArgs{/* m_chainparams */ chainparams, + /* m_accept_time */ accept_time, + /* m_bypass_limits */ bypass_limits, + /* m_coins_to_uncache */ coins_to_uncache, + /* m_test_accept */ test_accept, + /* m_allow_bip125_replacement */ true, + }; + } + + /** Parameters for test package mempool validation through testmempoolaccept. */ + static ATMPArgs PackageTestAccept(const CChainParams& chainparams, int64_t accept_time, + std::vector<COutPoint>& coins_to_uncache) { + return ATMPArgs{/* m_chainparams */ chainparams, + /* m_accept_time */ accept_time, + /* m_bypass_limits */ false, + /* m_coins_to_uncache */ coins_to_uncache, + /* m_test_accept */ true, + /* m_allow_bip125_replacement */ false, + }; + } + + // No default ctor to avoid exposing details to clients and allowing the possibility of + // mixing up the order of the arguments. Use static functions above instead. + ATMPArgs() = delete; }; // Single transaction acceptance @@ -466,14 +497,29 @@ private: // of checking a given transaction. struct Workspace { explicit Workspace(const CTransactionRef& ptx) : m_ptx(ptx), m_hash(ptx->GetHash()) {} + /** Txids of mempool transactions that this transaction directly conflicts with. */ std::set<uint256> m_conflicts; + /** Iterators to mempool entries that this transaction directly conflicts with. */ + CTxMemPool::setEntries m_iters_conflicting; + /** Iterators to all mempool entries that would be replaced by this transaction, including + * those it directly conflicts with and their descendants. */ CTxMemPool::setEntries m_all_conflicting; + /** All mempool ancestors of this transaction. */ CTxMemPool::setEntries m_ancestors; + /** Mempool entry constructed for this transaction. Constructed in PreChecks() but not + * inserted into the mempool until Finalize(). */ std::unique_ptr<CTxMemPoolEntry> m_entry; + /** Pointers to the transactions that have been removed from the mempool and replaced by + * this transaction, used to return to the MemPoolAccept caller. Only populated if + * validation is successful and the original transactions are removed. */ std::list<CTransactionRef> m_replaced_transactions; - bool m_replacement_transaction; + /** Virtual size of the transaction as used by the mempool, calculated using serialized size + * of the transaction and sigops. */ + int64_t m_vsize; + /** Fees paid by this transaction: total input amounts subtracted by total output amounts. */ CAmount m_base_fees; + /** Base fees + any fee delta set by the user with prioritisetransaction. */ CAmount m_modified_fees; /** Total modified fees of all transactions being replaced. */ CAmount m_conflicting_fees{0}; @@ -481,8 +527,12 @@ private: size_t m_conflicting_size{0}; const CTransactionRef& m_ptx; + /** Txid. */ const uint256& m_hash; TxValidationState m_state; + /** A temporary cache containing serialized transaction data for signature verification. + * Reused across PolicyScriptChecks and ConsensusScriptChecks. */ + PrecomputedTransactionData m_precomputed_txdata; }; // Run the policy checks on a given transaction, excluding any script checks. @@ -491,15 +541,23 @@ private: // only tests that are fast should be done here (to avoid CPU DoS). bool PreChecks(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + // Run checks for mempool replace-by-fee. + bool ReplacementChecks(Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + + // Enforce package mempool ancestor/descendant limits (distinct from individual + // ancestor/descendant limits done in PreChecks). + bool PackageMempoolChecks(const std::vector<CTransactionRef>& txns, + PackageValidationState& package_state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + // Run the script checks using our policy flags. As this can be slow, we should // only invoke this on transactions that have otherwise passed policy checks. - bool PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + bool PolicyScriptChecks(const ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Re-run the script checks, using consensus flags, and try to cache the // result in the scriptcache. This should be done after // PolicyScriptChecks(). This requires that all inputs either be in our // utxo set or in the mempool. - bool ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + bool ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Try to add the transaction to the mempool, removing any conflicts first. // Returns true if the transaction is in the mempool after any size @@ -509,7 +567,7 @@ private: // Compare a package's feerate against minimum allowed. bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs) { - CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); + CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } @@ -535,6 +593,9 @@ private: // in-mempool conflicts; see below). size_t m_limit_descendants; size_t m_limit_descendant_size; + + /** Whether the transaction(s) would replace any mempool transactions. If so, RBF rules apply. */ + bool m_rbf{false}; }; bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) @@ -550,14 +611,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Alias what we need out of ws TxValidationState& state = ws.m_state; - std::set<uint256>& setConflicts = ws.m_conflicts; - CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; - CTxMemPool::setEntries& setAncestors = ws.m_ancestors; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; - bool& fReplacementTransaction = ws.m_replacement_transaction; - CAmount& nModifiedFees = ws.m_modified_fees; - CAmount& nConflictingFees = ws.m_conflicting_fees; - size_t& nConflictingSize = ws.m_conflicting_size; if (!CheckTransaction(tx, state)) { return false; // state filled in by CheckTransaction @@ -585,10 +639,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!CheckFinalTx(m_active_chainstate.m_chain.Tip(), tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); - if (m_pool.exists(GenTxid(true, tx.GetWitnessHash()))) { + if (m_pool.exists(GenTxid::Wtxid(tx.GetWitnessHash()))) { // Exact transaction already exists in the mempool. return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-in-mempool"); - } else if (m_pool.exists(GenTxid(false, tx.GetHash()))) { + } else if (m_pool.exists(GenTxid::Txid(tx.GetHash()))) { // Transaction with the same non-witness data but different witness (same txid, different // wtxid) already exists in the mempool. return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-same-nonwitness-data-in-mempool"); @@ -603,7 +657,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Transaction conflicts with a mempool tx, but we're not allowing replacements. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "bip125-replacement-disallowed"); } - if (!setConflicts.count(ptxConflicting->GetHash())) + if (!ws.m_conflicts.count(ptxConflicting->GetHash())) { // Transactions that don't explicitly signal replaceability are // *not* replaceable with the current logic, even if one of their @@ -616,7 +670,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); } - setConflicts.insert(ptxConflicting->GetHash()); + ws.m_conflicts.insert(ptxConflicting->GetHash()); } } } @@ -680,9 +734,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) int64_t nSigOpsCost = GetTransactionSigOpCost(tx, m_view, STANDARD_SCRIPT_VERIFY_FLAGS); - // nModifiedFees includes any fee deltas from PrioritiseTransaction - nModifiedFees = ws.m_base_fees; - m_pool.ApplyDelta(hash, nModifiedFees); + // ws.m_modified_fees includes any fee deltas from PrioritiseTransaction + ws.m_modified_fees = ws.m_base_fees; + m_pool.ApplyDelta(hash, ws.m_modified_fees); // Keep track of transactions that spend a coinbase, which we re-scan // during reorgs to ensure COINBASE_MATURITY is still met. @@ -697,7 +751,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), fSpendsCoinbase, nSigOpsCost, lp)); - unsigned int nSize = entry->GetTxSize(); + ws.m_vsize = entry->GetTxSize(); if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-too-many-sigops", @@ -705,11 +759,11 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // No transactions are allowed below minRelayTxFee except from disconnected // blocks - if (!bypass_limits && !CheckFeeRate(nSize, nModifiedFees, state)) return false; + if (!bypass_limits && !CheckFeeRate(ws.m_vsize, ws.m_modified_fees, state)) return false; - const CTxMemPool::setEntries setIterConflicting = m_pool.GetIterSet(setConflicts); + ws.m_iters_conflicting = m_pool.GetIterSet(ws.m_conflicts); // Calculate in-mempool ancestors, up to a limit. - if (setConflicts.size() == 1) { + if (ws.m_conflicts.size() == 1) { // In general, when we receive an RBF transaction with mempool conflicts, we want to know whether we // would meet the chain limits after the conflicts have been removed. However, there isn't a practical // way to do this short of calculating the ancestor and descendant sets with an overlay cache of @@ -737,16 +791,16 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // the ancestor limits should be the same for both our new transaction and any conflicts). // We don't bother incrementing m_limit_descendants by the full removal count as that limit never comes // into force here (as we're only adding a single transaction). - assert(setIterConflicting.size() == 1); - CTxMemPool::txiter conflict = *setIterConflicting.begin(); + assert(ws.m_iters_conflicting.size() == 1); + CTxMemPool::txiter conflict = *ws.m_iters_conflicting.begin(); m_limit_descendants += 1; m_limit_descendant_size += conflict->GetSizeWithDescendants(); } std::string errString; - if (!m_pool.CalculateMemPoolAncestors(*entry, setAncestors, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, m_limit_descendant_size, errString)) { - setAncestors.clear(); + if (!m_pool.CalculateMemPoolAncestors(*entry, ws.m_ancestors, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, m_limit_descendant_size, errString)) { + ws.m_ancestors.clear(); // If CalculateMemPoolAncestors fails second time, we want the original error string. std::string dummy_err_string; // Contracting/payment channels CPFP carve-out: @@ -760,57 +814,85 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // to be secure by simply only having two immediately-spendable // outputs - one for each counterparty. For more info on the uses for // this, see https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html - if (nSize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || - !m_pool.CalculateMemPoolAncestors(*entry, setAncestors, 2, m_limit_ancestor_size, m_limit_descendants + 1, m_limit_descendant_size + EXTRA_DESCENDANT_TX_SIZE_LIMIT, dummy_err_string)) { + if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || + !m_pool.CalculateMemPoolAncestors(*entry, ws.m_ancestors, 2, m_limit_ancestor_size, m_limit_descendants + 1, m_limit_descendant_size + EXTRA_DESCENDANT_TX_SIZE_LIMIT, dummy_err_string)) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", errString); } } // A transaction that spends outputs that would be replaced by it is invalid. Now // that we have the set of all ancestors we can detect this - // pathological case by making sure setConflicts and setAncestors don't + // pathological case by making sure ws.m_conflicts and ws.m_ancestors don't // intersect. - if (const auto err_string{EntriesAndTxidsDisjoint(setAncestors, setConflicts, hash)}) { + if (const auto err_string{EntriesAndTxidsDisjoint(ws.m_ancestors, ws.m_conflicts, hash)}) { + // We classify this as a consensus error because a transaction depending on something it + // conflicts with would be inconsistent. return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-spends-conflicting-tx", *err_string); } + m_rbf = !ws.m_conflicts.empty(); + return true; +} - // If we don't hold the lock allConflicting might be incomplete; the - // subsequent RemoveStaged() and addUnchecked() calls don't guarantee - // mempool consistency for us. - fReplacementTransaction = setConflicts.size(); - if (fReplacementTransaction) - { - CFeeRate newFeeRate(nModifiedFees, nSize); - if (const auto err_string{PaysMoreThanConflicts(setIterConflicting, newFeeRate, hash)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); - } +bool MemPoolAccept::ReplacementChecks(Workspace& ws) +{ + AssertLockHeld(cs_main); + AssertLockHeld(m_pool.cs); - // Calculate all conflicting entries and enforce Rule #5. - if (const auto err_string{GetEntriesForConflicts(tx, m_pool, setIterConflicting, allConflicting)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, - "too many potential replacements", *err_string); - } - // Enforce Rule #2. - if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, setIterConflicting)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, - "replacement-adds-unconfirmed", *err_string); - } + const CTransaction& tx = *ws.m_ptx; + const uint256& hash = ws.m_hash; + TxValidationState& state = ws.m_state; - // Check if it's economically rational to mine this transaction rather - // than the ones it replaces. Enforce Rules #3 and #4. - for (CTxMemPool::txiter it : allConflicting) { - nConflictingFees += it->GetModifiedFee(); - nConflictingSize += it->GetTxSize(); - } - if (const auto err_string{PaysForRBF(nConflictingFees, nModifiedFees, nSize, hash)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); - } + CFeeRate newFeeRate(ws.m_modified_fees, ws.m_vsize); + // It's possible that the replacement pays more fees than its direct conflicts but not more + // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the + // replacement doesn't pay more fees than its direct conflicts, then we can be sure it's not + // more economically rational to mine. Before we go digging through the mempool for all + // transactions that would need to be removed (direct conflicts and all descendants), check + // that the replacement transaction pays more than its direct conflicts. + if (const auto err_string{PaysMoreThanConflicts(ws.m_iters_conflicting, newFeeRate, hash)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); + } + + // Calculate all conflicting entries and enforce BIP125 Rule #5. + if (const auto err_string{GetEntriesForConflicts(tx, m_pool, ws.m_iters_conflicting, ws.m_all_conflicting)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + "too many potential replacements", *err_string); + } + // Enforce BIP125 Rule #2. + if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, ws.m_iters_conflicting)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + "replacement-adds-unconfirmed", *err_string); + } + // Check if it's economically rational to mine this transaction rather than the ones it + // replaces and pays for its own relay fees. Enforce BIP125 Rules #3 and #4. + for (CTxMemPool::txiter it : ws.m_all_conflicting) { + ws.m_conflicting_fees += it->GetModifiedFee(); + ws.m_conflicting_size += it->GetTxSize(); + } + if (const auto err_string{PaysForRBF(ws.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, + ::incrementalRelayFee, hash)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } return true; } -bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txns, + PackageValidationState& package_state) +{ + AssertLockHeld(cs_main); + AssertLockHeld(m_pool.cs); + + std::string err_string; + if (!m_pool.CheckPackageLimits(txns, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, + m_limit_descendant_size, err_string)) { + // This is a package-wide error, separate from an individual transaction error. + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", err_string); + } + return true; +} + +bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws) { const CTransaction& tx = *ws.m_ptx; TxValidationState& state = ws.m_state; @@ -819,13 +901,13 @@ bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, Prec // Check input scripts and signatures. // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!CheckInputScripts(tx, state, m_view, scriptVerifyFlags, true, false, txdata)) { + if (!CheckInputScripts(tx, state, m_view, scriptVerifyFlags, true, false, ws.m_precomputed_txdata)) { // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we // need to turn both off, and compare against just turning off CLEANSTACK // to see if the failure is specifically due to witness validation. TxValidationState state_dummy; // Want reported failures to be from first CheckInputScripts - if (!tx.HasWitness() && CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && - !CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { + if (!tx.HasWitness() && CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, ws.m_precomputed_txdata) && + !CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, ws.m_precomputed_txdata)) { // Only the witness is missing, so the transaction itself may be fine. state.Invalid(TxValidationResult::TX_WITNESS_STRIPPED, state.GetRejectReason(), state.GetDebugMessage()); @@ -836,7 +918,7 @@ bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, Prec return true; } -bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; @@ -859,7 +941,8 @@ bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, P // invalid blocks (using TestBlockValidity), however allowing such // transactions into the mempool can be exploited as a DoS attack. unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(m_active_chainstate.m_chain.Tip(), chainparams.GetConsensus()); - if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata, m_active_chainstate.CoinsTip())) { + if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, + ws.m_precomputed_txdata, m_active_chainstate.CoinsTip())) { return error("%s: BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s", __func__, hash.ToString(), state.ToString()); } @@ -874,40 +957,33 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) TxValidationState& state = ws.m_state; const bool bypass_limits = args.m_bypass_limits; - CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; - CTxMemPool::setEntries& setAncestors = ws.m_ancestors; - const CAmount& nModifiedFees = ws.m_modified_fees; - const CAmount& nConflictingFees = ws.m_conflicting_fees; - const size_t& nConflictingSize = ws.m_conflicting_size; - const bool fReplacementTransaction = ws.m_replacement_transaction; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; // Remove conflicting transactions from the mempool - for (CTxMemPool::txiter it : allConflicting) + for (CTxMemPool::txiter it : ws.m_all_conflicting) { LogPrint(BCLog::MEMPOOL, "replacing tx %s with %s for %s additional fees, %d delta bytes\n", it->GetTx().GetHash().ToString(), hash.ToString(), - FormatMoney(nModifiedFees - nConflictingFees), - (int)entry->GetTxSize() - (int)nConflictingSize); + FormatMoney(ws.m_modified_fees - ws.m_conflicting_fees), + (int)entry->GetTxSize() - (int)ws.m_conflicting_size); ws.m_replaced_transactions.push_back(it->GetSharedTx()); } - m_pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); + m_pool.RemoveStaged(ws.m_all_conflicting, false, MemPoolRemovalReason::REPLACED); // This transaction should only count for fee estimation if: - // - it isn't a BIP 125 replacement transaction (may not be widely supported) // - it's not being re-added during a reorg which bypasses typical mempool fee limits // - the node is not behind // - the transaction is not dependent on any other transactions in the mempool - bool validForFeeEstimation = !fReplacementTransaction && !bypass_limits && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); + bool validForFeeEstimation = !bypass_limits && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); // Store transaction in memory - m_pool.addUnchecked(*entry, setAncestors, validForFeeEstimation); + m_pool.addUnchecked(*entry, ws.m_ancestors, validForFeeEstimation); // trim mempool and check if tx was trimmed if (!bypass_limits) { - LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip(), gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); - if (!m_pool.exists(hash)) + LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip(), gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + if (!m_pool.exists(GenTxid::Txid(hash))) return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); } return true; @@ -922,26 +998,24 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef if (!PreChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); - // Only compute the precomputed transaction data if we need to verify - // scripts (ie, other policy checks pass). We perform the inexpensive - // checks first and avoid hashing and signature verification unless those - // checks pass, to mitigate CPU exhaustion denial-of-service attacks. - PrecomputedTransactionData txdata; + if (m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); - if (!PolicyScriptChecks(args, ws, txdata)) return MempoolAcceptResult::Failure(ws.m_state); + // Perform the inexpensive checks first and avoid hashing and signature verification unless + // those checks pass, to mitigate CPU exhaustion denial-of-service attacks. + if (!PolicyScriptChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); - if (!ConsensusScriptChecks(args, ws, txdata)) return MempoolAcceptResult::Failure(ws.m_state); + if (!ConsensusScriptChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); // Tx was accepted, but not added if (args.m_test_accept) { - return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_base_fees); + return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees); } if (!Finalize(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence()); - return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_base_fees); + return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees); } PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std::vector<CTransactionRef>& txns, ATMPArgs& args) @@ -980,18 +1054,12 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // because it's unnecessary. Also, CPFP carve out can increase the limit for individual // transactions, but this exemption is not extended to packages in CheckPackageLimits(). std::string err_string; - if (txns.size() > 1 && - !m_pool.CheckPackageLimits(txns, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, - m_limit_descendant_size, err_string)) { - // All transactions must have individually passed mempool ancestor and descendant limits - // inside of PreChecks(), so this is separate from an individual transaction error. - package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", err_string); + if (txns.size() > 1 && !PackageMempoolChecks(txns, package_state)) { return PackageMempoolAcceptResult(package_state, std::move(results)); } for (Workspace& ws : workspaces) { - PrecomputedTransactionData txdata; - if (!PolicyScriptChecks(args, ws, txdata)) { + if (!PolicyScriptChecks(args, ws)) { // Exit early to avoid doing pointless work. Update the failed tx result; the rest are unfinished. package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); @@ -1001,7 +1069,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // When test_accept=true, transactions that pass PolicyScriptChecks are valid because there are // no further mempool checks (passing PolicyScriptChecks implies passing ConsensusScriptChecks). results.emplace(ws.m_ptx->GetWitnessHash(), - MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_base_fees)); + MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), + ws.m_vsize, ws.m_base_fees)); } } @@ -1018,9 +1087,7 @@ static MempoolAcceptResult AcceptToMemoryPoolWithTime(const CChainParams& chainp EXCLUSIVE_LOCKS_REQUIRED(cs_main) { std::vector<COutPoint> coins_to_uncache; - MemPoolAccept::ATMPArgs args { chainparams, nAcceptTime, bypass_limits, coins_to_uncache, - test_accept, /* m_allow_bip125_replacement */ true }; - + auto args = MemPoolAccept::ATMPArgs::SingleAccept(chainparams, nAcceptTime, bypass_limits, coins_to_uncache, test_accept); const MempoolAcceptResult result = MemPoolAccept(pool, active_chainstate).AcceptSingleTransaction(tx, args); if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { // Remove coins that were not present in the coins cache before calling @@ -1053,8 +1120,7 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx std::vector<COutPoint> coins_to_uncache; const CChainParams& chainparams = Params(); - MemPoolAccept::ATMPArgs args { chainparams, GetTime(), /* bypass_limits */ false, coins_to_uncache, - test_accept, /* m_allow_bip125_replacement */ false }; + auto args = MemPoolAccept::ATMPArgs::PackageTestAccept(chainparams, GetTime(), coins_to_uncache); const PackageMempoolAcceptResult result = MemPoolAccept(pool, active_chainstate).AcceptMultipleTransactions(package, args); // Uncache coins pertaining to transactions that were not submitted to the mempool. @@ -1090,10 +1156,15 @@ void CoinsViews::InitCache() m_cacheview = std::make_unique<CCoinsViewCache>(&m_catcherview); } -CChainState::CChainState(CTxMemPool* mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash) +CChainState::CChainState( + CTxMemPool* mempool, + BlockManager& blockman, + ChainstateManager& chainman, + std::optional<uint256> from_snapshot_blockhash) : m_mempool(mempool), m_params(::Params()), m_blockman(blockman), + m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( @@ -1230,12 +1301,6 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund AddCoins(inputs, tx, nHeight); } -void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) -{ - CTxUndo txundo; - UpdateCoins(tx, inputs, txundo, nHeight); -} - bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; const CScriptWitness *witness = &ptxTo->vin[nIn].scriptWitness; @@ -1263,7 +1328,7 @@ void InitScriptExecutionCache() { g_scriptExecutionCacheHasher.Write(nonce.begin(), 32); // nMaxCacheSize is unsigned. If -maxsigcachesize is set to zero, // setup_bytes creates the minimum possible cache (2 elements). - size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); + size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetIntArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); size_t nElems = g_scriptExecutionCache.setup_bytes(nMaxCacheSize); LogPrintf("Using %zu MiB out of %zu/2 requested for script execution cache, able to store %zu elements\n", (nElems*sizeof(uint256)) >>20, (nMaxCacheSize*2)>>20, nElems); @@ -1567,7 +1632,6 @@ static int64_t nTimeForks = 0; static int64_t nTimeVerify = 0; static int64_t nTimeConnect = 0; static int64_t nTimeIndex = 0; -static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; static int64_t nBlocksTotal = 0; @@ -1659,8 +1723,6 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // can be duplicated to remove the ability to spend the first instance -- even after // being sent to another address. // See BIP30, CVE-2012-1909, and http://r6.ca/blog/20120206T005236Z.html for more information. - // This logic is not necessary for memory pool transactions, as AcceptToMemoryPool - // already refuses previously-known transaction ids entirely. // This rule was originally applied to all blocks with a timestamp after March 15, 2012, 0:00 UTC. // Now that the whole chain is irreversibly beyond that time it is applied to all blocks except the // two in the chain that violate it. This prevents exploiting the issue against nodes during their @@ -1872,17 +1934,13 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); - int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; - LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime6 - nTime5), nTimeCallbacks * MICRO, nTimeCallbacks * MILLI / nBlocksTotal); - - TRACE7(validation, block_connected, - block.GetHash().ToString().c_str(), + TRACE6(validation, block_connected, + block.GetHash().data(), pindex->nHeight, block.vtx.size(), nInputs, nSigOpsCost, - GetTimeMicros() - nTimeStart, // in microseconds (µs) - block.GetHash().data() + GetTimeMicros() - nTimeStart // in microseconds (µs) ); return true; @@ -1892,7 +1950,7 @@ CoinsCacheSizeState CChainState::GetCoinsCacheSizeState() { return this->GetCoinsCacheSizeState( m_coinstip_cache_size_bytes, - gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( @@ -2027,8 +2085,8 @@ bool CChainState::FlushStateToDisk( } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { - LOG_TIME_SECONDS(strprintf("write coins cache to disk (%d coins, %.2fkB)", - coins_count, coins_mem_usage / 1000)); + LOG_TIME_MILLIS_WITH_CATEGORY(strprintf("write coins cache to disk (%d coins, %.2fkB)", + coins_count, coins_mem_usage / 1000), BCLog::BENCH); // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written @@ -2089,8 +2147,42 @@ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) res += warn; } +static void UpdateTipLog( + const CCoinsViewCache& coins_tip, + const CBlockIndex* tip, + const CChainParams& params, + const std::string& func_name, + const std::string& prefix, + const std::string& warning_messages) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ + + AssertLockHeld(::cs_main); + LogPrintf("%s%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", + prefix, func_name, + tip->GetBlockHash().ToString(), tip->nHeight, tip->nVersion, + log(tip->nChainWork.getdouble()) / log(2.0), (unsigned long)tip->nChainTx, + FormatISO8601DateTime(tip->GetBlockTime()), + GuessVerificationProgress(params.TxData(), tip), + coins_tip.DynamicMemoryUsage() * (1.0 / (1 << 20)), + coins_tip.GetCacheSize(), + !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages) : ""); +} + void CChainState::UpdateTip(const CBlockIndex* pindexNew) { + const auto& coins_tip = this->CoinsTip(); + + // The remainder of the function isn't relevant if we are not acting on + // the active chainstate, so return if need be. + if (this != &m_chainman.ActiveChainstate()) { + // Only log every so often so that we don't bury log messages at the tip. + constexpr int BACKGROUND_LOG_INTERVAL = 2000; + if (pindexNew->nHeight % BACKGROUND_LOG_INTERVAL == 0) { + UpdateTipLog(coins_tip, pindexNew, m_params, __func__, "[background validation] ", ""); + } + return; + } + // New best block if (m_mempool) { m_mempool->AddTransactionsUpdated(1); @@ -2118,12 +2210,7 @@ void CChainState::UpdateTip(const CBlockIndex* pindexNew) } } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, - pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, - log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, - FormatISO8601DateTime(pindexNew->GetBlockTime()), - GuessVerificationProgress(m_params.TxData(), pindexNew), this->CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), this->CoinsTip().GetCacheSize(), - !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages.original) : ""); + UpdateTipLog(coins_tip, pindexNew, m_params, __func__, "", warning_messages.original); } /** Disconnect m_chain's tip. @@ -2457,7 +2544,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex // any disconnected transactions back to the mempool. MaybeUpdateMempoolForReorg(disconnectpool, true); } - if (m_mempool) m_mempool->check(*this); + if (m_mempool) m_mempool->check(this->CoinsTip(), this->m_chain.Height() + 1); CheckForkWarningConditions(); @@ -2517,7 +2604,7 @@ bool CChainState::ActivateBestChain(BlockValidationState& state, std::shared_ptr CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; - int nStopAtHeight = gArgs.GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); + int nStopAtHeight = gArgs.GetIntArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during @@ -3174,7 +3261,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) { - LogPrintf("ERROR: %s: block %s is marked invalid\n", __func__, hash.ToString()); + LogPrint(BCLog::VALIDATION, "%s: block %s is marked invalid\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_CACHED_INVALID, "duplicate"); } return true; @@ -3189,16 +3276,18 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS CBlockIndex* pindexPrev = nullptr; BlockMap::iterator mi = m_block_index.find(block.hashPrevBlock); if (mi == m_block_index.end()) { - LogPrintf("ERROR: %s: prev block not found\n", __func__); + LogPrint(BCLog::VALIDATION, "%s: %s prev block not found\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); } pindexPrev = (*mi).second; if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { - LogPrintf("ERROR: %s: prev block invalid\n", __func__); + LogPrint(BCLog::VALIDATION, "%s: %s prev block invalid\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } - if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) - return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); + if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) { + LogPrint(BCLog::VALIDATION, "%s: Consensus::ContextualCheckBlockHeader: %s, %s\n", __func__, hash.ToString(), state.ToString()); + return false; + } /* Determine if this block descends from any block which has been found * invalid (m_failed_blocks), then mark pindexPrev and any blocks between @@ -3233,7 +3322,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS setDirtyBlockIndex.insert(invalid_walk); invalid_walk = invalid_walk->pprev; } - LogPrintf("ERROR: %s: prev block invalid\n", __func__); + LogPrint(BCLog::VALIDATION, "%s: %s prev block invalid\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } } @@ -3397,6 +3486,19 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s return true; } +MempoolAcceptResult ChainstateManager::ProcessTransaction(const CTransactionRef& tx, bool test_accept) +{ + CChainState& active_chainstate = ActiveChainstate(); + if (!active_chainstate.m_mempool) { + TxValidationState state; + state.Invalid(TxValidationResult::TX_NO_MEMPOOL, "no-mempool"); + return MempoolAcceptResult::Failure(state); + } + auto result = AcceptToMemoryPool(active_chainstate, *active_chainstate.m_mempool, tx, /*bypass_limits=*/ false, test_accept); + active_chainstate.m_mempool->check(active_chainstate.CoinsTip(), active_chainstate.m_chain.Height() + 1); + return result; +} + bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, CChainState& chainstate, @@ -3620,7 +3722,9 @@ bool BlockManager::LoadBlockIndex( pindex->nStatus |= BLOCK_FAILED_CHILD; setDirtyBlockIndex.insert(pindex); } - if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr)) { + if (pindex->IsAssumedValid() || + (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && + (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { block_index_candidates.insert(pindex); } if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) @@ -3703,7 +3807,7 @@ bool BlockManager::LoadBlockIndexDB(std::set<CBlockIndex*, CBlockIndexWorkCompar void CChainState::LoadMempool(const ArgsManager& args) { if (!m_mempool) return; - if (args.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + if (args.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { ::LoadMempool(*m_mempool, *this); } m_mempool->SetIsLoaded(!ShutdownRequested()); @@ -4199,12 +4303,33 @@ void CChainState::CheckBlockIndex() while (pindex != nullptr) { nNodes++; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; - if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + // Assumed-valid index entries will not have data since we haven't downloaded the + // full block yet. + if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA) && !pindex->IsAssumedValid()) { + pindexFirstMissing = pindex; + } if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotTransactionsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotChainValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotScriptsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; + + if (pindex->pprev != nullptr && !pindex->IsAssumedValid()) { + // Skip validity flag checks for BLOCK_ASSUMED_VALID index entries, since these + // *_VALID_MASK flags will not be present for index entries we are temporarily assuming + // valid. + if (pindexFirstNotTransactionsValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) { + pindexFirstNotTransactionsValid = pindex; + } + + if (pindexFirstNotChainValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) { + pindexFirstNotChainValid = pindex; + } + + if (pindexFirstNotScriptsValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) { + pindexFirstNotScriptsValid = pindex; + } + } // Begin: actual consistency checks. if (pindex->pprev == nullptr) { @@ -4215,7 +4340,9 @@ void CChainState::CheckBlockIndex() if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. - if (!fHavePruned) { + // Unless these indexes are assumed valid and pending block download on a + // background chainstate. + if (!fHavePruned && !pindex->IsAssumedValid()) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); assert(pindexFirstMissing == pindexFirstNeverProcessed); @@ -4224,7 +4351,16 @@ void CChainState::CheckBlockIndex() if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); } if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); - assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + if (pindex->IsAssumedValid()) { + // Assumed-valid blocks should have some nTx value. + assert(pindex->nTx > 0); + // Assumed-valid blocks should connect to the main chain. + assert((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE); + } else { + // Otherwise there should only be an nTx value if we have + // actually seen a block's transactions. + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + } // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to HaveTxsDownloaded(). assert((pindexFirstNeverProcessed == nullptr) == pindex->HaveTxsDownloaded()); assert((pindexFirstNotTransactionsValid == nullptr) == pindex->HaveTxsDownloaded()); @@ -4241,11 +4377,17 @@ void CChainState::CheckBlockIndex() } if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { if (pindexFirstInvalid == nullptr) { + const bool is_active = this == &m_chainman.ActiveChainstate(); + // If this block sorts at least as good as the current tip and // is valid and we have all data for its parents, it must be in // setBlockIndexCandidates. m_chain.Tip() must also be there // even if some data has been pruned. - if (pindexFirstMissing == nullptr || pindex == m_chain.Tip()) { + // + // Don't perform this check for the background chainstate since + // its setBlockIndexCandidates shouldn't have some entries (i.e. those past the + // snapshot block) which do exist in the block index for the active chainstate. + if (is_active && (pindexFirstMissing == nullptr || pindex == m_chain.Tip())) { assert(setBlockIndexCandidates.count(pindex)); } // If some parent is missing, then it could be that this block was in @@ -4383,7 +4525,7 @@ static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) { const CChainParams& chainparams = Params(); - int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; + int64_t nExpiryTimeout = gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat", "rb")}; CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { @@ -4428,7 +4570,7 @@ bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mocka // wallet(s) having loaded it while we were processing // mempool transactions; consider these as valid, instead of // failed, but mark them as 'already there' - if (pool.exists(tx->GetHash())) { + if (pool.exists(GenTxid::Txid(tx->GetHash()))) { ++already_there; } else { ++failed; @@ -4580,7 +4722,7 @@ CChainState& ChainstateManager::InitializeChainstate( if (to_modify) { throw std::logic_error("should not be overwriting a chainstate"); } - to_modify.reset(new CChainState(mempool, m_blockman, snapshot_blockhash)); + to_modify.reset(new CChainState(mempool, m_blockman, *this, snapshot_blockhash)); // Snapshot chainstates and initial IBD chaintates always become active. if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { @@ -4649,8 +4791,9 @@ bool ChainstateManager::ActivateSnapshot( static_cast<size_t>(current_coinsdb_cache_size * IBD_CACHE_PERC)); } - auto snapshot_chainstate = WITH_LOCK(::cs_main, return std::make_unique<CChainState>( - /* mempool */ nullptr, m_blockman, base_blockhash)); + auto snapshot_chainstate = WITH_LOCK(::cs_main, + return std::make_unique<CChainState>( + /* mempool */ nullptr, m_blockman, *this, base_blockhash)); { LOCK(::cs_main); @@ -4853,11 +4996,25 @@ bool ChainstateManager::PopulateAndValidateSnapshot( // Fake nChainTx so that GuessVerificationProgress reports accurately index->nChainTx = index->pprev ? index->pprev->nChainTx + index->nTx : 1; + // Mark unvalidated block index entries beneath the snapshot base block as assumed-valid. + if (!index->IsValid(BLOCK_VALID_SCRIPTS)) { + // This flag will be removed once the block is fully validated by a + // background chainstate. + index->nStatus |= BLOCK_ASSUMED_VALID; + } + // Fake BLOCK_OPT_WITNESS so that CChainState::NeedsRedownload() // won't ask to rewind the entire assumed-valid chain on startup. if (index->pprev && DeploymentActiveAt(*index, ::Params().GetConsensus(), Consensus::DEPLOYMENT_SEGWIT)) { index->nStatus |= BLOCK_OPT_WITNESS; } + + setDirtyBlockIndex.insert(index); + // Changes to the block index will be flushed to disk after this call + // returns in `ActivateSnapshot()`, when `MaybeRebalanceCaches()` is + // called, since we've added a snapshot chainstate and therefore will + // have to downsize the IBD chainstate, which will result in a call to + // `FlushStateToDisk(ALWAYS)`. } assert(index); @@ -4882,22 +5039,6 @@ bool ChainstateManager::IsSnapshotActive() const return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } -CChainState& ChainstateManager::ValidatedChainstate() const -{ - LOCK(::cs_main); - if (m_snapshot_chainstate && IsSnapshotValidated()) { - return *m_snapshot_chainstate.get(); - } - assert(m_ibd_chainstate); - return *m_ibd_chainstate.get(); -} - -bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const -{ - LOCK(::cs_main); - return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); -} - void ChainstateManager::Unload() { for (CChainState* chainstate : this->GetAll()) { diff --git a/src/validation.h b/src/validation.h index 078b988052..21cd389757 100644 --- a/src/validation.h +++ b/src/validation.h @@ -10,20 +10,14 @@ #include <config/bitcoin-config.h> #endif -#include <amount.h> #include <arith_uint256.h> #include <attributes.h> #include <chain.h> -#include <coins.h> -#include <consensus/validation.h> -#include <crypto/common.h> // for ReadLE64 +#include <consensus/amount.h> #include <fs.h> -#include <node/utxo_snapshot.h> #include <policy/feerate.h> #include <policy/packages.h> -#include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> -#include <serialize.h> #include <sync.h> #include <txdb.h> #include <txmempool.h> // For CTxMemPool::cs @@ -44,18 +38,13 @@ #include <vector> class CChainState; -class BlockValidationState; class CBlockTreeDB; -class CBlockUndo; class CChainParams; struct CCheckpointData; -class CInv; -class CConnman; -class CScriptCheck; class CTxMemPool; class ChainstateManager; +class SnapshotMetadata; struct ChainTxData; - struct DisconnectedBlockTransactions; struct PrecomputedTransactionData; struct LockPoints; @@ -86,7 +75,7 @@ static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; -/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; static const unsigned int DEFAULT_CHECKLEVEL = 3; @@ -111,6 +100,7 @@ extern RecursiveMutex cs_main; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; +/** Used to notify getblocktemplate RPC of new tips. */ extern uint256 g_best_block; /** Whether there are dedicated script-checking threads running. * False indicates all script checking is done on the main threadMessageHandler thread. @@ -168,14 +158,16 @@ struct MempoolAcceptResult { // The following fields are only present when m_result_type = ResultType::VALID /** Mempool transactions replaced by the tx per BIP 125 rules. */ const std::optional<std::list<CTransactionRef>> m_replaced_transactions; + /** Virtual size as used by the mempool, calculated using serialized size and sigops. */ + const std::optional<int64_t> m_vsize; /** Raw base fees in satoshis. */ const std::optional<CAmount> m_base_fees; static MempoolAcceptResult Failure(TxValidationState state) { return MempoolAcceptResult(state); } - static MempoolAcceptResult Success(std::list<CTransactionRef>&& replaced_txns, CAmount fees) { - return MempoolAcceptResult(std::move(replaced_txns), fees); + static MempoolAcceptResult Success(std::list<CTransactionRef>&& replaced_txns, int64_t vsize, CAmount fees) { + return MempoolAcceptResult(std::move(replaced_txns), vsize, fees); } // Private constructors. Use static methods MempoolAcceptResult::Success, etc. to construct. @@ -187,9 +179,9 @@ private: } /** Constructor for success case */ - explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, CAmount fees) + explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, int64_t vsize, CAmount fees) : m_result_type(ResultType::VALID), - m_replaced_transactions(std::move(replaced_txns)), m_base_fees(fees) {} + m_replaced_transactions(std::move(replaced_txns)), m_vsize{vsize}, m_base_fees(fees) {} }; /** @@ -216,9 +208,16 @@ struct PackageMempoolAcceptResult }; /** - * (Try to) add a transaction to the memory pool. - * @param[in] bypass_limits When true, don't enforce mempool fee limits. - * @param[in] test_accept When true, run validation checks but don't submit to mempool. + * Try to add a transaction to the mempool. This is an internal function and is + * exposed only for testing. Client code should use ChainstateManager::ProcessTransaction() + * + * @param[in] active_chainstate Reference to the active chainstate. + * @param[in] pool Reference to the node's mempool. + * @param[in] tx The transaction to submit for mempool acceptance. + * @param[in] bypass_limits When true, don't enforce mempool fee and capacity limits. + * @param[in] test_accept When true, run validation checks but don't submit to mempool. + * + * @returns a MempoolAcceptResult indicating whether the transaction was accepted/rejected with reason. */ MempoolAcceptResult AcceptToMemoryPool(CChainState& active_chainstate, CTxMemPool& pool, const CTransactionRef& tx, bool bypass_limits, bool test_accept=false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -239,9 +238,6 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx const Package& txns, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Apply the effects of this transaction on the UTXO set represented by view */ -void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); - /** Transaction validation functions */ /** @@ -595,9 +591,15 @@ public: //! CChainState instances. BlockManager& m_blockman; + //! The chainstate manager that owns this chainstate. The reference is + //! necessary so that this instance can check whether it is the active + //! chainstate within deeply nested method calls. + ChainstateManager& m_chainman; + explicit CChainState( CTxMemPool* mempool, BlockManager& blockman, + ChainstateManager& chainman, std::optional<uint256> from_snapshot_blockhash = std::nullopt); /** @@ -634,9 +636,10 @@ public: const std::optional<uint256> m_from_snapshot_blockhash; /** - * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and - * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be - * missing the data for the block. + * The set of all CBlockIndex entries with either BLOCK_VALID_TRANSACTIONS (for + * itself and all ancestors) *or* BLOCK_ASSUMED_VALID (if using background + * chainstates) and as good as our current tip or better. Entries may be failed, + * though, and pruning nodes may be missing the data for the block. */ std::set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; @@ -848,12 +851,6 @@ private: * *Background IBD chainstate*: an IBD chainstate for which the * IBD process is happening in the background while use of the * active (snapshot) chainstate allows the rest of the system to function. - * - * *Validated chainstate*: the most-work chainstate which has been validated - * locally via initial block download. This will be the snapshot chainstate - * if a snapshot was loaded and all blocks up to the snapshot starting point - * have been downloaded and validated (via background validation), otherwise - * it will be the IBD chainstate. */ class ChainstateManager { @@ -972,19 +969,6 @@ public: //! Is there a snapshot in use and has it been fully validated? bool IsSnapshotValidated() const { return m_snapshot_validated; } - //! @returns true if this chainstate is being used to validate an active - //! snapshot in the background. - bool IsBackgroundIBD(CChainState* chainstate) const; - - //! Return the most-work chainstate that has been fully validated. - //! - //! During background validation of a snapshot, this is the IBD chain. After - //! background validation has completed, this is the snapshot chain. - CChainState& ValidatedChainstate() const; - - CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } - CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } - /** * Process an incoming block. This only returns after the best known valid * block is made active. Note that it does not, however, guarantee that the @@ -1019,6 +1003,15 @@ public: */ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); + /** + * Try to add a transaction to the memory pool. + * + * @param[in] tx The transaction to submit for mempool acceptance. + * @param[in] test_accept When true, run validation checks but don't submit to mempool. + */ + [[nodiscard]] MempoolAcceptResult ProcessTransaction(const CTransactionRef& tx, bool test_accept=false) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Load the block tree and coins database from disk, initializing state if we're running with -reindex bool LoadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 1dc23374e3..74fc10ab25 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -61,9 +61,9 @@ bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory) { LOCK(cs_db); - auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>()); + auto inserted = g_dbenvs.emplace(fs::PathToString(env_directory), std::weak_ptr<BerkeleyEnvironment>()); if (inserted.second) { - auto env = std::make_shared<BerkeleyEnvironment>(env_directory.string()); + auto env = std::make_shared<BerkeleyEnvironment>(env_directory); inserted.first->second = env; return env; } @@ -101,7 +101,7 @@ void BerkeleyEnvironment::Close() if (error_file) fclose(error_file); - UnlockDirectory(strPath, ".walletlock"); + UnlockDirectory(fs::PathFromString(strPath), ".walletlock"); } void BerkeleyEnvironment::Reset() @@ -111,7 +111,7 @@ void BerkeleyEnvironment::Reset() fMockDb = false; } -BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string()) +BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(fs::PathToString(dir_path)) { Reset(); } @@ -129,24 +129,24 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) return true; } - fs::path pathIn = strPath; + fs::path pathIn = fs::PathFromString(strPath); TryCreateDirectories(pathIn); if (!LockDirectory(pathIn, ".walletlock")) { - LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath); - err = strprintf(_("Error initializing wallet database environment %s!"), Directory()); + LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance may be using it.\n", strPath); + err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory()))); return false; } fs::path pathLogDir = pathIn / "database"; TryCreateDirectories(pathLogDir); fs::path pathErrorFile = pathIn / "db.log"; - LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); + LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", fs::PathToString(pathLogDir), fs::PathToString(pathErrorFile)); unsigned int nEnvFlags = 0; if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) nEnvFlags |= DB_PRIVATE; - dbenv->set_lg_dir(pathLogDir.string().c_str()); + dbenv->set_lg_dir(fs::PathToString(pathLogDir).c_str()); dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet dbenv->set_lg_bsize(0x10000); dbenv->set_lg_max(1048576); @@ -173,7 +173,7 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2)); } Reset(); - err = strprintf(_("Error initializing wallet database environment %s!"), Directory()); + err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory()))); if (ret == DB_RUNRECOVERY) { err += Untranslated(" ") + _("This error could occur if this wallet was not shutdown cleanly and was last loaded using a build with a newer version of Berkeley DB. If so, please use the software that last loaded this wallet"); } @@ -261,7 +261,7 @@ bool BerkeleyDatabase::Verify(bilingual_str& errorStr) fs::path file_path = walletDir / strFile; LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); - LogPrintf("Using wallet %s\n", file_path.string()); + LogPrintf("Using wallet %s\n", fs::PathToString(file_path)); if (!env->Open(errorStr)) { return false; @@ -274,7 +274,7 @@ bool BerkeleyDatabase::Verify(bilingual_str& errorStr) Db db(env->dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); if (result != 0) { - errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), file_path); + errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), fs::quoted(fs::PathToString(file_path))); return false; } } @@ -375,7 +375,7 @@ void BerkeleyBatch::Flush() nMinutes = 1; if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault - env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetIntArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } } @@ -566,7 +566,7 @@ void BerkeleyEnvironment::Flush(bool fShutdown) dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); if (!fMockDb) { - fs::remove_all(fs::path(strPath) / "database"); + fs::remove_all(fs::PathFromString(strPath) / "database"); } } } @@ -614,21 +614,21 @@ bool BerkeleyDatabase::Backup(const std::string& strDest) const // Copy wallet file fs::path pathSrc = env->Directory() / strFile; - fs::path pathDest(strDest); + fs::path pathDest(fs::PathFromString(strDest)); if (fs::is_directory(pathDest)) - pathDest /= strFile; + pathDest /= fs::PathFromString(strFile); try { if (fs::equivalent(pathSrc, pathDest)) { - LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); + LogPrintf("cannot backup to wallet source file %s\n", fs::PathToString(pathDest)); return false; } fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strFile, pathDest.string()); + LogPrintf("copied %s to %s\n", strFile, fs::PathToString(pathDest)); return true; } catch (const fs::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); + LogPrintf("error copying %s to %s - %s\n", strFile, fs::PathToString(pathDest), fsbridge::get_filesystem_error_message(e)); return false; } } @@ -828,10 +828,10 @@ std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, con std::unique_ptr<BerkeleyDatabase> db; { LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor - std::string data_filename = data_file.filename().string(); + std::string data_filename = fs::PathToString(data_file.filename()); std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path()); if (env->m_databases.count(data_filename)) { - error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", (env->Directory() / data_filename).string())); + error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", fs::PathToString(env->Directory() / data_filename))); status = DatabaseStatus::FAILED_ALREADY_LOADED; return nullptr; } diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index a8209587d7..7d0f80518a 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -63,7 +63,7 @@ public: bool IsMock() const { return fMockDb; } bool IsInitialized() const { return fDbEnvInit; } - fs::path Directory() const { return strPath; } + fs::path Directory() const { return fs::PathFromString(strPath); } bool Open(bilingual_str& error); void Close(); @@ -113,7 +113,7 @@ public: */ bool Rewrite(const char* pszSkip=nullptr) override; - /** Indicate the a new database user has began using the database. */ + /** Indicate that a new database user has begun using the database. */ void AddRef() override; /** Indicate that database user has stopped using the database and that it could be flushed or closed. */ void RemoveRef() override; @@ -141,7 +141,7 @@ public: bool Verify(bilingual_str& error); /** Return path to main database filename */ - std::string Filename() override { return (env->Directory() / strFile).string(); } + std::string Filename() override { return fs::PathToString(env->Directory() / strFile); } std::string Format() override { return "bdb"; } /** diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 85cbec76b7..edd81e590f 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -9,9 +9,14 @@ #include <policy/feerate.h> #include <policy/fees.h> #include <primitives/transaction.h> +#include <script/keyorigin.h> +#include <script/signingprovider.h> #include <script/standard.h> #include <optional> +#include <algorithm> +#include <map> +#include <set> const int DEFAULT_MIN_DEPTH = 0; const int DEFAULT_MAX_DEPTH = 9999999; @@ -53,6 +58,8 @@ public: int m_min_depth = DEFAULT_MIN_DEPTH; //! Maximum chain depth value for coin availability int m_max_depth = DEFAULT_MAX_DEPTH; + //! SigningProvider that has pubkeys and scripts to do spend size estimation for external inputs + FlatSigningProvider m_external_provider; CCoinControl(); @@ -66,11 +73,32 @@ public: return (setSelected.count(output) > 0); } + bool IsExternalSelected(const COutPoint& output) const + { + return (m_external_txouts.count(output) > 0); + } + + bool GetExternalOutput(const COutPoint& outpoint, CTxOut& txout) const + { + const auto ext_it = m_external_txouts.find(outpoint); + if (ext_it == m_external_txouts.end()) { + return false; + } + txout = ext_it->second; + return true; + } + void Select(const COutPoint& output) { setSelected.insert(output); } + void SelectExternal(const COutPoint& outpoint, const CTxOut& txout) + { + setSelected.insert(outpoint); + m_external_txouts.emplace(outpoint, txout); + } + void UnSelect(const COutPoint& output) { setSelected.erase(output); @@ -88,6 +116,7 @@ public: private: std::set<COutPoint> setSelected; + std::map<COutPoint, CTxOut> m_external_txouts; }; #endif // BITCOIN_WALLET_COINCONTROL_H diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 1699424657..e1ca3fb379 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -4,10 +4,13 @@ #include <wallet/coinselection.h> +#include <consensus/amount.h> #include <policy/feerate.h> +#include <util/check.h> #include <util/system.h> #include <util/moneystr.h> +#include <numeric> #include <optional> // Descending order comparator @@ -168,6 +171,30 @@ bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& selectio return true; } +std::optional<std::pair<std::set<CInputCoin>, CAmount>> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value) +{ + std::set<CInputCoin> out_set; + CAmount value_ret = 0; + + std::vector<size_t> indexes; + indexes.resize(utxo_pool.size()); + std::iota(indexes.begin(), indexes.end(), 0); + Shuffle(indexes.begin(), indexes.end(), FastRandomContext()); + + CAmount selected_eff_value = 0; + for (const size_t i : indexes) { + const OutputGroup& group = utxo_pool.at(i); + Assume(group.GetSelectionAmount() > 0); + selected_eff_value += group.GetSelectionAmount(); + value_ret += group.m_value; + util::insert(out_set, group.m_outputs); + if (selected_eff_value >= target_value) { + return std::make_pair(out_set, value_ret); + } + } + return std::nullopt; +} + static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const CAmount& nTotalLower, const CAmount& nTargetValue, std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { @@ -279,13 +306,13 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group } if (LogAcceptCategory(BCLog::SELECTCOINS)) { - LogPrint(BCLog::SELECTCOINS, "SelectCoins() best subset: "); /* Continued */ + std::string log_message{"Coin selection best subset: "}; for (unsigned int i = 0; i < applicable_groups.size(); i++) { if (vfBest[i]) { - LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(applicable_groups[i].m_value)); /* Continued */ + log_message += strprintf("%s ", FormatMoney(applicable_groups[i].m_value)); } } - LogPrint(BCLog::SELECTCOINS, "total %s\n", FormatMoney(nBest)); + LogPrint(BCLog::SELECTCOINS, "%stotal %s\n", log_message, FormatMoney(nBest)); } } diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 35617d455b..e7d467660f 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -5,11 +5,13 @@ #ifndef BITCOIN_WALLET_COINSELECTION_H #define BITCOIN_WALLET_COINSELECTION_H -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> +#include <optional> + //! target minimum change amount static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees @@ -35,6 +37,18 @@ public: m_input_bytes = input_bytes; } + CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in) + { + outpoint = outpoint_in; + txout = txout_in; + effective_value = txout.nValue; + } + + CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in, int input_bytes) : CInputCoin(outpoint_in, txout_in) + { + m_input_bytes = input_bytes; + } + COutPoint outpoint; CTxOut txout; CAmount effective_value; @@ -174,7 +188,9 @@ struct OutputGroup * change_cost = effective_feerate * change_output_size + long_term_feerate * change_spend_size * * @param[in] inputs The selected inputs - * @param[in] change_cost The cost of creating change and spending it in the future. Only used if there is change. Must be 0 if there is no change. + * @param[in] change_cost The cost of creating change and spending it in the future. + * Only used if there is change, in which case it must be positive. + * Must be 0 if there is no change. * @param[in] target The amount targeted by the coin selection algorithm. * @param[in] use_effective_value Whether to use the input's effective value (when true) or the real value (when false). * @return The waste @@ -183,6 +199,15 @@ struct OutputGroup bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& selection_target, const CAmount& cost_of_change, std::set<CInputCoin>& out_set, CAmount& value_ret); +/** Select coins by Single Random Draw. OutputGroups are selected randomly from the eligible + * outputs until the target is satisfied + * + * @param[in] utxo_pool The positive effective value OutputGroups eligible for selection + * @param[in] target_value The target value to select for + * @returns If successful, a pair of set of outputs and total selected value, otherwise, std::nullopt + */ +std::optional<std::pair<std::set<CInputCoin>, CAmount>> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value); + // Original coin selection algorithm as a fallback bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& groups, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 8d5316e0af..305f6c31fc 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -12,7 +12,6 @@ std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) { - const size_t offset = wallet_dir.string().size() + (wallet_dir == wallet_dir.root_name() ? 0 : 1); std::vector<fs::path> paths; boost::system::error_code ec; @@ -20,17 +19,15 @@ std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) if (ec) { if (fs::is_directory(*it)) { it.no_push(); - LogPrintf("%s: %s %s -- skipping.\n", __func__, ec.message(), it->path().string()); + LogPrintf("%s: %s %s -- skipping.\n", __func__, ec.message(), fs::PathToString(it->path())); } else { - LogPrintf("%s: %s %s\n", __func__, ec.message(), it->path().string()); + LogPrintf("%s: %s %s\n", __func__, ec.message(), fs::PathToString(it->path())); } continue; } try { - // Get wallet path relative to walletdir by removing walletdir from the wallet path. - // This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60. - const fs::path path = it->path().string().substr(offset); + const fs::path path{it->path().lexically_relative(wallet_dir)}; if (it->status().type() == fs::directory_file && (IsBDBFile(BDBDataFile(it->path())) || IsSQLiteFile(SQLiteDataFile(it->path())))) { @@ -50,7 +47,7 @@ std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) } } } catch (const std::exception& e) { - LogPrintf("%s: Error scanning %s: %s\n", __func__, it->path().string(), e.what()); + LogPrintf("%s: Error scanning %s: %s\n", __func__, fs::PathToString(it->path()), e.what()); it.no_push(); } } @@ -85,7 +82,7 @@ bool IsBDBFile(const fs::path& path) // This check also prevents opening lock files. boost::system::error_code ec; auto size = fs::file_size(path, ec); - if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), fs::PathToString(path)); if (size < 4096) return false; fsbridge::ifstream file(path, std::ios::binary); @@ -109,7 +106,7 @@ bool IsSQLiteFile(const fs::path& path) // A SQLite Database file is at least 512 bytes. boost::system::error_code ec; auto size = fs::file_size(path, ec); - if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), fs::PathToString(path)); if (size < 512) return false; fsbridge::ifstream file(path, std::ios::binary); diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index c39c0c7e73..e50b4ca5f7 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -19,16 +19,16 @@ bool DumpWallet(CWallet& wallet, bilingual_str& error) return false; } - fs::path path = dump_filename; + fs::path path = fs::PathFromString(dump_filename); path = fs::absolute(path); if (fs::exists(path)) { - error = strprintf(_("File %s already exists. If you are sure this is what you want, move it out of the way first."), path.string()); + error = strprintf(_("File %s already exists. If you are sure this is what you want, move it out of the way first."), fs::PathToString(path)); return false; } fsbridge::ofstream dump_file; dump_file.open(path); if (dump_file.fail()) { - error = strprintf(_("Unable to open %s for writing"), path.string()); + error = strprintf(_("Unable to open %s for writing"), fs::PathToString(path)); return false; } @@ -114,10 +114,10 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling return false; } - fs::path dump_path = dump_filename; + fs::path dump_path = fs::PathFromString(dump_filename); dump_path = fs::absolute(dump_path); if (!fs::exists(dump_path)) { - error = strprintf(_("Dump file %s does not exist."), dump_path.string()); + error = strprintf(_("Dump file %s does not exist."), fs::PathToString(dump_path)); return false; } fsbridge::ifstream dump_file(dump_path); @@ -191,7 +191,7 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling // dummy chain interface bool ret = true; - std::shared_ptr<CWallet> wallet(new CWallet(nullptr /* chain */, name, std::move(database)), WalletToolReleaseWallet); + std::shared_ptr<CWallet> wallet(new CWallet(nullptr /* chain */, name, gArgs, std::move(database)), WalletToolReleaseWallet); { LOCK(wallet->cs_wallet); DBErrors load_wallet_ret = wallet->LoadWallet(); diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 434f211dc2..d6d625d9c1 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_WALLET_FEES_H #define BITCOIN_WALLET_FEES_H -#include <amount.h> +#include <consensus/amount.h> class CCoinControl; class CFeeRate; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 7abdbb0e55..7a5526a4cb 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -62,7 +62,6 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MINFEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-paytxfee=<amt>", strprintf("Fee rate (in %s/kvB) to add to transactions you send (default: %s)", CURRENCY_UNIT, FormatMoney(CFeeRate{DEFAULT_PAY_TX_FEE}.GetFeePerK())), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - argsman.AddArg("-rescan", "Rescan the block chain for missing wallet transactions on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); #ifdef ENABLE_EXTERNAL_SIGNER argsman.AddArg("-signer=<cmd>", "External signing tool, see doc/external-signer.md", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); #endif @@ -85,7 +84,7 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const #endif #ifdef USE_SQLITE - argsman.AddArg("-unsafesqlitesync", "Set SQLite synchronous=OFF to disable waiting for the database to sync to disk. This is unsafe and can cause data loss and corruption. This option is only used by tests to improve their performance (default: false)", ArgsManager::ALLOW_BOOL | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + argsman.AddArg("-unsafesqlitesync", "Set SQLite synchronous=OFF to disable waiting for the database to sync to disk. This is unsafe and can cause data loss and corruption. This option is only used by tests to improve their performance (default: false)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); #else argsman.AddHiddenArgs({"-unsafesqlitesync"}); #endif diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 9a8c1e3c02..57f1a6a67a 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -4,7 +4,7 @@ #include <interfaces/wallet.h> -#include <amount.h> +#include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <policy/fees.h> @@ -214,15 +214,17 @@ public: LOCK(m_wallet->cs_wallet); return m_wallet->DisplayAddress(dest); } - void lockCoin(const COutPoint& output) override + bool lockCoin(const COutPoint& output, const bool write_to_db) override { LOCK(m_wallet->cs_wallet); - return m_wallet->LockCoin(output); + std::unique_ptr<WalletBatch> batch = write_to_db ? std::make_unique<WalletBatch>(m_wallet->GetDatabase()) : nullptr; + return m_wallet->LockCoin(output, batch.get()); } - void unlockCoin(const COutPoint& output) override + bool unlockCoin(const COutPoint& output) override { LOCK(m_wallet->cs_wallet); - return m_wallet->UnlockCoin(output); + std::unique_ptr<WalletBatch> batch = std::make_unique<WalletBatch>(m_wallet->GetDatabase()); + return m_wallet->UnlockCoin(output, batch.get()); } bool isLockedCoin(const COutPoint& output) override { @@ -549,13 +551,13 @@ public: } std::string getWalletDir() override { - return GetWalletDir().string(); + return fs::PathToString(GetWalletDir()); } std::vector<std::string> listWalletDir() override { std::vector<std::string> paths; for (auto& path : ListDatabases(GetWalletDir())) { - paths.push_back(path.string()); + paths.push_back(fs::PathToString(path)); } return paths; } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 1b841026b8..7ef5a0cf55 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -25,25 +25,25 @@ bool VerifyWallets(WalletContext& context) ArgsManager& args = *Assert(context.args); if (args.IsArgSet("-walletdir")) { - fs::path wallet_dir = args.GetArg("-walletdir", ""); + fs::path wallet_dir = fs::PathFromString(args.GetArg("-walletdir", "")); boost::system::error_code error; // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); if (error || !fs::exists(wallet_dir)) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), fs::PathToString(wallet_dir))); return false; } else if (!fs::is_directory(wallet_dir)) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), fs::PathToString(wallet_dir))); return false; // The canonical path transforms relative paths into absolute ones, so we check the non-canonical version } else if (!wallet_dir.is_absolute()) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), fs::PathToString(wallet_dir))); return false; } - args.ForceSetArg("-walletdir", canonical_wallet_dir.string()); + args.ForceSetArg("-walletdir", fs::PathToString(canonical_wallet_dir)); } - LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); + LogPrintf("Using wallet directory %s\n", fs::PathToString(GetWalletDir())); chain.initMessage(_("Verifying wallet(s)…").translated); @@ -70,7 +70,7 @@ bool VerifyWallets(WalletContext& context) for (const auto& wallet : chain.getSettingsList("wallet")) { const auto& wallet_file = wallet.get_str(); - const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), wallet_file); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), fs::PathFromString(wallet_file)); if (!wallet_paths.insert(path).second) { chain.initWarning(strprintf(_("Ignoring duplicate -wallet %s."), wallet_file)); @@ -102,7 +102,7 @@ bool LoadWallets(WalletContext& context) std::set<fs::path> wallet_paths; for (const auto& wallet : chain.getSettingsList("wallet")) { const auto& name = wallet.get_str(); - if (!wallet_paths.insert(name).second) { + if (!wallet_paths.insert(fs::PathFromString(name)).second) { continue; } DatabaseOptions options; diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 98706dcdf8..2fb274b55f 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <consensus/consensus.h> #include <wallet/receive.h> #include <wallet/transaction.h> diff --git a/src/wallet/receive.h b/src/wallet/receive.h index b4b311636b..f659955fc6 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_RECEIVE_H #define BITCOIN_WALLET_RECEIVE_H -#include <amount.h> +#include <consensus/amount.h> #include <wallet/ismine.h> #include <wallet/transaction.h> #include <wallet/wallet.h> diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 72c60c8fe2..403c978680 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -57,7 +57,7 @@ static std::string DecodeDumpString(const std::string &str) { return ret.str(); } -static bool GetWalletAddressesForKey(LegacyScriptPubKeyMan* spk_man, const CWallet& wallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static bool GetWalletAddressesForKey(const LegacyScriptPubKeyMan* spk_man, const CWallet& wallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { bool fLabelFound = false; CKey key; @@ -550,7 +550,7 @@ RPCHelpMan importwallet() EnsureWalletIsUnlocked(*pwallet); fsbridge::ifstream file; - file.open(request.params[0].get_str(), std::ios::in | std::ios::ate); + file.open(fs::u8path(request.params[0].get_str()), std::ios::in | std::ios::ate); if (!file.is_open()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); } @@ -681,10 +681,10 @@ RPCHelpMan dumpprivkey() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; - LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); + const LegacyScriptPubKeyMan& spk_man = EnsureConstLegacyScriptPubKeyMan(*pwallet); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); @@ -731,11 +731,11 @@ RPCHelpMan dumpwallet() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; - CWallet& wallet = *pwallet; - LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(wallet); + const CWallet& wallet = *pwallet; + const LegacyScriptPubKeyMan& spk_man = EnsureConstLegacyScriptPubKeyMan(wallet); // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -745,7 +745,7 @@ RPCHelpMan dumpwallet() EnsureWalletIsUnlocked(wallet); - fs::path filepath = request.params[0].get_str(); + fs::path filepath = fs::u8path(request.params[0].get_str()); filepath = fs::absolute(filepath); /* Prevent arbitrary files from being overwritten. There have been reports @@ -754,7 +754,7 @@ RPCHelpMan dumpwallet() * It may also avoid other security issues. */ if (fs::exists(filepath)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, filepath.string() + " already exists. If you are sure this is what you want, move it out of the way first"); + throw JSONRPCError(RPC_INVALID_PARAMETER, filepath.u8string() + " already exists. If you are sure this is what you want, move it out of the way first"); } fsbridge::ofstream file; @@ -797,7 +797,7 @@ RPCHelpMan dumpwallet() CKey seed; if (spk_man.GetKey(seed_id, seed)) { CExtKey masterKey; - masterKey.SetSeed(seed.begin(), seed.size()); + masterKey.SetSeed(seed); file << "# extended private masterkey: " << EncodeExtKey(masterKey) << "\n\n"; } @@ -809,6 +809,9 @@ RPCHelpMan dumpwallet() std::string strLabel; CKey key; if (spk_man.GetKey(keyid, key)) { + CKeyMetadata metadata; + const auto it{spk_man.mapKeyMetadata.find(keyid)}; + if (it != spk_man.mapKeyMetadata.end()) metadata = it->second; file << strprintf("%s %s ", EncodeSecret(key), strTime); if (GetWalletAddressesForKey(&spk_man, wallet, keyid, strAddr, strLabel)) { file << strprintf("label=%s", strLabel); @@ -816,12 +819,12 @@ RPCHelpMan dumpwallet() file << "hdseed=1"; } else if (mapKeyPool.count(keyid)) { file << "reserve=1"; - } else if (spk_man.mapKeyMetadata[keyid].hdKeypath == "s") { + } else if (metadata.hdKeypath == "s") { file << "inactivehdseed=1"; } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (spk_man.mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(spk_man.mapKeyMetadata[keyid].key_origin.path) : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (metadata.has_key_origin ? " hdkeypath="+WriteHDKeypath(metadata.key_origin.path) : "")); } } file << "\n"; @@ -844,7 +847,7 @@ RPCHelpMan dumpwallet() file.close(); UniValue reply(UniValue::VOBJ); - reply.pushKV("filename", filepath.string()); + reply.pushKV("filename", filepath.u8string()); return reply; }, @@ -1439,7 +1442,7 @@ RPCHelpMan importmulti() "and coins using this key may not appear in the wallet. This error could be " "caused by pruning or data corruption (see bitcoind log for details) and could " "be dealt with by downloading and rescanning the relevant blocks (see -reindex " - "and -rescan options).", + "option and rescanblockchain RPC).", GetImportTimestamp(request, now), scannedTime - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW))); response.push_back(std::move(result)); } @@ -1488,7 +1491,7 @@ static UniValue ProcessDescriptorImport(CWallet& wallet, const UniValue& data, c } else { warnings.push_back("Range not given, using default keypool range"); range_start = 0; - range_end = gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE); + range_end = gArgs.GetIntArg("-keypool", DEFAULT_KEYPOOL_SIZE); } next_index = range_start; @@ -1744,7 +1747,7 @@ RPCHelpMan importdescriptors() "and coins using this desc may not appear in the wallet. This error could be " "caused by pruning or data corruption (see bitcoind log for details) and could " "be dealt with by downloading and rescanning the relevant blocks (see -reindex " - "and -rescan options).", + "option and rescanblockchain RPC).", GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW))); response.push_back(std::move(result)); } @@ -1788,7 +1791,7 @@ RPCHelpMan listdescriptors() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e922f4ede9..babb61b03a 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <core_io.h> #include <interfaces/chain.h> #include <key_io.h> @@ -43,6 +43,7 @@ #include <univalue.h> +#include <map> using interfaces::FoundBlock; @@ -102,7 +103,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques std::string wallet_name; if (GetWalletNameFromJSONRPCRequest(request, wallet_name)) { - std::shared_ptr<CWallet> pwallet = GetWallet(context, wallet_name); + const std::shared_ptr<CWallet> pwallet = GetWallet(context, wallet_name); if (!pwallet) throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Requested wallet does not exist or is not loaded"); return pwallet; } @@ -149,6 +150,15 @@ LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_cr return *spk_man; } +const LegacyScriptPubKeyMan& EnsureConstLegacyScriptPubKeyMan(const CWallet& wallet) +{ + const LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (!spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); + } + return *spk_man; +} + static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue& entry) { interfaces::Chain& chain = wallet.chain(); @@ -472,7 +482,7 @@ static RPCHelpMan sendtoaddress() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, - {RPCResult::Type::STR, "fee reason", "The transaction fee reason."} + {RPCResult::Type::STR, "fee_reason", "The transaction fee reason."} }, }, }, @@ -569,7 +579,7 @@ static RPCHelpMan listaddressgroupings() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -626,7 +636,7 @@ static RPCHelpMan signmessage() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; LOCK(pwallet->cs_wallet); @@ -728,7 +738,7 @@ static RPCHelpMan getreceivedbyaddress() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -766,7 +776,7 @@ static RPCHelpMan getreceivedbylabel() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -806,7 +816,7 @@ static RPCHelpMan getbalance() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -845,7 +855,7 @@ static RPCHelpMan getunconfirmedbalance() RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -899,7 +909,7 @@ static RPCHelpMan sendmany() { {RPCResult::Type::STR_HEX, "txid", "The transaction id for the send. Only 1 transaction is created regardless of\n" "the number of addresses."}, - {RPCResult::Type::STR, "fee reason", "The transaction fee reason."} + {RPCResult::Type::STR, "fee_reason", "The transaction fee reason."} }, }, }, @@ -1213,7 +1223,7 @@ static RPCHelpMan listreceivedbyaddress() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction"}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction"}, {RPCResult::Type::STR, "address", "The receiving address"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " received by the address"}, {RPCResult::Type::NUM, "confirmations", "The number of confirmations of the most recent transaction included"}, @@ -1233,7 +1243,7 @@ static RPCHelpMan listreceivedbyaddress() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -1261,7 +1271,7 @@ static RPCHelpMan listreceivedbylabel() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction"}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount received by addresses with this label"}, {RPCResult::Type::NUM, "confirmations", "The number of confirmations of the most recent transaction included"}, {RPCResult::Type::STR, "label", "The label of the receiving address. The default label is \"\""}, @@ -1275,7 +1285,7 @@ static RPCHelpMan listreceivedbylabel() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -1388,22 +1398,27 @@ static const std::vector<RPCResult> TransactionDescriptionString() { return{{RPCResult::Type::NUM, "confirmations", "The number of confirmations for the transaction. Negative confirmations means the\n" "transaction conflicted that many blocks ago."}, - {RPCResult::Type::BOOL, "generated", "Only present if transaction only input is a coinbase one."}, - {RPCResult::Type::BOOL, "trusted", "Only present if we consider transaction to be trusted and so safe to spend from."}, - {RPCResult::Type::STR_HEX, "blockhash", "The block hash containing the transaction."}, - {RPCResult::Type::NUM, "blockheight", "The block height containing the transaction."}, - {RPCResult::Type::NUM, "blockindex", "The index of the transaction in the block that includes it."}, - {RPCResult::Type::NUM_TIME, "blocktime", "The block time expressed in " + UNIX_EPOCH_TIME + "."}, + {RPCResult::Type::BOOL, "generated", /* optional */ true, "Only present if the transaction's only input is a coinbase one."}, + {RPCResult::Type::BOOL, "trusted", /* optional */ true, "Whether we consider the transaction to be trusted and safe to spend from.\n" + "Only present when the transaction has 0 confirmations (or negative confirmations, if conflicted)."}, + {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "The block hash containing the transaction."}, + {RPCResult::Type::NUM, "blockheight", /* optional */ true, "The block height containing the transaction."}, + {RPCResult::Type::NUM, "blockindex", /* optional */ true, "The index of the transaction in the block that includes it."}, + {RPCResult::Type::NUM_TIME, "blocktime", /* optional */ true, "The block time expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, {RPCResult::Type::ARR, "walletconflicts", "Conflicting transaction ids.", { {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, }}, + {RPCResult::Type::STR_HEX, "replaced_by_txid", /* optional */ true, "The txid if this tx was replaced."}, + {RPCResult::Type::STR_HEX, "replaces_txid", /* optional */ true, "The txid if the tx replaces one."}, + {RPCResult::Type::STR, "comment", /* optional */ true, ""}, + {RPCResult::Type::STR, "to", /* optional */ true, "If a comment to is associated with the transaction."}, {RPCResult::Type::NUM_TIME, "time", "The transaction time expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::NUM_TIME, "timereceived", "The time received expressed in " + UNIX_EPOCH_TIME + "."}, - {RPCResult::Type::STR, "comment", "If a comment is associated with the transaction, only present if not empty."}, + {RPCResult::Type::STR, "comment", /* optional */ true, "If a comment is associated with the transaction, only present if not empty."}, {RPCResult::Type::STR, "bip125-replaceable", "(\"yes|no|unknown\") Whether this transaction could be replaced due to BIP125 (replace-by-fee);\n" - "may be unknown for unconfirmed transactions not in the mempool"}}; + "may be unknown for unconfirmed transactions not in the mempool."}}; } static RPCHelpMan listtransactions() @@ -1423,7 +1438,7 @@ static RPCHelpMan listtransactions() { {RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction."}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction."}, {RPCResult::Type::STR, "address", "The bitcoin address of the transaction."}, {RPCResult::Type::STR, "category", "The transaction category.\n" "\"send\" Transactions sent.\n" @@ -1433,14 +1448,14 @@ static RPCHelpMan listtransactions() "\"orphan\" Orphaned coinbase transactions received."}, {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and is positive\n" "for all other categories"}, - {RPCResult::Type::STR, "label", "A comment for the address/transaction, if any"}, + {RPCResult::Type::STR, "label", /* optional */ true, "A comment for the address/transaction, if any"}, {RPCResult::Type::NUM, "vout", "the vout value"}, - {RPCResult::Type::STR_AMOUNT, "fee", "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" + {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" "'send' category of transactions."}, }, TransactionDescriptionString()), { - {RPCResult::Type::BOOL, "abandoned", "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" + {RPCResult::Type::BOOL, "abandoned", /* optional */ true, "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" "'send' category of transactions."}, })}, } @@ -1455,7 +1470,7 @@ static RPCHelpMan listtransactions() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -1537,7 +1552,7 @@ static RPCHelpMan listsinceblock() { {RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction."}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction."}, {RPCResult::Type::STR, "address", "The bitcoin address of the transaction."}, {RPCResult::Type::STR, "category", "The transaction category.\n" "\"send\" Transactions sent.\n" @@ -1548,18 +1563,17 @@ static RPCHelpMan listsinceblock() {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and is positive\n" "for all other categories"}, {RPCResult::Type::NUM, "vout", "the vout value"}, - {RPCResult::Type::STR_AMOUNT, "fee", "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" + {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" "'send' category of transactions."}, }, TransactionDescriptionString()), { - {RPCResult::Type::BOOL, "abandoned", "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" + {RPCResult::Type::BOOL, "abandoned", /* optional */ true, "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" "'send' category of transactions."}, - {RPCResult::Type::STR, "label", "A comment for the address/transaction, if any"}, - {RPCResult::Type::STR, "to", "If a comment to is associated with the transaction."}, + {RPCResult::Type::STR, "label", /* optional */ true, "A comment for the address/transaction, if any"}, })}, }}, - {RPCResult::Type::ARR, "removed", "<structure is the same as \"transactions\" above, only present if include_removed=true>\n" + {RPCResult::Type::ARR, "removed", /* optional */ true, "<structure is the same as \"transactions\" above, only present if include_removed=true>\n" "Note: transactions that were re-added in the active chain will appear as-is in this array, and may thus have a positive confirmation count." , {{RPCResult::Type::ELISION, "", ""},}}, {RPCResult::Type::STR_HEX, "lastblock", "The hash of the block (target_confirmations-1) from the best block on the main chain, or the genesis hash if the referenced block does not exist yet. This is typically used to feed back into listsinceblock the next time you call it. So you would generally use a target_confirmations of say 6, so you will be continually re-notified of transactions until they've reached 6 confirmations plus any new ones"}, @@ -1572,7 +1586,7 @@ static RPCHelpMan listsinceblock() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; const CWallet& wallet = *pwallet; @@ -1672,7 +1686,7 @@ static RPCHelpMan gettransaction() RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT}, - {RPCResult::Type::STR_AMOUNT, "fee", "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" + {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" "'send' category of transactions."}, }, TransactionDescriptionString()), @@ -1681,8 +1695,8 @@ static RPCHelpMan gettransaction() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction."}, - {RPCResult::Type::STR, "address", "The bitcoin address involved in the transaction."}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction."}, + {RPCResult::Type::STR, "address", /* optional */ true, "The bitcoin address involved in the transaction."}, {RPCResult::Type::STR, "category", "The transaction category.\n" "\"send\" Transactions sent.\n" "\"receive\" Non-coinbase transactions received.\n" @@ -1690,16 +1704,16 @@ static RPCHelpMan gettransaction() "\"immature\" Coinbase transactions received with 100 or fewer confirmations.\n" "\"orphan\" Orphaned coinbase transactions received."}, {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT}, - {RPCResult::Type::STR, "label", "A comment for the address/transaction, if any"}, + {RPCResult::Type::STR, "label", /* optional */ true, "A comment for the address/transaction, if any"}, {RPCResult::Type::NUM, "vout", "the vout value"}, - {RPCResult::Type::STR_AMOUNT, "fee", "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the \n" + {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the \n" "'send' category of transactions."}, - {RPCResult::Type::BOOL, "abandoned", "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" + {RPCResult::Type::BOOL, "abandoned", /* optional */ true, "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" "'send' category of transactions."}, }}, }}, {RPCResult::Type::STR_HEX, "hex", "Raw data for transaction"}, - {RPCResult::Type::OBJ, "decoded", "Optional, the decoded transaction (only present when `verbose` is passed)", + {RPCResult::Type::OBJ, "decoded", /* optional */ true, "The decoded transaction (only present when `verbose` is passed)", { {RPCResult::Type::ELISION, "", "Equivalent to the RPC decoderawtransaction method, or the RPC getrawtransaction method when `verbose` is passed."}, }}, @@ -1713,7 +1727,7 @@ static RPCHelpMan gettransaction() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -1759,7 +1773,7 @@ static RPCHelpMan gettransaction() if (verbose) { UniValue decoded(UniValue::VOBJ); - TxToUniv(*wtx.tx, uint256(), pwallet->chain().rpcEnableDeprecated("addresses"), decoded, false); + TxToUniv(*wtx.tx, uint256(), decoded, false); entry.pushKV("decoded", decoded); } @@ -1824,7 +1838,7 @@ static RPCHelpMan backupwallet() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -1850,7 +1864,7 @@ static RPCHelpMan keypoolrefill() "\nFills the keypool."+ HELP_REQUIRING_PASSPHRASE, { - {"newsize", RPCArg::Type::NUM, RPCArg::Default{100}, "The new keypool size"}, + {"newsize", RPCArg::Type::NUM, RPCArg::DefaultHint{strprintf("%u, or as set by -keypool", DEFAULT_KEYPOOL_SIZE)}, "The new keypool size"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -1889,6 +1903,33 @@ static RPCHelpMan keypoolrefill() } +static RPCHelpMan newkeypool() +{ + return RPCHelpMan{"newkeypool", + "\nEntirely clears and refills the keypool."+ + HELP_REQUIRING_PASSPHRASE, + {}, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + HelpExampleCli("newkeypool", "") + + HelpExampleRpc("newkeypool", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return NullUniValue; + + LOCK(pwallet->cs_wallet); + + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); + spk_man.NewKeyPool(); + + return NullUniValue; +}, + }; +} + + static RPCHelpMan walletpassphrase() { return RPCHelpMan{"walletpassphrase", @@ -2137,8 +2178,9 @@ static RPCHelpMan lockunspent() "If no transaction outputs are specified when unlocking then all current locked transaction outputs are unlocked.\n" "A locked transaction output will not be chosen by automatic coin selection, when spending bitcoins.\n" "Manually selected coins are automatically unlocked.\n" - "Locks are stored in memory only. Nodes start with zero locked outputs, and the locked output list\n" - "is always cleared (by virtue of process exit) when a node stops or fails.\n" + "Locks are stored in memory only, unless persistent=true, in which case they will be written to the\n" + "wallet database and loaded on node start. Unwritten (persistent=false) locks are always cleared\n" + "(by virtue of process exit) when a node stops or fails. Unlocking will clear both persistent and not.\n" "Also see the listunspent call\n", { {"unlock", RPCArg::Type::BOOL, RPCArg::Optional::NO, "Whether to unlock (true) or lock (false) the specified transactions"}, @@ -2152,6 +2194,7 @@ static RPCHelpMan lockunspent() }, }, }, + {"persistent", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether to write/erase this lock in the wallet database, or keep the change in memory only. Ignored for unlocking."}, }, RPCResult{ RPCResult::Type::BOOL, "", "Whether the command was successful or not" @@ -2165,6 +2208,8 @@ static RPCHelpMan lockunspent() + HelpExampleCli("listlockunspent", "") + "\nUnlock the transaction again\n" + HelpExampleCli("lockunspent", "true \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\"") + + "\nLock the transaction persistently in the wallet database\n" + + HelpExampleCli("lockunspent", "false \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\" true") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("lockunspent", "false, \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\"") }, @@ -2183,9 +2228,13 @@ static RPCHelpMan lockunspent() bool fUnlock = request.params[0].get_bool(); + const bool persistent{request.params[2].isNull() ? false : request.params[2].get_bool()}; + if (request.params[1].isNull()) { - if (fUnlock) - pwallet->UnlockAllCoins(); + if (fUnlock) { + if (!pwallet->UnlockAllCoins()) + throw JSONRPCError(RPC_WALLET_ERROR, "Unlocking coins failed"); + } return true; } @@ -2236,17 +2285,24 @@ static RPCHelpMan lockunspent() throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected locked output"); } - if (!fUnlock && is_locked) { + if (!fUnlock && is_locked && !persistent) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, output already locked"); } outputs.push_back(outpt); } + std::unique_ptr<WalletBatch> batch = nullptr; + // Unlock is always persistent + if (fUnlock || persistent) batch = std::make_unique<WalletBatch>(pwallet->GetDatabase()); + // Atomically set (un)locked status for the outputs. for (const COutPoint& outpt : outputs) { - if (fUnlock) pwallet->UnlockCoin(outpt); - else pwallet->LockCoin(outpt); + if (fUnlock) { + if (!pwallet->UnlockCoin(outpt, batch.get())) throw JSONRPCError(RPC_WALLET_ERROR, "Unlocking coin failed"); + } else { + if (!pwallet->LockCoin(outpt, batch.get())) throw JSONRPCError(RPC_WALLET_ERROR, "Locking coin failed"); + } } return true; @@ -2284,7 +2340,7 @@ static RPCHelpMan listlockunspent() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; LOCK(pwallet->cs_wallet); @@ -2362,9 +2418,9 @@ static RPCHelpMan getbalances() {RPCResult::Type::STR_AMOUNT, "trusted", "trusted balance (outputs created by the wallet or confirmed outputs)"}, {RPCResult::Type::STR_AMOUNT, "untrusted_pending", "untrusted pending balance (outputs created by others that are in the mempool)"}, {RPCResult::Type::STR_AMOUNT, "immature", "balance from immature coinbase outputs"}, - {RPCResult::Type::STR_AMOUNT, "used", "(only present if avoid_reuse is set) balance from coins sent to addresses that were previously spent from (potentially privacy violating)"}, + {RPCResult::Type::STR_AMOUNT, "used", /* optional */ true, "(only present if avoid_reuse is set) balance from coins sent to addresses that were previously spent from (potentially privacy violating)"}, }}, - {RPCResult::Type::OBJ, "watchonly", "watchonly balances (not present if wallet does not watch anything)", + {RPCResult::Type::OBJ, "watchonly", /* optional */ true, "watchonly balances (not present if wallet does not watch anything)", { {RPCResult::Type::STR_AMOUNT, "trusted", "trusted balance (outputs created by the wallet or confirmed outputs)"}, {RPCResult::Type::STR_AMOUNT, "untrusted_pending", "untrusted pending balance (outputs created by others that are in the mempool)"}, @@ -2377,9 +2433,9 @@ static RPCHelpMan getbalances() HelpExampleRpc("getbalances", "")}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const rpc_wallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> rpc_wallet = GetWalletForJSONRPCRequest(request); if (!rpc_wallet) return NullUniValue; - CWallet& wallet = *rpc_wallet; + const CWallet& wallet = *rpc_wallet; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -2431,9 +2487,9 @@ static RPCHelpMan getwalletinfo() {RPCResult::Type::STR_AMOUNT, "unconfirmed_balance", "DEPRECATED. Identical to getbalances().mine.untrusted_pending"}, {RPCResult::Type::STR_AMOUNT, "immature_balance", "DEPRECATED. Identical to getbalances().mine.immature"}, {RPCResult::Type::NUM, "txcount", "the total number of transactions in the wallet"}, - {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool. Legacy wallets only."}, + {RPCResult::Type::NUM_TIME, "keypoololdest", /* optional */ true, "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool. Legacy wallets only."}, {RPCResult::Type::NUM, "keypoolsize", "how many new keys are pre-generated (only counts external keys)"}, - {RPCResult::Type::NUM, "keypoolsize_hd_internal", "how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)"}, + {RPCResult::Type::NUM, "keypoolsize_hd_internal", /* optional */ true, "how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)"}, {RPCResult::Type::NUM_TIME, "unlocked_until", /* optional */ true, "the " + UNIX_EPOCH_TIME + " until which the wallet is unlocked for transfers, or 0 if the wallet is locked (only present for passphrase-encrypted wallets)"}, {RPCResult::Type::STR_AMOUNT, "paytxfee", "the transaction fee configuration, set in " + CURRENCY_UNIT + "/kvB"}, {RPCResult::Type::STR_HEX, "hdseedid", /* optional */ true, "the Hash160 of the HD seed (only present when HD is enabled)"}, @@ -2453,7 +2509,7 @@ static RPCHelpMan getwalletinfo() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; // Make sure the results are valid at least up to the most recent block @@ -2536,7 +2592,7 @@ static RPCHelpMan listwalletdir() UniValue wallets(UniValue::VARR); for (const auto& path : ListDatabases(GetWalletDir())) { UniValue wallet(UniValue::VOBJ); - wallet.pushKV("name", path.string()); + wallet.pushKV("name", path.u8string()); wallets.push_back(wallet); } @@ -2614,7 +2670,7 @@ static RPCHelpMan loadwallet() return RPCHelpMan{"loadwallet", "\nLoads a wallet from a wallet file or directory." "\nNote that all wallet command-line options used when starting bitcoind will be" - "\napplied to the new wallet (eg -rescan, etc).\n", + "\napplied to the new wallet.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, @@ -2724,7 +2780,7 @@ static RPCHelpMan createwallet() {"blank", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, RPCArg::Default{false}, "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, - {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, + {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{true}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, {"external_signer", RPCArg::Type::BOOL, RPCArg::Default{false}, "Use an external signer such as a hardware wallet. Requires -signer to be configured. Wallet creation will fail if keys cannot be fetched. Requires disable_private_keys and descriptors set to true."}, }, @@ -2766,12 +2822,11 @@ static RPCHelpMan createwallet() if (!request.params[4].isNull() && request.params[4].get_bool()) { flags |= WALLET_FLAG_AVOID_REUSE; } - if (!request.params[5].isNull() && request.params[5].get_bool()) { + if (request.params[5].isNull() || request.params[5].get_bool()) { #ifndef USE_SQLITE throw JSONRPCError(RPC_WALLET_ERROR, "Compiled without sqlite support (required for descriptor wallets)"); #endif flags |= WALLET_FLAG_DESCRIPTORS; - warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet")); } if (!request.params[7].isNull() && request.params[7].get_bool()) { #ifdef ENABLE_EXTERNAL_SIGNER @@ -2794,7 +2849,7 @@ static RPCHelpMan createwallet() options.create_passphrase = passphrase; bilingual_str error; std::optional<bool> load_on_start = request.params[6].isNull() ? std::nullopt : std::optional<bool>(request.params[6].get_bool()); - std::shared_ptr<CWallet> wallet = CreateWallet(context, request.params[0].get_str(), load_on_start, options, status, error, warnings); + const std::shared_ptr<CWallet> wallet = CreateWallet(context, request.params[0].get_str(), load_on_start, options, status, error, warnings); if (!wallet) { RPCErrorCode code = status == DatabaseStatus::FAILED_ENCRYPT ? RPC_WALLET_ENCRYPTION_FAILED : RPC_WALLET_ERROR; throw JSONRPCError(code, error.original); @@ -2837,7 +2892,7 @@ static RPCHelpMan restorewallet() WalletContext& context = EnsureWalletContext(request.context); - std::string backup_file = request.params[1].get_str(); + auto backup_file = fs::u8path(request.params[1].get_str()); if (!fs::exists(backup_file)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Backup file does not exist"); @@ -2845,14 +2900,14 @@ static RPCHelpMan restorewallet() std::string wallet_name = request.params[0].get_str(); - const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), wallet_name); + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), fs::u8path(wallet_name)); if (fs::exists(wallet_path)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Wallet name already exists."); } if (!TryCreateDirectories(wallet_path)) { - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Failed to create database path '%s'. Database already exists.", wallet_path.string())); + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Failed to create database path '%s'. Database already exists.", wallet_path.u8string())); } auto wallet_file = wallet_path / "wallet.dat"; @@ -2955,17 +3010,20 @@ static RPCHelpMan listunspent() { {RPCResult::Type::STR_HEX, "txid", "the transaction id"}, {RPCResult::Type::NUM, "vout", "the vout value"}, - {RPCResult::Type::STR, "address", "the bitcoin address"}, - {RPCResult::Type::STR, "label", "The associated label, or \"\" for the default label"}, + {RPCResult::Type::STR, "address", /* optional */ true, "the bitcoin address"}, + {RPCResult::Type::STR, "label", /* optional */ true, "The associated label, or \"\" for the default label"}, {RPCResult::Type::STR, "scriptPubKey", "the script key"}, {RPCResult::Type::STR_AMOUNT, "amount", "the transaction output amount in " + CURRENCY_UNIT}, {RPCResult::Type::NUM, "confirmations", "The number of confirmations"}, - {RPCResult::Type::STR_HEX, "redeemScript", "The redeemScript if scriptPubKey is P2SH"}, - {RPCResult::Type::STR, "witnessScript", "witnessScript if the scriptPubKey is P2WSH or P2SH-P2WSH"}, + {RPCResult::Type::NUM, "ancestorcount", /* optional */ true, "The number of in-mempool ancestor transactions, including this one (if transaction is in the mempool)"}, + {RPCResult::Type::NUM, "ancestorsize", /* optional */ true, "The virtual transaction size of in-mempool ancestors, including this one (if transaction is in the mempool)"}, + {RPCResult::Type::STR_AMOUNT, "ancestorfees", /* optional */ true, "The total fees of in-mempool ancestors (including this one) with fee deltas used for mining priority in " + CURRENCY_ATOM + " (if transaction is in the mempool)"}, + {RPCResult::Type::STR_HEX, "redeemScript", /* optional */ true, "The redeemScript if scriptPubKey is P2SH"}, + {RPCResult::Type::STR, "witnessScript", /* optional */ true, "witnessScript if the scriptPubKey is P2WSH or P2SH-P2WSH"}, {RPCResult::Type::BOOL, "spendable", "Whether we have the private keys to spend this output"}, {RPCResult::Type::BOOL, "solvable", "Whether we know how to spend this output, ignoring the lack of keys"}, - {RPCResult::Type::BOOL, "reused", "(only present if avoid_reuse is set) Whether this output is reused/dirty (sent to an address that was previously spent from)"}, - {RPCResult::Type::STR, "desc", "(only when solvable) A descriptor for spending this output"}, + {RPCResult::Type::BOOL, "reused", /* optional */ true, "(only present if avoid_reuse is set) Whether this output is reused/dirty (sent to an address that was previously spent from)"}, + {RPCResult::Type::STR, "desc", /* optional */ true, "(only when solvable) A descriptor for spending this output"}, {RPCResult::Type::BOOL, "safe", "Whether this output is considered safe to spend. Unconfirmed transactions\n" "from outside keys and unconfirmed replacement transactions are considered unsafe\n" "and are not eligible for spending by fundrawtransaction and sendtoaddress."}, @@ -2981,7 +3039,7 @@ static RPCHelpMan listunspent() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; int nMinDepth = 1; @@ -3126,6 +3184,16 @@ static RPCHelpMan listunspent() entry.pushKV("scriptPubKey", HexStr(scriptPubKey)); entry.pushKV("amount", ValueFromAmount(out.tx->tx->vout[out.i].nValue)); entry.pushKV("confirmations", out.nDepth); + if (!out.nDepth) { + size_t ancestor_count, descendant_count, ancestor_size; + CAmount ancestor_fees; + pwallet->chain().getTransactionAncestry(out.tx->GetHash(), ancestor_count, descendant_count, &ancestor_size, &ancestor_fees); + if (ancestor_count) { + entry.pushKV("ancestorcount", uint64_t(ancestor_count)); + entry.pushKV("ancestorsize", uint64_t(ancestor_size)); + entry.pushKV("ancestorfees", uint64_t(ancestor_fees)); + } + } entry.pushKV("spendable", out.fSpendable); entry.pushKV("solvable", out.fSolvable); if (out.fSolvable) { @@ -3145,6 +3213,34 @@ static RPCHelpMan listunspent() }; } +// Only includes key documentation where the key is snake_case in all RPC methods. MixedCase keys can be added later. +static std::vector<RPCArg> FundTxDoc() +{ + return { + {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, + {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, + {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" + "Allows this transaction to be replaced by a transaction with higher fees"}, + {"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" + "Used for fee estimation during coin selection.", + { + {"pubkeys", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Public keys involved in this transaction.", + { + {"pubkey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A public key"}, + }}, + {"scripts", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Scripts involved in this transaction.", + { + {"script", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A script"}, + }}, + {"descriptors", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Descriptors that provide solving data for this transaction.", + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A descriptor"}, + }}, + }}, + }; +} + void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, int& change_position, const UniValue& options, CCoinControl& coinControl, bool override_min_fee) { // Make sure the results are valid at least up to the most recent block @@ -3182,6 +3278,7 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, {"fee_rate", UniValueType()}, // will be checked by AmountFromValue() in SetFeeEstimateMode() {"feeRate", UniValueType()}, // will be checked by AmountFromValue() below {"psbt", UniValueType(UniValue::VBOOL)}, + {"solving_data", UniValueType(UniValue::VOBJ)}, {"subtractFeeFromOutputs", UniValueType(UniValue::VARR)}, {"subtract_fee_from_outputs", UniValueType(UniValue::VARR)}, {"replaceable", UniValueType(UniValue::VBOOL)}, @@ -3258,6 +3355,54 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, coinControl.fAllowWatchOnly = ParseIncludeWatchonly(NullUniValue, wallet); } + if (options.exists("solving_data")) { + const UniValue solving_data = options["solving_data"].get_obj(); + if (solving_data.exists("pubkeys")) { + for (const UniValue& pk_univ : solving_data["pubkeys"].get_array().getValues()) { + const std::string& pk_str = pk_univ.get_str(); + if (!IsHex(pk_str)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not hex", pk_str)); + } + const std::vector<unsigned char> data(ParseHex(pk_str)); + const CPubKey pubkey(data.begin(), data.end()); + if (!pubkey.IsFullyValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not a valid public key", pk_str)); + } + coinControl.m_external_provider.pubkeys.emplace(pubkey.GetID(), pubkey); + // Add witness script for pubkeys + const CScript wit_script = GetScriptForDestination(WitnessV0KeyHash(pubkey)); + coinControl.m_external_provider.scripts.emplace(CScriptID(wit_script), wit_script); + } + } + + if (solving_data.exists("scripts")) { + for (const UniValue& script_univ : solving_data["scripts"].get_array().getValues()) { + const std::string& script_str = script_univ.get_str(); + if (!IsHex(script_str)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not hex", script_str)); + } + std::vector<unsigned char> script_data(ParseHex(script_str)); + const CScript script(script_data.begin(), script_data.end()); + coinControl.m_external_provider.scripts.emplace(CScriptID(script), script); + } + } + + if (solving_data.exists("descriptors")) { + for (const UniValue& desc_univ : solving_data["descriptors"].get_array().getValues()) { + const std::string& desc_str = desc_univ.get_str(); + FlatSigningProvider desc_out; + std::string error; + std::vector<CScript> scripts_temp; + std::unique_ptr<Descriptor> desc = Parse(desc_str, desc_out, error, true); + if (!desc) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Unable to parse descriptor '%s': %s", desc_str, error)); + } + desc->Expand(0, desc_out, scripts_temp, desc_out); + coinControl.m_external_provider = Merge(coinControl.m_external_provider, desc_out); + } + } + } + if (tx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); @@ -3275,6 +3420,19 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, setSubtractFeeFromOutputs.insert(pos); } + // Fetch specified UTXOs from the UTXO set to get the scriptPubKeys and values of the outputs being selected + // and to match with the given solving_data. Only used for non-wallet outputs. + std::map<COutPoint, Coin> coins; + for (const CTxIn& txin : tx.vin) { + coins[txin.prevout]; // Create empty map entry keyed by prevout. + } + wallet.chain().findCoins(coins); + for (const auto& coin : coins) { + if (!coin.second.out.IsNull()) { + coinControl.SelectExternal(coin.first, coin.second.out); + } + } + bilingual_str error; if (!FundTransaction(wallet, tx, fee_out, change_position, error, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { @@ -3290,8 +3448,9 @@ static RPCHelpMan fundrawtransaction() "No existing outputs will be modified unless \"subtractFeeFromOutputs\" is specified.\n" "Note that inputs which were signed may need to be resigned after completion since in/outputs have been added.\n" "The inputs added will not be signed, use signrawtransactionwithkey\n" - " or signrawtransactionwithwallet for that.\n" - "Note that all existing inputs must have their previous output transaction be in the wallet.\n" + "or signrawtransactionwithwallet for that.\n" + "All existing inputs must either have their previous output transaction be in the wallet\n" + "or be in the UTXO set. Solving data must be provided for non-wallet inputs.\n" "Note that all inputs selected must be of standard form and P2SH scripts must be\n" "in the wallet using importaddress or addmultisigaddress (to calculate fees).\n" "You can see whether this is the case by checking the \"solvable\" field in the listunspent output.\n" @@ -3299,6 +3458,7 @@ static RPCHelpMan fundrawtransaction() { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{true}, "For a transaction with existing inputs, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -3321,12 +3481,8 @@ static RPCHelpMan fundrawtransaction() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, }, + FundTxDoc()), "options"}, {"iswitness", RPCArg::Type::BOOL, RPCArg::DefaultHint{"depends on heuristic tests"}, "Whether the transaction hex is a serialized witness transaction.\n" "If iswitness is not present, heuristic tests will be used in decoding.\n" @@ -3429,6 +3585,10 @@ RPCHelpMan signrawtransactionwithwallet() { {RPCResult::Type::STR_HEX, "txid", "The hash of the referenced, previous transaction"}, {RPCResult::Type::NUM, "vout", "The index of the output to spent and used as input"}, + {RPCResult::Type::ARR, "witness", "", + { + {RPCResult::Type::STR_HEX, "witness", ""}, + }}, {RPCResult::Type::STR_HEX, "scriptSig", "The hex-encoded signature script"}, {RPCResult::Type::NUM, "sequence", "Script sequence number"}, {RPCResult::Type::STR, "error", "Verification or signing error related to the input"}, @@ -3442,7 +3602,7 @@ RPCHelpMan signrawtransactionwithwallet() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR, UniValue::VSTR}, true); @@ -3764,7 +3924,6 @@ public: obj.pushKV("embedded", std::move(subobj)); } else if (which_type == TxoutType::MULTISIG) { // Also report some information on multisig scripts (which do not have a corresponding address). - // TODO: abstract out the common functionality between this logic and ExtractDestinations. obj.pushKV("sigsrequired", solutions_data[0][0]); UniValue pubkeys(UniValue::VARR); for (size_t i = 1; i < solutions_data.size() - 1; ++i) { @@ -3908,7 +4067,7 @@ RPCHelpMan getaddressinfo() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; LOCK(pwallet->cs_wallet); @@ -4015,7 +4174,7 @@ static RPCHelpMan getaddressesbylabel() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; LOCK(pwallet->cs_wallet); @@ -4076,7 +4235,7 @@ static RPCHelpMan listlabels() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; LOCK(pwallet->cs_wallet); @@ -4132,6 +4291,7 @@ static RPCHelpMan send() " \"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{false}, "If inputs are specified, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -4141,9 +4301,6 @@ static RPCHelpMan send() {"change_address", RPCArg::Type::STR_HEX, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" @@ -4166,18 +4323,17 @@ static RPCHelpMan send() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, }, + FundTxDoc()), "options"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", { {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, - {RPCResult::Type::STR_HEX, "txid", "The transaction id for the send. Only 1 transaction is created regardless of the number of addresses."}, - {RPCResult::Type::STR_HEX, "hex", "If add_to_wallet is false, the hex-encoded raw transaction with signature(s)"}, - {RPCResult::Type::STR, "psbt", "If more signatures are needed, or if add_to_wallet is false, the base64-encoded (partially) signed transaction"} + {RPCResult::Type::STR_HEX, "txid", /* optional */ true, "The transaction id for the send. Only 1 transaction is created regardless of the number of addresses."}, + {RPCResult::Type::STR_HEX, "hex", /* optional */ true, "If add_to_wallet is false, the hex-encoded raw transaction with signature(s)"}, + {RPCResult::Type::STR, "psbt", /* optional */ true, "If more signatures are needed, or if add_to_wallet is false, the base64-encoded (partially) signed transaction"} } }, RPCExamples{"" @@ -4385,7 +4541,7 @@ static RPCHelpMan walletprocesspsbt() HELP_REQUIRING_PASSPHRASE, { {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction base64 string"}, - {"sign", RPCArg::Type::BOOL, RPCArg::Default{true}, "Also sign the transaction when updating"}, + {"sign", RPCArg::Type::BOOL, RPCArg::Default{true}, "Also sign the transaction when updating (requires wallet to be unlocked)"}, {"sighashtype", RPCArg::Type::STR, RPCArg::Default{"DEFAULT"}, "The signature hash type to sign with if not specified by the PSBT. Must be one of\n" " \"DEFAULT\"\n" " \"ALL\"\n" @@ -4408,7 +4564,7 @@ static RPCHelpMan walletprocesspsbt() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; const CWallet& wallet{*pwallet}; @@ -4432,6 +4588,9 @@ static RPCHelpMan walletprocesspsbt() bool sign = request.params[1].isNull() ? true : request.params[1].get_bool(); bool bip32derivs = request.params[3].isNull() ? true : request.params[3].get_bool(); bool complete = true; + + if (sign) EnsureWalletIsUnlocked(*pwallet); + const TransactionError err{wallet.FillPSBT(psbtx, complete, nHashType, sign, bip32derivs)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); @@ -4452,7 +4611,9 @@ static RPCHelpMan walletcreatefundedpsbt() { return RPCHelpMan{"walletcreatefundedpsbt", "\nCreates and funds a transaction in the Partially Signed Transaction format.\n" - "Implements the Creator and Updater roles.\n", + "Implements the Creator and Updater roles.\n" + "All existing inputs must either have their previous output transaction be in the wallet\n" + "or be in the UTXO set. Solving data must be provided for non-wallet inputs.\n", { {"inputs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "Leave empty to add inputs automatically. See add_inputs option.", { @@ -4484,6 +4645,7 @@ static RPCHelpMan walletcreatefundedpsbt() }, {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{false}, "If inputs are specified, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -4504,12 +4666,8 @@ static RPCHelpMan walletcreatefundedpsbt() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, }, + FundTxDoc()), "options"}, {"bip32derivs", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include BIP 32 derivation paths for public keys if we know them"}, }, @@ -4753,6 +4911,7 @@ static const CRPCCommand commands[] = { "wallet", &listwallets, }, { "wallet", &loadwallet, }, { "wallet", &lockunspent, }, + { "wallet", &newkeypool, }, { "wallet", &removeprunedfunds, }, { "wallet", &rescanblockchain, }, { "wallet", &send, }, diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 8b88ffe8ed..3a85fc0c64 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -34,6 +34,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques void EnsureWalletIsUnlocked(const CWallet&); WalletContext& EnsureWalletContext(const std::any& context); LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create = false); +const LegacyScriptPubKeyMan& EnsureConstLegacyScriptPubKeyMan(const CWallet& wallet); RPCHelpMan getaddressinfo(); RPCHelpMan signrawtransactionwithwallet(); diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index ea045eb6d8..241d77c9de 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -45,7 +45,7 @@ bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::v // Call Salvage with fAggressive=true to // get as much data as possible. // Rewrite salvaged data to fresh wallet file - // Set -rescan so any missing transactions will be + // Rescan so any missing transactions will be // found. int64_t now = GetTime(); std::string newFilename = strprintf("%s.%d.bak", filename, now); @@ -133,7 +133,7 @@ bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::v } DbTxn* ptxn = env->TxnBegin(); - CWallet dummyWallet(nullptr, "", CreateDummyWalletDatabase()); + CWallet dummyWallet(nullptr, "", gArgs, CreateDummyWalletDatabase()); for (KeyValPair& row : salvagedData) { /* Filter for only private key type KV pairs to be added to the salvaged wallet */ diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index fe41f9b8cc..9173c790d4 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -331,7 +331,7 @@ bool LegacyScriptPubKeyMan::TopUpInactiveHDChain(const CKeyID seed_id, int64_t i CHDChain& chain = it->second; // Top up key pool - int64_t target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + int64_t target_size = std::max(gArgs.GetIntArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); // "size" of the keypools. Not really the size, actually the difference between index and the chain counter // Since chain counter is 1 based and index is 0 based, one of them needs to be offset by 1. @@ -400,7 +400,7 @@ void LegacyScriptPubKeyMan::UpgradeKeyMetadata() CKey key; GetKey(meta.hd_seed_id, key); CExtKey masterKey; - masterKey.SetSeed(key.begin(), key.size()); + masterKey.SetSeed(key); // Add to map CKeyID master_id = masterKey.key.GetPubKey().GetID(); std::copy(master_id.begin(), master_id.begin() + 4, meta.key_origin.fingerprint); @@ -489,7 +489,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, int new_version, bilingual } // Regenerate the keypool if upgraded to HD if (hd_upgrade) { - if (!TopUp()) { + if (!NewKeyPool()) { error = _("Unable to generate keys"); return false; } @@ -1085,7 +1085,7 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& if (!GetKey(hd_chain.seed_id, seed)) throw std::runtime_error(std::string(__func__) + ": seed not found"); - masterKey.SetSeed(seed.begin(), seed.size()); + masterKey.SetSeed(seed); // derive m/0' // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) @@ -1259,7 +1259,7 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) if (kpSize > 0) nTargetSize = kpSize; else - nTargetSize = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); + nTargetSize = std::max(gArgs.GetIntArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); // count amount of available keys (internal, external) // make sure the keypool of external and internal keys fits the user selected target (-keypool) @@ -1764,7 +1764,7 @@ bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) if (size > 0) { target_size = size; } else { - target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + target_size = std::max(gArgs.GetIntArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); } // Calculate the new range_end diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 4a7a268982..5470177440 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -2,9 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> #include <policy/policy.h> +#include <script/signingprovider.h> #include <util/check.h> #include <util/fees.h> #include <util/moneystr.h> @@ -31,21 +33,27 @@ std::string COutput::ToString() const return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) { CMutableTransaction txn; txn.vin.push_back(CTxIn(COutPoint())); - if (!wallet->DummySignInput(txn.vin[0], txout, use_max_sig)) { + if (!provider || !DummySignInput(*provider, txn.vin[0], txout, use_max_sig)) { return -1; } return GetVirtualTransactionInputSize(txn.vin[0]); } +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +{ + const std::unique_ptr<SigningProvider> provider = wallet->GetSolvingProvider(txout.scriptPubKey); + return CalculateMaximumSignedInputSize(txout, provider.get(), use_max_sig); +} + // txouts needs to be in the order of tx.vin -TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, bool use_max_sig) +TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control) { CMutableTransaction txNew(tx); - if (!wallet->DummySignTx(txNew, txouts, use_max_sig)) { + if (!wallet->DummySignTx(txNew, txouts, coin_control)) { return TxSize{-1, -1}; } CTransaction ctx(txNew); @@ -54,19 +62,27 @@ TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *walle return TxSize{vsize, weight}; } -TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig) +TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const CCoinControl* coin_control) { std::vector<CTxOut> txouts; + // Look up the inputs. The inputs are either in the wallet, or in coin_control. for (const CTxIn& input : tx.vin) { const auto mi = wallet->mapWallet.find(input.prevout.hash); // Can not estimate size without knowing the input details - if (mi == wallet->mapWallet.end()) { + if (mi != wallet->mapWallet.end()) { + assert(input.prevout.n < mi->second.tx->vout.size()); + txouts.emplace_back(mi->second.tx->vout.at(input.prevout.n)); + } else if (coin_control) { + CTxOut txout; + if (!coin_control->GetExternalOutput(input.prevout, txout)) { + return TxSize{-1, -1}; + } + txouts.emplace_back(txout); + } else { return TxSize{-1, -1}; } - assert(input.prevout.n < mi->second.tx->vout.size()); - txouts.emplace_back(mi->second.tx->vout[input.prevout.n]); } - return CalculateMaximumSignedTxSize(tx, wallet, txouts, use_max_sig); + return CalculateMaximumSignedTxSize(tx, wallet, txouts, coin_control); } void AvailableCoins(const CWallet& wallet, std::vector<COutput>& vCoins, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) @@ -387,6 +403,15 @@ bool AttemptSelection(const CWallet& wallet, const CAmount& nTargetValue, const results.emplace_back(std::make_tuple(waste, std::move(knapsack_coins), knapsack_value)); } + // We include the minimum final change for SRD as we do want to avoid making really small change. + // KnapsackSolver does not need this because it includes MIN_CHANGE internally. + const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + MIN_FINAL_CHANGE; + auto srd_result = SelectCoinsSRD(positive_groups, srd_target); + if (srd_result != std::nullopt) { + const auto waste = GetSelectionWaste(srd_result->first, coin_selection_params.m_cost_of_change, srd_target, !coin_selection_params.m_subtract_fee_outputs); + results.emplace_back(std::make_tuple(waste, std::move(srd_result->first), srd_result->second)); + } + if (results.size() == 0) { // No solution found return false; @@ -426,32 +451,40 @@ bool SelectCoins(const CWallet& wallet, const std::vector<COutput>& vAvailableCo std::vector<COutPoint> vPresetInputs; coin_control.ListSelected(vPresetInputs); - for (const COutPoint& outpoint : vPresetInputs) - { + for (const COutPoint& outpoint : vPresetInputs) { + int input_bytes = -1; + CTxOut txout; std::map<uint256, CWalletTx>::const_iterator it = wallet.mapWallet.find(outpoint.hash); - if (it != wallet.mapWallet.end()) - { + if (it != wallet.mapWallet.end()) { const CWalletTx& wtx = it->second; // Clearly invalid input, fail if (wtx.tx->vout.size() <= outpoint.n) { return false; } - // Just to calculate the marginal byte size - CInputCoin coin(wtx.tx, outpoint.n, GetTxSpendSize(wallet, wtx, outpoint.n, false)); - nValueFromPresetInputs += coin.txout.nValue; - if (coin.m_input_bytes <= 0) { - return false; // Not solvable, can't estimate size for fee - } - coin.effective_value = coin.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(coin.m_input_bytes); - if (coin_selection_params.m_subtract_fee_outputs) { - value_to_select -= coin.txout.nValue; - } else { - value_to_select -= coin.effective_value; + input_bytes = GetTxSpendSize(wallet, wtx, outpoint.n, false); + txout = wtx.tx->vout.at(outpoint.n); + } + if (input_bytes == -1) { + // The input is external. We either did not find the tx in mapWallet, or we did but couldn't compute the input size with wallet data + if (!coin_control.GetExternalOutput(outpoint, txout)) { + // Not ours, and we don't have solving data. + return false; } - setPresetCoins.insert(coin); + input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /* use_max_sig */ true); + } + + CInputCoin coin(outpoint, txout, input_bytes); + nValueFromPresetInputs += coin.txout.nValue; + if (coin.m_input_bytes == -1) { + return false; // Not solvable, can't estimate size for fee + } + coin.effective_value = coin.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(coin.m_input_bytes); + if (coin_selection_params.m_subtract_fee_outputs) { + value_to_select -= coin.txout.nValue; } else { - return false; // TODO: Allow non-wallet inputs + value_to_select -= coin.effective_value; } + setPresetCoins.insert(coin); } // remove preset inputs from vCoins so that Coin Selection doesn't pick them. @@ -779,10 +812,10 @@ static bool CreateTransactionInternal( } // Calculate the transaction fee - TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, coin_control.fAllowWatchOnly); + TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); int nBytes = tx_sizes.vsize; - if (nBytes < 0) { - error = _("Signing transaction failed"); + if (nBytes == -1) { + error = _("Missing solving data for estimating transaction size"); return false; } nFeeRet = coin_selection_params.m_effective_feerate.GetFee(nBytes); @@ -804,7 +837,7 @@ static bool CreateTransactionInternal( txNew.vout.erase(change_position); // Because we have dropped this change, the tx size and required fee will be different, so let's recalculate those - tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, coin_control.fAllowWatchOnly); + tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); nBytes = tx_sizes.vsize; fee_needed = coin_selection_params.m_effective_feerate.GetFee(nBytes); } diff --git a/src/wallet/spend.h b/src/wallet/spend.h index e39f134dc3..7467dd9fa3 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_SPEND_H #define BITCOIN_WALLET_SPEND_H +#include <consensus/amount.h> #include <wallet/coinselection.h> #include <wallet/transaction.h> #include <wallet/wallet.h> @@ -66,6 +67,7 @@ public: //Get the marginal bytes of spending the specified output int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); +int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); struct TxSize { int64_t vsize{-1}; @@ -76,8 +78,8 @@ struct TxSize { * Use DummySignatureCreator, which inserts 71 byte signatures everywhere. * NOTE: this requires that all inputs must be in mapWallet (eg the tx should * be AllInputsMine). */ -TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector<CTxOut>& txouts, bool use_max_sig = false); -TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, bool use_max_sig = false) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); +TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control = nullptr); +TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const CCoinControl* coin_control = nullptr) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); /** * populate vCoins with vector of available COutputs. diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 2e60aca017..c493b96248 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -67,7 +67,7 @@ static void SetPragma(sqlite3* db, const std::string& key, const std::string& va } SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock) - : WalletDatabase(), m_mock(mock), m_dir_path(dir_path.string()), m_file_path(file_path.string()) + : WalletDatabase(), m_mock(mock), m_dir_path(fs::PathToString(dir_path)), m_file_path(fs::PathToString(file_path)) { { LOCK(g_sqlite_mutex); @@ -206,12 +206,16 @@ void SQLiteDatabase::Open() if (m_db == nullptr) { if (!m_mock) { - TryCreateDirectories(m_dir_path); + TryCreateDirectories(fs::PathFromString(m_dir_path)); } int ret = sqlite3_open_v2(m_file_path.c_str(), &m_db, flags, nullptr); if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to open database: %s\n", sqlite3_errstr(ret))); } + ret = sqlite3_extended_result_codes(m_db, 1); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to enable extended result codes: %s\n", sqlite3_errstr(ret))); + } } if (sqlite3_db_readonly(m_db, "main") != 0) { @@ -224,7 +228,7 @@ void SQLiteDatabase::Open() // Now begin a transaction to acquire the exclusive lock. This lock won't be released until we close because of the exclusive locking mode. int ret = sqlite3_exec(m_db, "BEGIN EXCLUSIVE TRANSACTION", nullptr, nullptr, nullptr); if (ret != SQLITE_OK) { - throw std::runtime_error("SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?\n"); + throw std::runtime_error("SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of " PACKAGE_NAME "?\n"); } ret = sqlite3_exec(m_db, "COMMIT", nullptr, nullptr, nullptr); if (ret != SQLITE_OK) { diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 5d51809241..a8a7e12a32 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <node/context.h> #include <primitives/transaction.h> #include <random.h> @@ -28,19 +28,9 @@ BOOST_FIXTURE_TEST_SUITE(coinselector_tests, WalletTestingSetup) typedef std::set<CInputCoin> CoinSet; -static std::vector<COutput> vCoins; -static NodeContext testNode; -static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); -static CAmount balance = 0; - -CoinEligibilityFilter filter_standard(1, 6, 0); -CoinEligibilityFilter filter_confirmed(1, 1, 0); -CoinEligibilityFilter filter_standard_extra(6, 6, 0); -CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, - /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); +static const CoinEligibilityFilter filter_standard(1, 6, 0); +static const CoinEligibilityFilter filter_confirmed(1, 1, 0); +static const CoinEligibilityFilter filter_standard_extra(6, 6, 0); static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set) { @@ -62,9 +52,8 @@ static void add_coin(const CAmount& nValue, int nInput, CoinSet& set, CAmount fe set.insert(coin); } -static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) +static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) { - balance += nValue; static int nextLockTime = 0; CMutableTransaction tx; tx.nLockTime = nextLockTime++; // so all transactions get different hashes @@ -82,24 +71,19 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() tx.vin.resize(1); } - CWalletTx* wtx = wallet.AddToWallet(MakeTransactionRef(std::move(tx)), /* confirm= */ {}); + uint256 txid = tx.GetHash(); + + LOCK(wallet.cs_wallet); + auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)))); + assert(ret.second); + CWalletTx& wtx = (*ret.first).second; if (fIsFromMe) { - wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); - wtx->m_is_cache_empty = false; + wtx.m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); + wtx.m_is_cache_empty = false; } - COutput output(wallet, *wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); - vCoins.push_back(output); -} -static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) -{ - add_coin(testWallet, nValue, nAge, fIsFromMe, nInput, spendable); -} - -static void empty_wallet(void) -{ - vCoins.clear(); - balance = 0; + COutput output(wallet, wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); + coins.push_back(output); } static bool equal_sets(CoinSet a, CoinSet b) @@ -142,20 +126,20 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) return static_groups; } -inline std::vector<OutputGroup>& KnapsackGroupOutputs(const CoinEligibilityFilter& filter) +inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput>& coins, CWallet& wallet, const CoinEligibilityFilter& filter) { + CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, + /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), + /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), + /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); static std::vector<OutputGroup> static_groups; - static_groups = GroupOutputs(testWallet, vCoins, coin_selection_params, filter, /* positive_only */false); + static_groups = GroupOutputs(wallet, coins, coin_selection_params, filter, /* positive_only */false); return static_groups; } // Branch and bound coin selection tests BOOST_AUTO_TEST_CASE(bnb_search_test) { - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); - // Setup std::vector<CInputCoin> utxo_pool; CoinSet selection; @@ -288,196 +272,213 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(3000), /* long_term_feerate= */ CFeeRate(1000), /* discard_feerate= */ CFeeRate(1000), /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); - CoinSet setCoinsRet; - CAmount nValueRet; - empty_wallet(); - add_coin(1); - vCoins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail - BOOST_CHECK(!SelectCoinsBnB(GroupCoins(vCoins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); - - // Test fees subtracted from output: - empty_wallet(); - add_coin(1 * CENT); - vCoins.at(0).nInputBytes = 40; - coin_selection_params_bnb.m_subtract_fee_outputs = true; - BOOST_CHECK(SelectCoinsBnB(GroupCoins(vCoins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); - BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); - - // Make sure that can use BnB when there are preset inputs - empty_wallet(); { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); - wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); - add_coin(*wallet, 5 * CENT, 6 * 24, false, 0, true); - add_coin(*wallet, 3 * CENT, 6 * 24, false, 0, true); - add_coin(*wallet, 2 * CENT, 6 * 24, false, 0, true); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + + std::vector<COutput> coins; + CoinSet setCoinsRet; + CAmount nValueRet; + + add_coin(coins, *wallet, 1); + coins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail + BOOST_CHECK(!SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); + + // Test fees subtracted from output: + coins.clear(); + add_coin(coins, *wallet, 1 * CENT); + coins.at(0).nInputBytes = 40; + coin_selection_params_bnb.m_subtract_fee_outputs = true; + BOOST_CHECK(SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); + BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); + } + + { + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + + std::vector<COutput> coins; + CoinSet setCoinsRet; + CAmount nValueRet; + + add_coin(coins, *wallet, 5 * CENT, 6 * 24, false, 0, true); + add_coin(coins, *wallet, 3 * CENT, 6 * 24, false, 0, true); + add_coin(coins, *wallet, 2 * CENT, 6 * 24, false, 0, true); CCoinControl coin_control; coin_control.fAllowOtherInputs = true; - coin_control.Select(COutPoint(vCoins.at(0).tx->GetHash(), vCoins.at(0).i)); + coin_control.Select(COutPoint(coins.at(0).tx->GetHash(), coins.at(0).i)); coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); - BOOST_CHECK(SelectCoins(*wallet, vCoins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb)); + BOOST_CHECK(SelectCoins(*wallet, coins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb)); } } BOOST_AUTO_TEST_CASE(knapsack_solver_test) { + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + CoinSet setCoinsRet, setCoinsRet2; CAmount nValueRet; - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); + std::vector<COutput> coins; // test multiple times to allow for differences in the shuffle order for (int i = 0; i < RUN_TESTS; i++) { - empty_wallet(); + coins.clear(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); - add_coin(1*CENT, 4); // add a new 1 cent coin + add_coin(coins, *wallet, 1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // but we can find a new 1 cent - BOOST_CHECK(KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); - add_coin(2*CENT); // add a mature 2 cent coin + add_coin(coins, *wallet, 2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!KnapsackSolver(3 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(3 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // we can make 3 cents of new coins - BOOST_CHECK(KnapsackSolver(3 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(3 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 3 * CENT); - add_coin(5*CENT); // add a mature 5 cent coin, - add_coin(10*CENT, 3, true); // a new 10 cent coin sent from one of our own addresses - add_coin(20*CENT); // and a mature 20 cent coin + add_coin(coins, *wallet, 5*CENT); // add a mature 5 cent coin, + add_coin(coins, *wallet, 10*CENT, 3, true); // a new 10 cent coin sent from one of our own addresses + add_coin(coins, *wallet, 20*CENT); // and a mature 20 cent coin // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_standard_extra), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard_extra), setCoinsRet, nValueRet)); // but we can make 37 cents if we accept new coins from ourself - BOOST_CHECK(KnapsackSolver(37 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(37 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 37 * CENT); // and we can make 38 cents if we accept all new coins - BOOST_CHECK(KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - BOOST_CHECK(KnapsackSolver(34 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(34 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - BOOST_CHECK(KnapsackSolver(7 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(7 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 7 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - BOOST_CHECK(KnapsackSolver(8 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(8 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK(nValueRet == 8 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - BOOST_CHECK(KnapsackSolver(9 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(9 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 10 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // now clear out the wallet and start again to test choosing between subsets of smaller coins and the next biggest coin - empty_wallet(); + coins.clear(); - add_coin( 6*CENT); - add_coin( 7*CENT); - add_coin( 8*CENT); - add_coin(20*CENT); - add_coin(30*CENT); // now we have 6+7+8+20+30 = 71 cents total + add_coin(coins, *wallet, 6*CENT); + add_coin(coins, *wallet, 7*CENT); + add_coin(coins, *wallet, 8*CENT); + add_coin(coins, *wallet, 20*CENT); + add_coin(coins, *wallet, 30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - BOOST_CHECK(KnapsackSolver(71 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); - BOOST_CHECK(!KnapsackSolver(72 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(71 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(72 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - add_coin( 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total + add_coin(coins, *wallet, 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); - add_coin( 18*CENT); // now we have 5+6+7+8+18+20+30 + add_coin(coins, *wallet, 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - BOOST_CHECK(KnapsackSolver(11 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(11 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 11 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // check that the smallest bigger coin is used - add_coin( 1*COIN); - add_coin( 2*COIN); - add_coin( 3*COIN); - add_coin( 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - BOOST_CHECK(KnapsackSolver(95 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + add_coin(coins, *wallet, 1*COIN); + add_coin(coins, *wallet, 2*COIN); + add_coin(coins, *wallet, 3*COIN); + add_coin(coins, *wallet, 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents + BOOST_CHECK(KnapsackSolver(95 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - BOOST_CHECK(KnapsackSolver(195 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(195 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // empty the wallet and start again, now with fractions of a cent, to test small change avoidance - empty_wallet(); - add_coin(MIN_CHANGE * 1 / 10); - add_coin(MIN_CHANGE * 2 / 10); - add_coin(MIN_CHANGE * 3 / 10); - add_coin(MIN_CHANGE * 4 / 10); - add_coin(MIN_CHANGE * 5 / 10); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 1 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 2 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 3 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // but if we add a bigger coin, small change is avoided - add_coin(1111*MIN_CHANGE); + add_coin(coins, *wallet, 1111*MIN_CHANGE); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // if we add more small coins: - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 7 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); // and try again to make 1.0 * MIN_CHANGE - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) // they tried to consolidate 10 50k coins into one 500k coin, and ended up with 50k in change - empty_wallet(); + coins.clear(); for (int j = 0; j < 20; j++) - add_coin(50000 * COIN); + add_coin(coins, *wallet, 50000 * COIN); - BOOST_CHECK(KnapsackSolver(500000 * COIN, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(500000 * COIN, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 10U); // in ten coins @@ -485,79 +486,79 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // we need to try finding an exact subset anyway // sometimes it will fail, and so we use the next biggest coin: - empty_wallet(); - add_coin(MIN_CHANGE * 5 / 10); - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 7 / 10); - add_coin(1111 * MIN_CHANGE); - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); + add_coin(coins, *wallet, 1111 * MIN_CHANGE); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1111 * MIN_CHANGE); // we get the bigger coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // but sometimes it's possible, and we use an exact subset (0.4 + 0.6 = 1.0) - empty_wallet(); - add_coin(MIN_CHANGE * 4 / 10); - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 8 / 10); - add_coin(1111 * MIN_CHANGE); - BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 8 / 10); + add_coin(coins, *wallet, 1111 * MIN_CHANGE); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // in two coins 0.4+0.6 // test avoiding small change - empty_wallet(); - add_coin(MIN_CHANGE * 5 / 100); - add_coin(MIN_CHANGE * 1); - add_coin(MIN_CHANGE * 100); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 100); + add_coin(coins, *wallet, MIN_CHANGE * 1); + add_coin(coins, *wallet, MIN_CHANGE * 100); // trying to make 100.01 from these three coins - BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 10001 / 100, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 10001 / 100, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 9990 / 100, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 9990 / 100, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 101 * MIN_CHANGE); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); - } - - // test with many inputs - for (CAmount amt=1500; amt < COIN; amt*=10) { - empty_wallet(); - // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) - for (uint16_t j = 0; j < 676; j++) - add_coin(amt); - - // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. - for (int i = 0; i < RUN_TESTS; i++) { - BOOST_CHECK(KnapsackSolver(2000, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); - - if (amt - 2000 < MIN_CHANGE) { - // needs more than one input: - uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); - CAmount returnValue = amt * returnSize; - BOOST_CHECK_EQUAL(nValueRet, returnValue); - BOOST_CHECK_EQUAL(setCoinsRet.size(), returnSize); - } else { - // one input is sufficient: - BOOST_CHECK_EQUAL(nValueRet, amt); - BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - } - } - } - - // test randomness - { - empty_wallet(); - for (int i2 = 0; i2 < 100; i2++) - add_coin(COIN); - - // Again, we only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. - for (int i = 0; i < RUN_TESTS; i++) { + } + + // test with many inputs + for (CAmount amt=1500; amt < COIN; amt*=10) { + coins.clear(); + // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) + for (uint16_t j = 0; j < 676; j++) + add_coin(coins, *wallet, amt); + + // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. + for (int i = 0; i < RUN_TESTS; i++) { + BOOST_CHECK(KnapsackSolver(2000, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); + + if (amt - 2000 < MIN_CHANGE) { + // needs more than one input: + uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); + CAmount returnValue = amt * returnSize; + BOOST_CHECK_EQUAL(nValueRet, returnValue); + BOOST_CHECK_EQUAL(setCoinsRet.size(), returnSize); + } else { + // one input is sufficient: + BOOST_CHECK_EQUAL(nValueRet, amt); + BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); + } + } + } + + // test randomness + { + coins.clear(); + for (int i2 = 0; i2 < 100; i2++) + add_coin(coins, *wallet, COIN); + + // Again, we only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. + for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(coins), setCoinsRet2, nValueRet)); BOOST_CHECK(!equal_sets(setCoinsRet, setCoinsRet2)); int fails = 0; @@ -567,66 +568,67 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // When choosing 1 from 100 identical coins, 1% of the time, this test will choose the same coin twice // which will cause it to fail. // To avoid that issue, run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(coins), setCoinsRet2, nValueRet)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } BOOST_CHECK_NE(fails, RANDOM_REPEATS); - } - - // add 75 cents in small change. not enough to make 90 cents, - // then try making 90 cents. there are multiple competing "smallest bigger" coins, - // one of which should be picked at random - add_coin(5 * CENT); - add_coin(10 * CENT); - add_coin(15 * CENT); - add_coin(20 * CENT); - add_coin(25 * CENT); - - for (int i = 0; i < RUN_TESTS; i++) { + } + + // add 75 cents in small change. not enough to make 90 cents, + // then try making 90 cents. there are multiple competing "smallest bigger" coins, + // one of which should be picked at random + add_coin(coins, *wallet, 5 * CENT); + add_coin(coins, *wallet, 10 * CENT); + add_coin(coins, *wallet, 15 * CENT); + add_coin(coins, *wallet, 20 * CENT); + add_coin(coins, *wallet, 25 * CENT); + + for (int i = 0; i < RUN_TESTS; i++) { int fails = 0; for (int j = 0; j < RANDOM_REPEATS; j++) { - BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(coins), setCoinsRet2, nValueRet)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } BOOST_CHECK_NE(fails, RANDOM_REPEATS); - } - } - - empty_wallet(); + } + } } BOOST_AUTO_TEST_CASE(ApproximateBestSubset) { + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + CoinSet setCoinsRet; CAmount nValueRet; - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); - - empty_wallet(); + std::vector<COutput> coins; // Test vValue sort order for (int i = 0; i < 1000; i++) - add_coin(1000 * COIN); - add_coin(3 * COIN); + add_coin(coins, *wallet, 1000 * COIN); + add_coin(coins, *wallet, 3 * COIN); - BOOST_CHECK(KnapsackSolver(1003 * COIN, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1003 * COIN, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1003 * COIN); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); - - empty_wallet(); } // Tests that with the ideal conditions, the coin selector will always be able to find a solution that can pay the target value BOOST_AUTO_TEST_CASE(SelectCoins_test) { - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); // Random generator stuff std::default_random_engine generator; @@ -636,12 +638,15 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) // Run this test 100 times for (int i = 0; i < 100; ++i) { - empty_wallet(); + std::vector<COutput> coins; + CAmount balance{0}; // Make a wallet with 1000 exponentially distributed random inputs for (int j = 0; j < 1000; ++j) { - add_coin((CAmount)(distribution(generator)*10000000)); + CAmount val = distribution(generator)*10000000; + add_coin(coins, *wallet, val); + balance += val; } // Generate a random fee rate in the range of 100 - 400 @@ -658,7 +663,7 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CoinSet out_set; CAmount out_value = 0; CCoinControl cc; - BOOST_CHECK(SelectCoins(testWallet, vCoins, target, out_set, out_value, cc, cs_params)); + BOOST_CHECK(SelectCoins(*wallet, coins, target, out_set, out_value, cc, cs_params)); BOOST_CHECK_GE(out_value, target); } } @@ -724,12 +729,25 @@ BOOST_AUTO_TEST_CASE(waste_test) BOOST_CHECK_LT(waste_nochange2, waste_nochange1); selection.clear(); - // 0 Waste only when fee == long term fee, no change, and no excess + // No Waste when fee == long_term_fee, no change, and no excess add_coin(1 * COIN, 1, selection, fee, fee); add_coin(2 * COIN, 2, selection, fee, fee); - const CAmount exact_target = in_amt - 2 * fee; - BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, 0, exact_target)); + const CAmount exact_target{in_amt - fee * 2}; + BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, /* change_cost */ 0, exact_target)); + selection.clear(); + // No Waste when (fee - long_term_fee) == (-cost_of_change), and no excess + const CAmount new_change_cost{fee_diff * 2}; + add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); + BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, new_change_cost, target)); + selection.clear(); + + // No Waste when (fee - long_term_fee) == (-excess), no change cost + const CAmount new_target{in_amt - fee * 2 - fee_diff * 2}; + add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); + BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, /* change cost */ 0, new_target)); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 16cb7e0baf..dba3f35025 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -16,7 +16,7 @@ BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, std::string& database_filename) { fs::path data_file = BDBDataFile(path); - database_filename = data_file.filename().string(); + database_filename = fs::PathToString(data_file.filename()); return GetBerkeleyEnv(data_file.parent_path()); } @@ -25,11 +25,7 @@ BOOST_AUTO_TEST_CASE(getwalletenv_file) std::string test_name = "test_name.dat"; const fs::path datadir = gArgs.GetDataDirNet(); fs::path file_path = datadir / test_name; -#if BOOST_VERSION >= 107700 - std::ofstream f(BOOST_FILESYSTEM_C_STR(file_path)); -#else - std::ofstream f(file_path.BOOST_FILESYSTEM_C_STR); -#endif // BOOST_VERSION >= 107700 + fs::ofstream f(file_path); f.close(); std::string filename; diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp new file mode 100644 index 0000000000..e8b49f1220 --- /dev/null +++ b/src/wallet/test/fuzz/notifications.cpp @@ -0,0 +1,173 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <util/translation.h> +#include <wallet/context.h> +#include <wallet/receive.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> +#include <wallet/walletutil.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +namespace { +const TestingSetup* g_setup; + +void initialize_setup() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); +} + +/** + * Wraps a descriptor wallet for fuzzing. The constructor writes the sqlite db + * to disk, the destructor deletes it. + */ +struct FuzzedWallet { + ArgsManager args; + WalletContext context; + std::shared_ptr<CWallet> wallet; + FuzzedWallet(const std::string& name) + { + context.args = &args; + context.chain = g_setup->m_node.chain.get(); + + DatabaseOptions options; + options.require_create = true; + options.create_flags = WALLET_FLAG_DESCRIPTORS; + const std::optional<bool> load_on_start; + gArgs.ForceSetArg("-keypool", "0"); // Avoid timeout in TopUp() + + DatabaseStatus status; + bilingual_str error; + std::vector<bilingual_str> warnings; + wallet = CreateWallet(context, name, load_on_start, options, status, error, warnings); + assert(wallet); + assert(error.empty()); + assert(warnings.empty()); + assert(wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + } + ~FuzzedWallet() + { + const auto name{wallet->GetName()}; + std::vector<bilingual_str> warnings; + std::optional<bool> load_on_start; + assert(RemoveWallet(context, wallet, load_on_start, warnings)); + assert(warnings.empty()); + UnloadWallet(std::move(wallet)); + fs::remove_all(GetWalletDir() / name); + } + CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) + { + auto type{fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)}; + if (type == OutputType::BECH32M) { + type = OutputType::BECH32; // TODO: Setup taproot descriptor and remove this line + } + CTxDestination dest; + bilingual_str error; + if (fuzzed_data_provider.ConsumeBool()) { + assert(wallet->GetNewDestination(type, "", dest, error)); + } else { + assert(wallet->GetNewChangeDestination(type, dest, error)); + } + assert(error.empty()); + return GetScriptForDestination(dest); + } +}; + +FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + // The total amount, to be distributed to the wallets a and b in txs + // without fee. Thus, the balance of the wallets should always equal the + // total amount. + const auto total_amount{ConsumeMoney(fuzzed_data_provider)}; + FuzzedWallet a{"fuzzed_wallet_a"}; + FuzzedWallet b{"fuzzed_wallet_b"}; + + // Keep track of all coins in this test. + // Each tuple in the chain represents the coins and the block created with + // those coins. Once the block is mined, the next tuple will have an empty + // block and the freshly mined coins. + using Coins = std::set<std::tuple<CAmount, COutPoint>>; + std::vector<std::tuple<Coins, CBlock>> chain; + { + // Add the initial entry + chain.emplace_back(); + auto& [coins, block]{chain.back()}; + coins.emplace(total_amount, COutPoint{uint256::ONE, 1}); + } + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 200) + { + CallOneOf( + fuzzed_data_provider, + [&] { + auto& [coins_orig, block]{chain.back()}; + // Copy the coins for this block and consume all of them + Coins coins = coins_orig; + while (!coins.empty()) { + // Create a new tx + CMutableTransaction tx{}; + // Add some coins as inputs to it + auto num_inputs{fuzzed_data_provider.ConsumeIntegralInRange<int>(1, coins.size())}; + CAmount in{0}; + while (num_inputs-- > 0) { + const auto& [coin_amt, coin_outpoint]{*coins.begin()}; + in += coin_amt; + tx.vin.emplace_back(coin_outpoint); + coins.erase(coins.begin()); + } + // Create some outputs spending all inputs, without fee + LIMITED_WHILE(in > 0 && fuzzed_data_provider.ConsumeBool(), 100) + { + const auto out_value{ConsumeMoney(fuzzed_data_provider, in)}; + in -= out_value; + auto& wallet{fuzzed_data_provider.ConsumeBool() ? a : b}; + tx.vout.emplace_back(out_value, wallet.GetScriptPubKey(fuzzed_data_provider)); + } + // Spend the remaining input value, if any + auto& wallet{fuzzed_data_provider.ConsumeBool() ? a : b}; + tx.vout.emplace_back(in, wallet.GetScriptPubKey(fuzzed_data_provider)); + // Add tx to block + block.vtx.emplace_back(MakeTransactionRef(tx)); + } + // Mine block + a.wallet->blockConnected(block, chain.size()); + b.wallet->blockConnected(block, chain.size()); + // Store the coins for the next block + Coins coins_new; + for (const auto& tx : block.vtx) { + uint32_t i{0}; + for (const auto& out : tx->vout) { + coins_new.emplace(out.nValue, COutPoint{tx->GetHash(), i++}); + } + } + chain.emplace_back(coins_new, CBlock{}); + }, + [&] { + if (chain.size() <= 1) return; // The first entry can't be removed + auto& [coins, block]{chain.back()}; + if (block.vtx.empty()) return; // Can only disconnect if the block was submitted first + // Disconnect block + a.wallet->blockDisconnected(block, chain.size() - 1); + b.wallet->blockDisconnected(block, chain.size() - 1); + chain.pop_back(); + }); + auto& [coins, first_block]{chain.front()}; + if (!first_block.vtx.empty()) { + // Only check balance when at least one block was submitted + const auto bal_a{GetBalance(*a.wallet).m_mine_trusted}; + const auto bal_b{GetBalance(*b.wallet).m_mine_trusted}; + assert(total_amount == bal_a + bal_b); + } + } +} +} // namespace diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 53c972c46d..170675c035 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -32,11 +32,7 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainNam fs::create_directories(m_walletdir_path_cases["default"]); fs::create_directories(m_walletdir_path_cases["custom"]); fs::create_directories(m_walletdir_path_cases["relative"]); -#if BOOST_VERSION >= 107700 - std::ofstream f(BOOST_FILESYSTEM_C_STR(m_walletdir_path_cases["file"])); -#else - std::ofstream f(m_walletdir_path_cases["file"].BOOST_FILESYSTEM_C_STR); -#endif // BOOST_VERSION >= 107700 + fs::ofstream f(m_walletdir_path_cases["file"]); f.close(); } @@ -50,5 +46,5 @@ InitWalletDirTestingSetup::~InitWalletDirTestingSetup() void InitWalletDirTestingSetup::SetWalletDir(const fs::path& walletdir_path) { - gArgs.ForceSetArg("-walletdir", walletdir_path.string()); + gArgs.ForceSetArg("-walletdir", fs::PathToString(walletdir_path)); } diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 45e1b8c4b8..222c2bf4b7 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -17,7 +17,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) SetWalletDir(m_walletdir_path_cases["default"]); bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetArg("-walletdir", ""); + fs::path walletdir = fs::PathFromString(gArgs.GetArg("-walletdir", "")); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -27,7 +27,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) SetWalletDir(m_walletdir_path_cases["custom"]); bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetArg("-walletdir", ""); + fs::path walletdir = fs::PathFromString(gArgs.GetArg("-walletdir", "")); fs::path expected_path = fs::canonical(m_walletdir_path_cases["custom"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) SetWalletDir(m_walletdir_path_cases["trailing"]); bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetArg("-walletdir", ""); + fs::path walletdir = fs::PathFromString(gArgs.GetArg("-walletdir", "")); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -77,7 +77,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) SetWalletDir(m_walletdir_path_cases["trailing2"]); bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetArg("-walletdir", ""); + fs::path walletdir = fs::PathFromString(gArgs.GetArg("-walletdir", "")); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 5d25885bd4..dda202d55e 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -34,7 +34,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); @@ -85,7 +85,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); @@ -102,7 +102,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -126,7 +126,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -178,7 +178,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -196,7 +196,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -211,7 +211,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -230,7 +230,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -261,7 +261,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -282,7 +282,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -308,7 +308,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -334,7 +334,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -361,7 +361,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // OP_RETURN { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -375,7 +375,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unspendable { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -389,7 +389,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unknown { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -403,7 +403,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Nonstandard { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 8a97f7779d..dd24fa2c19 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -13,10 +13,22 @@ BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) +static void import_descriptor(CWallet& wallet, const std::string& descriptor) + EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + AssertLockHeld(wallet.cs_wallet); + FlatSigningProvider provider; + std::string error; + std::unique_ptr<Descriptor> desc = Parse(descriptor, provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 10, 0); + wallet.AddWalletDescriptor(w_desc, provider, "", false); +} + BOOST_AUTO_TEST_CASE(psbt_updater_test) { - auto spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(m_wallet.cs_wallet, spk_man->cs_KeyStore); + LOCK(m_wallet.cs_wallet); + m_wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); // Create prevtxs and add to wallet CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); @@ -29,27 +41,10 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) s_prev_tx2 >> prev_tx2; m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2)); - // Add scripts - CScript rs1; - CDataStream s_rs1(ParseHex("475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae"), SER_NETWORK, PROTOCOL_VERSION); - s_rs1 >> rs1; - spk_man->AddCScript(rs1); - - CScript rs2; - CDataStream s_rs2(ParseHex("2200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903"), SER_NETWORK, PROTOCOL_VERSION); - s_rs2 >> rs2; - spk_man->AddCScript(rs2); - - CScript ws1; - CDataStream s_ws1(ParseHex("47522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae"), SER_NETWORK, PROTOCOL_VERSION); - s_ws1 >> ws1; - spk_man->AddCScript(ws1); - - // Add hd seed - CKey key = DecodeSecret("5KSSJQ7UNfFGwVgpCZDSHm5rVNhMFcFtvWM3zQ8mW4qNDEN7LFd"); // Mainnet and uncompressed form of cUkG8i1RFfWGWy5ziR11zJ5V4U4W3viSFCfyJmZnvQaUsd1xuF3T - CPubKey master_pub_key = spk_man->DeriveNewSeed(key); - spk_man->SetHDSeed(master_pub_key); - spk_man->NewKeyPool(); + // Import descriptors for keys and scripts + import_descriptor(m_wallet, "sh(multi(2,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/0h,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/1h))"); + import_descriptor(m_wallet, "sh(wsh(multi(2,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/2h,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/3h)))"); + import_descriptor(m_wallet, "wpkh(xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/*h)"); // Call FillPSBT PartiallySignedTransaction psbtx; @@ -71,7 +66,8 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Try to sign the mutated input SignatureData sigdata; - BOOST_CHECK(spk_man->FillPSBT(psbtx, PrecomputePSBTData(psbtx), SIGHASH_ALL, true, true) != TransactionError::OK); + BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true) != TransactionError::OK); + //BOOST_CHECK(spk_man->FillPSBT(psbtx, PrecomputePSBTData(psbtx), SIGHASH_ALL, true, true) != TransactionError::OK); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index 347a436429..0e78855ced 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -17,7 +17,7 @@ BOOST_FIXTURE_TEST_SUITE(scriptpubkeyman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(CanProvide) { // Set up wallet and keyman variables. - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); LegacyScriptPubKeyMan& keyman = *wallet.GetOrCreateLegacyScriptPubKeyMan(); // Make a 1 of 2 multisig script diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index e779b2450f..926f28686d 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <policy/fees.h> #include <validation.h> #include <wallet/coincontrol.h> @@ -16,7 +17,7 @@ BOOST_FIXTURE_TEST_SUITE(spend_tests, WalletTestingSetup) BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - auto wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), coinbaseKey); + auto wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), m_args, coinbaseKey); // Check that a subtract-from-recipient transaction slightly less than the // coinbase input amount does not create a change output (because it would @@ -32,6 +33,8 @@ BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) CCoinControl coin_control; coin_control.m_feerate.emplace(10000); coin_control.fOverrideFeeRate = true; + // We need to use a change type with high cost of change so that the leftover amount will be dropped to fee instead of added as a change output + coin_control.m_change_type = OutputType::LEGACY; FeeCalculation fee_calc; BOOST_CHECK(CreateTransaction(*wallet, {recipient}, tx, fee, change_pos, error, coin_control, fee_calc)); BOOST_CHECK_EQUAL(tx->vout.size(), 1); diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index c3061b93c0..93a3404d2c 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -6,6 +6,7 @@ #include <chain.h> #include <key.h> +#include <key_io.h> #include <test/util/setup_common.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -14,18 +15,25 @@ #include <memory> -std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key) +std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, ArgsManager& args, const CKey& key) { - auto wallet = std::make_unique<CWallet>(&chain, "", CreateMockWalletDatabase()); + auto wallet = std::make_unique<CWallet>(&chain, "", args, CreateMockWalletDatabase()); { LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(cchain.Height(), cchain.Tip()->GetBlockHash()); } wallet->LoadWallet(); { - auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); - spk_man->AddKeyPubKey(key, key.GetPubKey()); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + + FlatSigningProvider provider; + std::string error; + std::unique_ptr<Descriptor> desc = Parse("combo(" + EncodeSecret(key) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet->AddWalletDescriptor(w_desc, provider, "", false)) assert(false); } WalletRescanReserver reserver(*wallet); reserver.reserve(); diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index 288c111571..3adb82b85f 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -7,6 +7,7 @@ #include <memory> +class ArgsManager; class CChain; class CKey; class CWallet; @@ -14,6 +15,6 @@ namespace interfaces { class Chain; } // namespace interfaces -std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key); +std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, ArgsManager& args, const CKey& key); #endif // BITCOIN_WALLET_TEST_UTIL_H diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index fc744ebe5b..762702522e 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -4,11 +4,18 @@ #include <wallet/test/wallet_test_fixture.h> +#include <scheduler.h> + WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), - m_wallet(m_node.chain.get(), "", CreateMockWalletDatabase()) + m_wallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()) { m_wallet.LoadWallet(); m_chain_notifications_handler = m_node.chain->handleNotifications({ &m_wallet, [](CWallet*) {} }); m_wallet_client->registerRpcs(); } + +WalletTestingSetup::~WalletTestingSetup() +{ + if (m_node.scheduler) m_node.scheduler->stop(); +} diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index ab7fb8c42b..8bf2d36227 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -19,6 +19,7 @@ */ struct WalletTestingSetup : public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + ~WalletTestingSetup(); std::unique_ptr<interfaces::WalletClient> m_wallet_client = interfaces::MakeWalletClient(*m_node.chain, *Assert(m_node.args)); CWallet m_wallet; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 5431a38bee..4499eb5903 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -11,6 +11,7 @@ #include <vector> #include <interfaces/chain.h> +#include <key_io.h> #include <node/blockstorage.h> #include <node/context.h> #include <policy/policy.h> @@ -40,9 +41,10 @@ static_assert(WALLET_INCREMENTAL_RELAY_FEE >= DEFAULT_INCREMENTAL_RELAY_FEE, "wa BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) -static std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) +static const std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) { DatabaseOptions options; + options.create_flags = WALLET_FLAG_DESCRIPTORS; DatabaseStatus status; bilingual_str error; std::vector<bilingual_str> warnings; @@ -77,9 +79,13 @@ static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t in static void AddKey(CWallet& wallet, const CKey& key) { - auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); - spk_man->AddKeyPubKey(key, key.GetPubKey()); + LOCK(wallet.cs_wallet); + FlatSigningProvider provider; + std::string error; + std::unique_ptr<Descriptor> desc = Parse("combo(" + EncodeSecret(key) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet.AddWalletDescriptor(w_desc, provider, "", false)) assert(false); } BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) @@ -92,9 +98,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions fails to read an unknown start block. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -111,9 +118,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -137,9 +145,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -162,9 +171,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -198,7 +208,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // before the missing block, and success for a key whose creation time is // after. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); WalletContext context; @@ -232,8 +242,8 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) "seconds of key creation, and could contain transactions pertaining to the key. As a result, " "transactions and coins using this key may not appear in the wallet. This error could be caused " "by pruning or data corruption (see bitcoind log for details) and could be dealt with by " - "downloading and rescanning the relevant blocks (see -reindex and -rescan " - "options).\"}},{\"success\":true}]", + "downloading and rescanning the relevant blocks (see -reindex option and rescanblockchain " + "RPC).\"}},{\"success\":true}]", 0, oldTip->GetBlockTimeMax(), TIMESTAMP_WINDOW)); RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt); } @@ -258,13 +268,13 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) SetMockTime(KEY_TIME); m_coinbase_txns.emplace_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); - std::string backup_file = (gArgs.GetDataDirNet() / "wallet.backup").string(); + std::string backup_file = fs::PathToString(gArgs.GetDataDirNet() / "wallet.backup"); // Import key into wallet and call dumpwallet to create backup file. { WalletContext context; context.args = &gArgs; - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); @@ -286,7 +296,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); @@ -319,11 +329,13 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // debit functions. BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); - auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); CWalletTx wtx(m_coinbase_txns.back()); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); + wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash(), 0); @@ -336,7 +348,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); - BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); + AddKey(wallet, coinbaseKey); BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 50*COIN); } @@ -491,7 +503,7 @@ public: ListCoinsTestingSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), coinbaseKey); + wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), m_args, coinbaseKey); } ~ListCoinsTestingSetup() @@ -593,14 +605,26 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); - wallet->SetupLegacyScriptPubKeyMan(); - wallet->SetMinVersion(FEATURE_LATEST); - wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); - BOOST_CHECK(!wallet->TopUpKeyPool(1000)); - CTxDestination dest; - bilingual_str error; - BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + { + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); + wallet->SetupLegacyScriptPubKeyMan(); + wallet->SetMinVersion(FEATURE_LATEST); + wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + BOOST_CHECK(!wallet->TopUpKeyPool(1000)); + CTxDestination dest; + bilingual_str error; + BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + } + { + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetMinVersion(FEATURE_LATEST); + wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + CTxDestination dest; + bilingual_str error; + BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + } } // Explicit calculation which is used to test the wallet constant diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 094221adf2..1ccef31056 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_TRANSACTION_H #define BITCOIN_WALLET_TRANSACTION_H -#include <amount.h> +#include <consensus/amount.h> #include <primitives/transaction.h> #include <serialize.h> #include <wallet/ismine.h> @@ -19,25 +19,6 @@ typedef std::map<std::string, std::string> mapValue_t; - -static inline void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) -{ - if (!mapValue.count("n")) - { - nOrderPos = -1; // TODO: calculate elsewhere - return; - } - nOrderPos = atoi64(mapValue["n"]); -} - - -static inline void WriteOrderPos(const int64_t& nOrderPos, mapValue_t& mapValue) -{ - if (nOrderPos == -1) - return; - mapValue["n"] = ToString(nOrderPos); -} - /** Legacy class used for deserializing vtxPrev for backwards compatibility. * vtxPrev was removed in commit 93a18a3650292afbb441a47d1fa1b94aeb0164e3, * but old wallet.dat files may still contain vtxPrev vectors of CMerkleTxs. @@ -181,7 +162,8 @@ public: int block_height; uint256 hashBlock; int nIndex; - Confirmation(Status s = UNCONFIRMED, int b = 0, uint256 h = uint256(), int i = 0) : status(s), block_height(b), hashBlock(h), nIndex(i) {} + Confirmation(Status status = UNCONFIRMED, int block_height = 0, uint256 block_hash = uint256(), int block_index = 0) + : status{status}, block_height{block_height}, hashBlock{block_hash}, nIndex{block_index} {} }; Confirmation m_confirm; @@ -192,7 +174,9 @@ public: mapValue_t mapValueCopy = mapValue; mapValueCopy["fromaccount"] = ""; - WriteOrderPos(nOrderPos, mapValueCopy); + if (nOrderPos != -1) { + mapValueCopy["n"] = ToString(nOrderPos); + } if (nTimeSmart) { mapValueCopy["timesmart"] = strprintf("%u", nTimeSmart); } @@ -232,8 +216,10 @@ public: setConfirmed(); } - ReadOrderPos(nOrderPos, mapValue); - nTimeSmart = mapValue.count("timesmart") ? (unsigned int)atoi64(mapValue["timesmart"]) : 0; + const auto it_op = mapValue.find("n"); + nOrderPos = (it_op != mapValue.end()) ? LocaleIndependentAtoi<int64_t>(it_op->second) : -1; + const auto it_ts = mapValue.find("timesmart"); + nTimeSmart = (it_ts != mapValue.end()) ? static_cast<unsigned int>(LocaleIndependentAtoi<int64_t>(it_ts->second)) : 0; mapValue.erase("fromaccount"); mapValue.erase("spent"); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 70349b2455..7f60dd6906 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -6,6 +6,7 @@ #include <wallet/wallet.h> #include <chain.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <external_signer.h> @@ -220,7 +221,7 @@ std::shared_ptr<CWallet> LoadWalletInternal(WalletContext& context, const std::s } context.chain->initMessage(_("Loading wallet…").translated); - std::shared_ptr<CWallet> wallet = CWallet::Create(context, name, std::move(database), options.create_flags, error, warnings); + const std::shared_ptr<CWallet> wallet = CWallet::Create(context, name, std::move(database), options.create_flags, error, warnings); if (!wallet) { error = Untranslated("Wallet loading failed.") + Untranslated(" ") + error; status = DatabaseStatus::FAILED_LOAD; @@ -300,7 +301,7 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& // Make the wallet context.chain->initMessage(_("Loading wallet…").translated); - std::shared_ptr<CWallet> wallet = CWallet::Create(context, name, std::move(database), wallet_creation_flags, error, warnings); + const std::shared_ptr<CWallet> wallet = CWallet::Create(context, name, std::move(database), wallet_creation_flags, error, warnings); if (!wallet) { error = Untranslated("Wallet creation failed.") + Untranslated(" ") + error; status = DatabaseStatus::FAILED_CREATE; @@ -589,11 +590,16 @@ bool CWallet::IsSpent(const uint256& hash, unsigned int n) const return false; } -void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid) +void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid, WalletBatch* batch) { mapTxSpends.insert(std::make_pair(outpoint, wtxid)); - setLockedCoins.erase(outpoint); + if (batch) { + UnlockCoin(outpoint, batch); + } else { + WalletBatch temp_batch(GetDatabase()); + UnlockCoin(outpoint, &temp_batch); + } std::pair<TxSpends::iterator, TxSpends::iterator> range; range = mapTxSpends.equal_range(outpoint); @@ -601,7 +607,7 @@ void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid) } -void CWallet::AddToSpends(const uint256& wtxid) +void CWallet::AddToSpends(const uint256& wtxid, WalletBatch* batch) { auto it = mapWallet.find(wtxid); assert(it != mapWallet.end()); @@ -610,7 +616,7 @@ void CWallet::AddToSpends(const uint256& wtxid) return; for (const CTxIn& txin : thisTx.tx->vin) - AddToSpends(txin.prevout, wtxid); + AddToSpends(txin.prevout, wtxid, batch); } bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) @@ -879,7 +885,7 @@ bool CWallet::IsSpentKey(const uint256& hash, unsigned int n) const return false; } -CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx, bool fFlushOnClose) +CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx, bool fFlushOnClose, bool rescanning_old_block) { LOCK(cs_wallet); @@ -909,8 +915,8 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio wtx.nTimeReceived = chain().getAdjustedTime(); wtx.nOrderPos = IncOrderPosNext(&batch); wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); - wtx.nTimeSmart = ComputeTimeSmart(wtx); - AddToSpends(hash); + wtx.nTimeSmart = ComputeTimeSmart(wtx, rescanning_old_block); + AddToSpends(hash, &batch); } if (!fInsertedNew) @@ -953,7 +959,7 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio #if HAVE_SYSTEM // notify an external script when a wallet transaction comes in or is updated - std::string strCmd = gArgs.GetArg("-walletnotify", ""); + std::string strCmd = m_args.GetArg("-walletnotify", ""); if (!strCmd.empty()) { @@ -1026,7 +1032,7 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx return true; } -bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate, bool rescanning_old_block) { const CTransaction& tx = *ptx; { @@ -1064,7 +1070,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Co // Block disconnection override an abandoned tx as unconfirmed // which means user may have to call abandontransaction again - return AddToWallet(MakeTransactionRef(tx), confirm, /* update_wtx= */ nullptr, /* fFlushOnClose= */ false); + return AddToWallet(MakeTransactionRef(tx), confirm, /* update_wtx= */ nullptr, /* fFlushOnClose= */ false, rescanning_old_block); } } return false; @@ -1193,9 +1199,9 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c } } -void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool update_tx) +void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool update_tx, bool rescanning_old_block) { - if (!AddToWalletIfInvolvingMe(ptx, confirm, update_tx)) + if (!AddToWalletIfInvolvingMe(ptx, confirm, update_tx, rescanning_old_block)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1206,7 +1212,7 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmatio void CWallet::transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) { LOCK(cs_wallet); - SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); + SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { @@ -1247,7 +1253,7 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe // distinguishing between conflicted and unconfirmed transactions are // imperfect, and could be improved in general, see // https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Wallet-Transaction-Conflict-Tracking - SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); + SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); } } @@ -1275,7 +1281,7 @@ void CWallet::blockDisconnected(const CBlock& block, int height) m_last_block_processed_height = height - 1; m_last_block_processed = block.hashPrevBlock; for (const CTransactionRef& ptx : block.vtx) { - SyncTransaction(ptx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); + SyncTransaction(ptx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); } } @@ -1288,7 +1294,7 @@ void CWallet::updatedBlockTip() void CWallet::BlockUntilSyncedToCurrentChain() const { AssertLockNotHeld(cs_wallet); // Skip the queue-draining stuff if we know we're caught up with - // ::ChainActive().Tip(), otherwise put a callback in the validation interface queue and wait + // chain().Tip(), otherwise put a callback in the validation interface queue and wait // for the queue to drain enough to execute it (indicating we are caught up // at least with the time we entered this function). uint256 last_block_hash = WITH_LOCK(cs_wallet, return m_last_block_processed); @@ -1443,19 +1449,13 @@ bool CWallet::AddWalletFlags(uint64_t flags) // Helper for producing a max-sized low-S low-R signature (eg 71 bytes) // or a max-sized low-S signature (e.g. 72 bytes) if use_max_sig is true -bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) const +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) { // Fill in dummy signatures for fee calculation. const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; - std::unique_ptr<SigningProvider> provider = GetSolvingProvider(scriptPubKey); - if (!provider) { - // We don't know about this scriptpbuKey; - return false; - } - - if (!ProduceSignature(*provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { + if (!ProduceSignature(provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { return false; } UpdateInput(tx_in, sigdata); @@ -1463,14 +1463,21 @@ bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig } // Helper for producing a bunch of max-sized low-S low-R signatures (eg 71 bytes) -bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, bool use_max_sig) const +bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, const CCoinControl* coin_control) const { // Fill in dummy signatures for fee calculation. int nIn = 0; for (const auto& txout : txouts) { - if (!DummySignInput(txNew.vin[nIn], txout, use_max_sig)) { - return false; + CTxIn& txin = txNew.vin[nIn]; + // Use max sig if watch only inputs were used or if this particular input is an external input + // to ensure a sufficient fee is attained for the requested feerate. + const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(txin.prevout)); + const std::unique_ptr<SigningProvider> provider = GetSolvingProvider(txout.scriptPubKey); + if (!provider || !DummySignInput(*provider, txin, txout, use_max_sig)) { + if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, use_max_sig)) { + return false; + } } nIn++; @@ -1595,7 +1602,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Rescan started from block %s...\n", start_block.ToString()); fAbortRescan = false; - ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup + ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if rescan required on startup (e.g. due to corruption) uint256 tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); uint256 end_hash = tip_hash; if (max_height) chain().findAncestorByHeight(tip_hash, *max_height, FoundBlock().hash(end_hash)); @@ -1638,7 +1645,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], {CWalletTx::Status::CONFIRMED, block_height, block_hash, (int)posInBlock}, fUpdate); + SyncTransaction(block.vtx[posInBlock], {CWalletTx::Status::CONFIRMED, block_height, block_hash, (int)posInBlock}, fUpdate, /* rescanning_old_block */ true); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; @@ -2002,10 +2009,7 @@ DBErrors CWallet::LoadWallet() assert(m_internal_spk_managers.empty()); } - if (nLoadWalletRet != DBErrors::LOAD_OK) - return nLoadWalletRet; - - return DBErrors::LOAD_OK; + return nLoadWalletRet; } DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) @@ -2260,22 +2264,36 @@ bool CWallet::DisplayAddress(const CTxDestination& dest) return signer_spk_man->DisplayAddress(scriptPubKey, signer); } -void CWallet::LockCoin(const COutPoint& output) +bool CWallet::LockCoin(const COutPoint& output, WalletBatch* batch) { AssertLockHeld(cs_wallet); setLockedCoins.insert(output); + if (batch) { + return batch->WriteLockedUTXO(output); + } + return true; } -void CWallet::UnlockCoin(const COutPoint& output) +bool CWallet::UnlockCoin(const COutPoint& output, WalletBatch* batch) { AssertLockHeld(cs_wallet); - setLockedCoins.erase(output); + bool was_locked = setLockedCoins.erase(output); + if (batch && was_locked) { + return batch->EraseLockedUTXO(output); + } + return true; } -void CWallet::UnlockAllCoins() +bool CWallet::UnlockAllCoins() { AssertLockHeld(cs_wallet); + bool success = true; + WalletBatch batch(GetDatabase()); + for (auto it = setLockedCoins.begin(); it != setLockedCoins.end(); ++it) { + success &= batch.EraseLockedUTXO(*it); + } setLockedCoins.clear(); + return success; } bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const @@ -2365,6 +2383,8 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { * - If sending a transaction, assign its timestamp to the current time. * - If receiving a transaction outside a block, assign its timestamp to the * current time. + * - If receiving a transaction during a rescanning process, assign all its + * (not already known) transactions' timestamps to the block time. * - If receiving a block with a future timestamp, assign all its (not already * known) transactions' timestamps to the current time. * - If receiving a block with a past timestamp, before the most recent known @@ -2379,38 +2399,43 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { * https://bitcointalk.org/?topic=54527, or * https://github.com/bitcoin/bitcoin/pull/1393. */ -unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const +unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx, bool rescanning_old_block) const { unsigned int nTimeSmart = wtx.nTimeReceived; if (!wtx.isUnconfirmed() && !wtx.isAbandoned()) { int64_t blocktime; - if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(blocktime))) { - int64_t latestNow = wtx.nTimeReceived; - int64_t latestEntry = 0; - - // Tolerate times up to the last timestamp in the wallet not more than 5 minutes into the future - int64_t latestTolerated = latestNow + 300; - const TxItems& txOrdered = wtxOrdered; - for (auto it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { - CWalletTx* const pwtx = it->second; - if (pwtx == &wtx) { - continue; - } - int64_t nSmartTime; - nSmartTime = pwtx->nTimeSmart; - if (!nSmartTime) { - nSmartTime = pwtx->nTimeReceived; - } - if (nSmartTime <= latestTolerated) { - latestEntry = nSmartTime; - if (nSmartTime > latestNow) { - latestNow = nSmartTime; + int64_t block_max_time; + if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(blocktime).maxTime(block_max_time))) { + if (rescanning_old_block) { + nTimeSmart = block_max_time; + } else { + int64_t latestNow = wtx.nTimeReceived; + int64_t latestEntry = 0; + + // Tolerate times up to the last timestamp in the wallet not more than 5 minutes into the future + int64_t latestTolerated = latestNow + 300; + const TxItems& txOrdered = wtxOrdered; + for (auto it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { + CWalletTx* const pwtx = it->second; + if (pwtx == &wtx) { + continue; + } + int64_t nSmartTime; + nSmartTime = pwtx->nTimeSmart; + if (!nSmartTime) { + nSmartTime = pwtx->nTimeReceived; + } + if (nSmartTime <= latestTolerated) { + latestEntry = nSmartTime; + if (nSmartTime > latestNow) { + latestNow = nSmartTime; + } + break; } - break; } - } - nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); + nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); + } } else { WalletLogPrintf("%s: found %s in block %s not in index\n", __func__, wtx.GetHash().ToString(), wtx.m_confirm.hashBlock.ToString()); } @@ -2490,16 +2515,16 @@ std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, cons // 2. Path to an existing directory. // 3. Path to a symlink to a directory. // 4. For backwards compatibility, the name of a data file in -walletdir. - const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), name); + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), fs::PathFromString(name)); fs::file_type path_type = fs::symlink_status(wallet_path).type(); if (!(path_type == fs::file_not_found || path_type == fs::directory_file || (path_type == fs::symlink_file && fs::is_directory(wallet_path)) || - (path_type == fs::regular_file && fs::path(name).filename() == name))) { + (path_type == fs::regular_file && fs::PathFromString(name).filename() == fs::PathFromString(name)))) { error_string = Untranslated(strprintf( "Invalid -wallet path '%s'. -wallet path should point to a directory where wallet.dat and " "database/log.?????????? files can be stored, a location where such a directory could be created, " "or (for backwards compatibility) the name of an existing data file in -walletdir (%s)", - name, GetWalletDir())); + name, fs::quoted(fs::PathToString(GetWalletDir())))); status = DatabaseStatus::FAILED_BAD_PATH; return nullptr; } @@ -2515,7 +2540,8 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri int64_t nStart = GetTimeMillis(); // TODO: Can't use std::make_shared because we need a custom deleter but // should be possible to use std::allocate_shared. - std::shared_ptr<CWallet> walletInstance(new CWallet(chain, name, std::move(database)), ReleaseWallet); + const std::shared_ptr<CWallet> walletInstance(new CWallet(chain, name, args, std::move(database)), ReleaseWallet); + bool rescan_required = false; DBErrors nLoadWalletRet = walletInstance->LoadWallet(); if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { @@ -2536,6 +2562,10 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri { error = strprintf(_("Wallet needed to be rewritten: restart %s to complete"), PACKAGE_NAME); return nullptr; + } else if (nLoadWalletRet == DBErrors::NEED_RESCAN) { + warnings.push_back(strprintf(_("Error reading %s! Transaction data may be missing or incorrect." + " Rescanning wallet."), walletFile)); + rescan_required = true; } else { error = strprintf(_("Error loading %s"), walletFile); @@ -2702,11 +2732,11 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri walletInstance->m_default_max_tx_fee = max_fee.value(); } - if (gArgs.IsArgSet("-consolidatefeerate")) { - if (std::optional<CAmount> consolidate_feerate = ParseMoney(gArgs.GetArg("-consolidatefeerate", ""))) { + if (args.IsArgSet("-consolidatefeerate")) { + if (std::optional<CAmount> consolidate_feerate = ParseMoney(args.GetArg("-consolidatefeerate", ""))) { walletInstance->m_consolidate_feerate = CFeeRate(*consolidate_feerate); } else { - error = AmountErrMsg("consolidatefeerate", gArgs.GetArg("-consolidatefeerate", "")); + error = AmountErrMsg("consolidatefeerate", args.GetArg("-consolidatefeerate", "")); return nullptr; } } @@ -2716,7 +2746,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri _("The wallet will avoid paying less than the minimum relay fee.")); } - walletInstance->m_confirm_target = args.GetArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); + walletInstance->m_confirm_target = args.GetIntArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); walletInstance->m_spend_zero_conf_change = args.GetBoolArg("-spendzeroconfchange", DEFAULT_SPEND_ZEROCONF_CHANGE); walletInstance->m_signal_rbf = args.GetBoolArg("-walletrbf", DEFAULT_WALLET_RBF); @@ -2727,7 +2757,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri LOCK(walletInstance->cs_wallet); - if (chain && !AttachChain(walletInstance, *chain, error, warnings)) { + if (chain && !AttachChain(walletInstance, *chain, rescan_required, error, warnings)) { return nullptr; } @@ -2749,7 +2779,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri return walletInstance; } -bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interfaces::Chain& chain, bilingual_str& error, std::vector<bilingual_str>& warnings) +bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interfaces::Chain& chain, const bool rescan_required, bilingual_str& error, std::vector<bilingual_str>& warnings) { LOCK(walletInstance->cs_wallet); // allow setting the chain if it hasn't been set already but prevent changing it @@ -2766,8 +2796,9 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf // interface. walletInstance->m_chain_notifications_handler = walletInstance->chain().handleNotifications(walletInstance); + // If rescan_required = true, rescan_height remains equal to 0 int rescan_height = 0; - if (!gArgs.GetBoolArg("-rescan", false)) + if (!rescan_required) { WalletBatch batch(walletInstance->GetDatabase()); CBlockLocator locator; @@ -3129,7 +3160,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() // Get the extended key CExtKey master_key; - master_key.SetSeed(seed_key.begin(), seed_key.size()); + master_key.SetSeed(seed_key); for (bool internal : {false, true}) { for (OutputType t : OUTPUT_TYPES) { @@ -3204,7 +3235,8 @@ void CWallet::LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool intern auto spk_man = m_spk_managers.at(id).get(); spk_mans[type] = spk_man; - if (spk_mans_other[type] == spk_man) { + const auto it = spk_mans_other.find(type); + if (it != spk_mans_other.end() && it->second == spk_man) { spk_mans_other.erase(type); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 2dc9eff712..3855ad821d 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_WALLET_WALLET_H #define BITCOIN_WALLET_WALLET_H -#include <amount.h> +#include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <outputtype.h> @@ -256,13 +256,13 @@ private: */ typedef std::multimap<COutPoint, uint256> TxSpends; TxSpends mapTxSpends GUARDED_BY(cs_wallet); - void AddToSpends(const COutPoint& outpoint, const uint256& wtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void AddToSpends(const uint256& wtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void AddToSpends(const COutPoint& outpoint, const uint256& wtxid, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void AddToSpends(const uint256& wtxid, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** - * Add a transaction to the wallet, or update it. pIndex and posInBlock should + * Add a transaction to the wallet, or update it. confirm.block_* should * be set when the transaction was known to be included in a block. When - * pIndex == nullptr, then wallet state is not updated in AddToWallet, but + * block_hash.IsNull(), then wallet state is not updated in AddToWallet, but * notifications happen and cached balances are marked dirty. * * If fUpdate is true, existing transactions will be updated. @@ -270,9 +270,12 @@ private: * assumption that any further notification of a transaction that was considered * abandoned is an indication that it is not safe to be considered abandoned. * Abandoned state should probably be more carefully tracked via different - * posInBlock signals or by checking mempool presence when necessary. + * chain notifications or by checking mempool presence when necessary. + * + * Should be called with rescanning_old_block set to true, if the transaction is + * not discovered in real time, but during a rescan of old blocks. */ - bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate, bool rescanning_old_block) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); @@ -282,9 +285,7 @@ private: void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected/ScanForWalletTransactions. - * Should be called with non-zero block_hash and posInBlock if this is for a transaction that is included in a block. */ - void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true, bool rescanning_old_block = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** WalletFlags set on this wallet. */ std::atomic<uint64_t> m_wallet_flags{0}; @@ -297,6 +298,9 @@ private: //! Unset the blank wallet flag and saves it to disk void UnsetBlankWalletFlag(WalletBatch& batch) override; + /** Provider of aplication-wide arguments. */ + const ArgsManager& m_args; + /** Interface for accessing chain state. */ interfaces::Chain* m_chain; @@ -334,7 +338,7 @@ private: * block locator and m_last_block_processed, and registering for * notifications about new blocks and transactions. */ - static bool AttachChain(const std::shared_ptr<CWallet>& wallet, interfaces::Chain& chain, bilingual_str& error, std::vector<bilingual_str>& warnings); + static bool AttachChain(const std::shared_ptr<CWallet>& wallet, interfaces::Chain& chain, const bool rescan_required, bilingual_str& error, std::vector<bilingual_str>& warnings); public: /** @@ -358,8 +362,9 @@ public: unsigned int nMasterKeyMaxID = 0; /** Construct wallet with specified name and database implementation. */ - CWallet(interfaces::Chain* chain, const std::string& name, std::unique_ptr<WalletDatabase> database) - : m_chain(chain), + CWallet(interfaces::Chain* chain, const std::string& name, const ArgsManager& args, std::unique_ptr<WalletDatabase> database) + : m_args(args), + m_chain(chain), m_name(name), m_database(std::move(database)) { @@ -449,9 +454,9 @@ public: bool DisplayAddress(const CTxDestination& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsLockedCoin(uint256 hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void LockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void UnlockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void UnlockAllCoins() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool LockCoin(const COutPoint& output, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool UnlockCoin(const COutPoint& output, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool UnlockAllCoins() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ListLockedCoins(std::vector<COutPoint>& vOutpts) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /* @@ -484,7 +489,7 @@ public: bool EncryptWallet(const SecureString& strWalletPassphrase); void GetKeyBirthTimes(std::map<CKeyID, int64_t> &mapKeyBirth) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - unsigned int ComputeTimeSmart(const CWalletTx& wtx) const; + unsigned int ComputeTimeSmart(const CWalletTx& wtx, bool rescanning_old_block) const; /** * Increment the next transaction order id @@ -503,7 +508,7 @@ public: //! @return true if wtx is changed and needs to be saved to disk, otherwise false using UpdateWalletTxFn = std::function<bool(CWalletTx& wtx, bool new_tx)>; - CWalletTx* AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true); + CWalletTx* AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true, bool rescanning_old_block = false); bool LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) override; void blockConnected(const CBlock& block, int height) override; @@ -573,14 +578,13 @@ public: /** Pass this transaction to node for mempool insertion and relay to peers if flag set to true */ bool SubmitTxMemoryPoolAndRelay(const CWalletTx& wtx, std::string& err_string, bool relay) const; - bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, bool use_max_sig = false) const + bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, const CCoinControl* coin_control = nullptr) const { std::vector<CTxOut> v_txouts(txouts.size()); std::copy(txouts.begin(), txouts.end(), v_txouts.begin()); - return DummySignTx(txNew, v_txouts, use_max_sig); + return DummySignTx(txNew, v_txouts, coin_control); } - bool DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, bool use_max_sig = false) const; - bool DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig = false) const; + bool DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, const CCoinControl* coin_control = nullptr) const; bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -925,4 +929,6 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); //! Remove wallet name from persistent configuration so it will not be loaded on startup. bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig); + #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 03464cd2c8..c920d4af51 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -40,6 +40,7 @@ const std::string FLAGS{"flags"}; const std::string HDCHAIN{"hdchain"}; const std::string KEYMETA{"keymeta"}; const std::string KEY{"key"}; +const std::string LOCKED_UTXO{"lockedutxo"}; const std::string MASTER_KEY{"mkey"}; const std::string MINVERSION{"minversion"}; const std::string NAME{"name"}; @@ -284,6 +285,16 @@ bool WalletBatch::WriteDescriptorCacheItems(const uint256& desc_id, const Descri return true; } +bool WalletBatch::WriteLockedUTXO(const COutPoint& output) +{ + return WriteIC(std::make_pair(DBKeys::LOCKED_UTXO, std::make_pair(output.hash, output.n)), uint8_t{'1'}); +} + +bool WalletBatch::EraseLockedUTXO(const COutPoint& output) +{ + return EraseIC(std::make_pair(DBKeys::LOCKED_UTXO, std::make_pair(output.hash, output.n))); +} + class CWalletScanState { public: unsigned int nKeys{0}; @@ -300,6 +311,7 @@ public: std::map<std::pair<uint256, CKeyID>, CKey> m_descriptor_keys; std::map<std::pair<uint256, CKeyID>, std::pair<CPubKey, std::vector<unsigned char>>> m_descriptor_crypt_keys; std::map<uint160, CHDChain> m_hd_chains; + bool tx_corrupt{false}; CWalletScanState() { } @@ -334,7 +346,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, // LoadToWallet call below creates a new CWalletTx that fill_wtx // callback fills with transaction metadata. auto fill_wtx = [&](CWalletTx& wtx, bool new_tx) { - assert(new_tx); + if(!new_tx) { + // There's some corruption here since the tx we just tried to load was already in the wallet. + // We don't consider this type of corruption critical, and can fix it by removing tx data and + // rescanning. + wss.tx_corrupt = true; + return false; + } ssValue >> wtx; if (wtx.GetHash() != hash) return false; @@ -701,6 +719,12 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, wss.m_descriptor_crypt_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), std::make_pair(pubkey, privkey))); wss.fIsEncrypted = true; + } else if (strType == DBKeys::LOCKED_UTXO) { + uint256 hash; + uint32_t n; + ssKey >> hash; + ssKey >> n; + pwallet->LockCoin(COutPoint(hash, n)); } else if (strType != DBKeys::BESTBLOCK && strType != DBKeys::BESTBLOCK_NOMERKLE && strType != DBKeys::MINVERSION && strType != DBKeys::ACENTRY && strType != DBKeys::VERSION && strType != DBKeys::SETTINGS && @@ -738,6 +762,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) { CWalletScanState wss; bool fNoncriticalErrors = false; + bool rescan_required = false; DBErrors result = DBErrors::LOAD_OK; LOCK(pwallet->cs_wallet); @@ -801,12 +826,17 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) } else if (strType == DBKeys::FLAGS) { // reading the wallet flags can only fail if unknown flags are present result = DBErrors::TOO_NEW; + } else if (wss.tx_corrupt) { + pwallet->WalletLogPrintf("Error: Corrupt transaction found. This can be fixed by removing transactions from wallet and rescanning.\n"); + // Set tx_corrupt back to false so that the error is only printed once (per corrupt tx) + wss.tx_corrupt = false; + result = DBErrors::CORRUPT; } else { // Leave other errors alone, if we try to fix them we might make things worse. fNoncriticalErrors = true; // ... but do warn the user there is something wrong. if (strType == DBKeys::TX) // Rescan if there is a bad transaction record: - gArgs.SoftSetBoolArg("-rescan", true); + rescan_required = true; } } if (!strErr.empty()) @@ -842,8 +872,11 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) ((DescriptorScriptPubKeyMan*)spk_man)->AddCryptedKey(desc_key_pair.first.second, desc_key_pair.second.first, desc_key_pair.second.second); } - if (fNoncriticalErrors && result == DBErrors::LOAD_OK) + if (rescan_required && result == DBErrors::LOAD_OK) { + result = DBErrors::NEED_RESCAN; + } else if (fNoncriticalErrors && result == DBErrors::LOAD_OK) { result = DBErrors::NONCRITICAL_ERROR; + } // Any wallet corruption at all: skip any rewriting or // upgrading, we don't want to make it worse. @@ -1073,7 +1106,7 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas try { exists = fs::symlink_status(path).type() != fs::file_not_found; } catch (const fs::filesystem_error& e) { - error = Untranslated(strprintf("Failed to access database path '%s': %s", path.string(), fsbridge::get_filesystem_error_message(e))); + error = Untranslated(strprintf("Failed to access database path '%s': %s", fs::PathToString(path), fsbridge::get_filesystem_error_message(e))); status = DatabaseStatus::FAILED_BAD_PATH; return nullptr; } @@ -1085,33 +1118,33 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas } if (IsSQLiteFile(SQLiteDataFile(path))) { if (format) { - error = Untranslated(strprintf("Failed to load database path '%s'. Data is in ambiguous format.", path.string())); + error = Untranslated(strprintf("Failed to load database path '%s'. Data is in ambiguous format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } format = DatabaseFormat::SQLITE; } } else if (options.require_existing) { - error = Untranslated(strprintf("Failed to load database path '%s'. Path does not exist.", path.string())); + error = Untranslated(strprintf("Failed to load database path '%s'. Path does not exist.", fs::PathToString(path))); status = DatabaseStatus::FAILED_NOT_FOUND; return nullptr; } if (!format && options.require_existing) { - error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in recognized format.", path.string())); + error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in recognized format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } if (format && options.require_create) { - error = Untranslated(strprintf("Failed to create database path '%s'. Database already exists.", path.string())); + error = Untranslated(strprintf("Failed to create database path '%s'. Database already exists.", fs::PathToString(path))); status = DatabaseStatus::FAILED_ALREADY_EXISTS; return nullptr; } // A db already exists so format is set, but options also specifies the format, so make sure they agree if (format && options.require_format && format != options.require_format) { - error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", path.string())); + error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } @@ -1133,7 +1166,7 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas #ifdef USE_SQLITE return MakeSQLiteDatabase(path, options, status, error); #endif - error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support SQLite database format.", path.string())); + error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support SQLite database format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } @@ -1141,7 +1174,7 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas #ifdef USE_BDB return MakeBerkeleyDatabase(path, options, status, error); #endif - error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support Berkeley DB database format.", path.string())); + error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support Berkeley DB database format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } @@ -1155,9 +1188,9 @@ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase() /** Return object for accessing temporary in-memory database. */ std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() { -#ifdef USE_BDB - return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); -#elif USE_SQLITE +#ifdef USE_SQLITE return std::make_unique<SQLiteDatabase>("", "", true); +#elif USE_BDB + return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); #endif } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 25c2ec5909..9c752623b3 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -6,7 +6,6 @@ #ifndef BITCOIN_WALLET_WALLETDB_H #define BITCOIN_WALLET_WALLETDB_H -#include <amount.h> #include <script/sign.h> #include <wallet/db.h> #include <wallet/walletutil.h> @@ -48,7 +47,8 @@ enum class DBErrors NONCRITICAL_ERROR, TOO_NEW, LOAD_FAIL, - NEED_REWRITE + NEED_REWRITE, + NEED_RESCAN }; namespace DBKeys { @@ -65,6 +65,7 @@ extern const std::string FLAGS; extern const std::string HDCHAIN; extern const std::string KEY; extern const std::string KEYMETA; +extern const std::string LOCKED_UTXO; extern const std::string MASTER_KEY; extern const std::string MINVERSION; extern const std::string NAME; @@ -250,6 +251,9 @@ public: bool WriteDescriptorLastHardenedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index); bool WriteDescriptorCacheItems(const uint256& desc_id, const DescriptorCache& cache); + bool WriteLockedUTXO(const COutPoint& output); + bool EraseLockedUTXO(const COutPoint& output); + /// Write destination data key,value tuple to database bool WriteDestData(const std::string &address, const std::string &key, const std::string &value); /// Erase destination data tuple from wallet database diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 50b6c9d29f..b609ba6881 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -40,7 +40,7 @@ static void WalletCreate(CWallet* wallet_instance, uint64_t wallet_creation_flag wallet_instance->TopUpKeyPool(); } -static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, DatabaseOptions options) +static const std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, const ArgsManager& args, DatabaseOptions options) { DatabaseStatus status; bilingual_str error; @@ -51,7 +51,7 @@ static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::pa } // dummy chain interface - std::shared_ptr<CWallet> wallet_instance{new CWallet(nullptr /* chain */, name, std::move(database)), WalletToolReleaseWallet}; + std::shared_ptr<CWallet> wallet_instance{new CWallet(nullptr /* chain */, name, args, std::move(database)), WalletToolReleaseWallet}; DBErrors load_wallet_ret; try { load_wallet_ret = wallet_instance->LoadWallet(); @@ -76,6 +76,10 @@ static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::pa } else if (load_wallet_ret == DBErrors::NEED_REWRITE) { tfm::format(std::cerr, "Wallet needed to be rewritten: restart %s to complete", PACKAGE_NAME); return nullptr; + } else if (load_wallet_ret == DBErrors::NEED_RESCAN) { + tfm::format(std::cerr, "Error reading %s! Some transaction data might be missing or" + " incorrect. Wallet requires a rescan.", + name); } else { tfm::format(std::cerr, "Error loading %s", name); return nullptr; @@ -116,22 +120,38 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) tfm::format(std::cerr, "The -descriptors option can only be used with the 'create' command.\n"); return false; } + if (args.IsArgSet("-legacy") && command != "create") { + tfm::format(std::cerr, "The -legacy option can only be used with the 'create' command.\n"); + return false; + } if (command == "create" && !args.IsArgSet("-wallet")) { tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); return false; } const std::string name = args.GetArg("-wallet", ""); - const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), name); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), fs::PathFromString(name)); if (command == "create") { DatabaseOptions options; options.require_create = true; - if (args.GetBoolArg("-descriptors", false)) { + // If -legacy is set, use it. Otherwise default to false. + bool make_legacy = args.GetBoolArg("-legacy", false); + // If neither -legacy nor -descriptors is set, default to true. If -descriptors is set, use its value. + bool make_descriptors = (!args.IsArgSet("-descriptors") && !args.IsArgSet("-legacy")) || (args.IsArgSet("-descriptors") && args.GetBoolArg("-descriptors", true)); + if (make_legacy && make_descriptors) { + tfm::format(std::cerr, "Only one of -legacy or -descriptors can be set to true, not both\n"); + return false; + } + if (!make_legacy && !make_descriptors) { + tfm::format(std::cerr, "One of -legacy or -descriptors must be set to true (or omitted)\n"); + return false; + } + if (make_descriptors) { options.create_flags |= WALLET_FLAG_DESCRIPTORS; options.require_format = DatabaseFormat::SQLITE; } - std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); + const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (wallet_instance) { WalletShowInfo(wallet_instance.get()); wallet_instance->Close(); @@ -139,7 +159,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) } else if (command == "info") { DatabaseOptions options; options.require_existing = true; - std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); + const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (!wallet_instance) return false; WalletShowInfo(wallet_instance.get()); wallet_instance->Close(); @@ -164,7 +184,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) } else if (command == "dump") { DatabaseOptions options; options.require_existing = true; - std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); + const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (!wallet_instance) return false; bilingual_str error; bool ret = DumpWallet(*wallet_instance, error); diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 1c518daba6..7f813432b3 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -12,7 +12,7 @@ fs::path GetWalletDir() fs::path path; if (gArgs.IsArgSet("-walletdir")) { - path = gArgs.GetArg("-walletdir", ""); + path = fs::PathFromString(gArgs.GetArg("-walletdir", "")); if (!fs::is_directory(path)) { // If the path specified doesn't exist, we return the deliberately // invalid empty string. diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 86f47d71f3..a53de34db4 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -47,7 +47,7 @@ CZMQNotificationInterface* CZMQNotificationInterface::Create() std::unique_ptr<CZMQAbstractNotifier> notifier = factory(); notifier->SetType(entry.first); notifier->SetAddress(address); - notifier->SetOutboundMessageHighWaterMark(static_cast<int>(gArgs.GetArg(arg + "hwm", CZMQAbstractNotifier::DEFAULT_ZMQ_SNDHWM))); + notifier->SetOutboundMessageHighWaterMark(static_cast<int>(gArgs.GetIntArg(arg + "hwm", CZMQAbstractNotifier::DEFAULT_ZMQ_SNDHWM))); notifiers.push_back(std::move(notifier)); } } |
