diff options
Diffstat (limited to 'src')
146 files changed, 3151 insertions, 1818 deletions
diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 7091ee3e11..32690fe484 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -19,7 +19,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, vector<COutput tx.nLockTime = nextLockTime++; // so all transactions get different hashes tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - CWalletTx* wtx = new CWalletTx(&wallet, tx); + CWalletTx* wtx = new CWalletTx(&wallet, MakeTransactionRef(std::move(tx))); int nAge = 6 * 24; COutput output(wtx, nInput, nAge, true, true); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 168006154f..737d3572ae 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -22,7 +22,7 @@ static void RIPEMD160(benchmark::State& state) uint8_t hash[CRIPEMD160::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); while (state.KeepRunning()) - CRIPEMD160().Write(begin_ptr(in), in.size()).Finalize(hash); + CRIPEMD160().Write(in.data(), in.size()).Finalize(hash); } static void SHA1(benchmark::State& state) @@ -30,7 +30,7 @@ static void SHA1(benchmark::State& state) uint8_t hash[CSHA1::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); while (state.KeepRunning()) - CSHA1().Write(begin_ptr(in), in.size()).Finalize(hash); + CSHA1().Write(in.data(), in.size()).Finalize(hash); } static void SHA256(benchmark::State& state) @@ -38,7 +38,7 @@ static void SHA256(benchmark::State& state) uint8_t hash[CSHA256::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); while (state.KeepRunning()) - CSHA256().Write(begin_ptr(in), in.size()).Finalize(hash); + CSHA256().Write(in.data(), in.size()).Finalize(hash); } static void SHA256_32b(benchmark::State& state) @@ -46,7 +46,7 @@ static void SHA256_32b(benchmark::State& state) std::vector<uint8_t> in(32,0); while (state.KeepRunning()) { for (int i = 0; i < 1000000; i++) { - CSHA256().Write(begin_ptr(in), in.size()).Finalize(&in[0]); + CSHA256().Write(in.data(), in.size()).Finalize(&in[0]); } } } @@ -56,7 +56,7 @@ static void SHA512(benchmark::State& state) uint8_t hash[CSHA512::OUTPUT_SIZE]; std::vector<uint8_t> in(BUFFER_SIZE,0); while (state.KeepRunning()) - CSHA512().Write(begin_ptr(in), in.size()).Finalize(hash); + CSHA512().Write(in.data(), in.size()).Finalize(hash); } static void SipHash_32b(benchmark::State& state) diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index dc3940cdbd..91b8c44472 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -91,7 +91,7 @@ static void VerifyScriptBench(benchmark::State& state) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << txSpend; int csuccess = bitcoinconsensus_verify_script_with_amount( - begin_ptr(txCredit.vout[0].scriptPubKey), + txCredit.vout[0].scriptPubKey.data(), txCredit.vout[0].scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, flags, nullptr); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 346d8e180d..1ed1449f03 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -623,7 +623,7 @@ static int CommandLineRawTx(int argc, char* argv[]) argv++; } - CTransaction txDecodeTmp; + CMutableTransaction tx; int startArg; if (!fCreateBlank) { @@ -636,15 +636,13 @@ static int CommandLineRawTx(int argc, char* argv[]) if (strHexTx == "-") // "-" implies standard input strHexTx = readStdin(); - if (!DecodeHexTx(txDecodeTmp, strHexTx, true)) + if (!DecodeHexTx(tx, strHexTx, true)) throw std::runtime_error("invalid transaction encoding"); startArg = 2; } else startArg = 1; - CMutableTransaction tx(txDecodeTmp); - for (int i = startArg; i < argc; i++) { std::string arg = argv[i]; std::string key, value; diff --git a/src/coins.h b/src/coins.h index d295b3c940..dd6ef6cc3a 100644 --- a/src/coins.h +++ b/src/coins.h @@ -467,7 +467,6 @@ public: friend class CCoinsModifier; private: - CCoinsMap::iterator FetchCoins(const uint256 &txid); CCoinsMap::const_iterator FetchCoins(const uint256 &txid) const; /** diff --git a/src/core_io.h b/src/core_io.h index 5aecbc4489..7642bc6d6e 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -11,13 +11,14 @@ class CBlock; class CScript; class CTransaction; +struct CMutableTransaction; class uint256; class UniValue; // core_read.cpp CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); -bool DecodeHexTx(CTransaction& tx, const std::string& strHexTx, bool fTryNoWitness = false); +bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx, bool fTryNoWitness = false); bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); uint256 ParseHashUV(const UniValue& v, const std::string& strName); uint256 ParseHashStr(const std::string&, const std::string& strName); @@ -25,7 +26,7 @@ std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strN // core_write.cpp std::string FormatScript(const CScript& script); -std::string EncodeHexTx(const CTransaction& tx); +std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry); diff --git a/src/core_read.cpp b/src/core_read.cpp index 7cfda6dd6d..b6d0285459 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -90,7 +90,7 @@ CScript ParseScript(const std::string& s) return result; } -bool DecodeHexTx(CTransaction& tx, const std::string& strHexTx, bool fTryNoWitness) +bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx, bool fTryNoWitness) { if (!IsHex(strHexTx)) return false; diff --git a/src/core_write.cpp b/src/core_write.cpp index ea01ddc10d..9f859ba9e1 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -116,9 +116,9 @@ string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode) return str; } -string EncodeHexTx(const CTransaction& tx) +string EncodeHexTx(const CTransaction& tx, const int serialFlags) { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serialFlags); ssTx << tx; return HexStr(ssTx.begin(), ssTx.end()); } diff --git a/src/crypto/ctaes/ctaes.c b/src/crypto/ctaes/ctaes.c index 2389fc0bb2..55962bf252 100644 --- a/src/crypto/ctaes/ctaes.c +++ b/src/crypto/ctaes/ctaes.c @@ -134,7 +134,7 @@ static void SubBytes(AES_state *s, int inv) { D = U7; } - /* Non-linear transformation (identical to the code in SubBytes) */ + /* Non-linear transformation (shared between the forward and backward case) */ M1 = T13 & T6; M6 = T3 & T16; M11 = T1 & T15; @@ -469,9 +469,9 @@ static void AES_encrypt(const AES_state* rounds, int nrounds, unsigned char* cip static void AES_decrypt(const AES_state* rounds, int nrounds, unsigned char* plain16, const unsigned char* cipher16) { /* Most AES decryption implementations use the alternate scheme - * (the Equivalent Inverse Cipher), which looks more like encryption, but - * needs different round constants. We can't reuse any code here anyway, so - * don't bother. */ + * (the Equivalent Inverse Cipher), which allows for more code reuse between + * the encryption and decryption code, but requires separate setup for both. + */ AES_state s = {{0}}; int round; diff --git a/src/init.cpp b/src/init.cpp index ba5fe4152a..71971a7642 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -380,6 +380,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), Params(CBaseChainParams::MAIN).GetDefaultPort(), Params(CBaseChainParams::TESTNET).GetDefaultPort())); strUsage += HelpMessageOpt("-proxy=<ip:port>", _("Connect through SOCKS5 proxy")); strUsage += HelpMessageOpt("-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), DEFAULT_PROXYRANDOMIZE)); + strUsage += HelpMessageOpt("-rpcserialversion", strprintf(_("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(>0) (default: %d)"), DEFAULT_RPC_SERIALIZE_VERSION)); strUsage += HelpMessageOpt("-seednode=<ip>", _("Connect to a node to retrieve peer addresses, and disconnect")); strUsage += HelpMessageOpt("-timeout=<n>", strprintf(_("Specify connection timeout in milliseconds (minimum: 1, default: %d)"), DEFAULT_CONNECT_TIMEOUT)); strUsage += HelpMessageOpt("-torcontrol=<ip>:<port>", strprintf(_("Tor control port to use if onion listening enabled (default: %s)"), DEFAULT_TOR_CONTROL)); @@ -984,6 +985,9 @@ bool AppInitParameterInteraction() if (GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); + if (GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) + return InitError("rpcserialversion must be non-negative."); + nMaxTipAge = GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); fEnableReplacement = GetBoolArg("-mempoolreplacement", DEFAULT_ENABLE_REPLACEMENT); diff --git a/src/leveldb/.travis.yml b/src/leveldb/.travis.yml new file mode 100644 index 0000000000..f5bd74c454 --- /dev/null +++ b/src/leveldb/.travis.yml @@ -0,0 +1,13 @@ +language: cpp +compiler: +- clang +- gcc +os: +- linux +- osx +sudo: false +before_install: +- echo $LANG +- echo $LC_ALL +script: +- make -j 4 check diff --git a/src/leveldb/Makefile b/src/leveldb/Makefile index 2bd2cadcdd..07a5a1ead6 100644 --- a/src/leveldb/Makefile +++ b/src/leveldb/Makefile @@ -20,208 +20,395 @@ $(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \ # this file is generated by the previous line to set build flags and sources include build_config.mk +TESTS = \ + db/autocompact_test \ + db/c_test \ + db/corruption_test \ + db/db_test \ + db/dbformat_test \ + db/fault_injection_test \ + db/filename_test \ + db/log_test \ + db/recovery_test \ + db/skiplist_test \ + db/version_edit_test \ + db/version_set_test \ + db/write_batch_test \ + helpers/memenv/memenv_test \ + issues/issue178_test \ + issues/issue200_test \ + table/filter_block_test \ + table/table_test \ + util/arena_test \ + util/bloom_test \ + util/cache_test \ + util/coding_test \ + util/crc32c_test \ + util/env_test \ + util/hash_test + +UTILS = \ + db/db_bench \ + db/leveldbutil + +# Put the object files in a subdirectory, but the application at the top of the object dir. +PROGNAMES := $(notdir $(TESTS) $(UTILS)) + +# On Linux may need libkyotocabinet-dev for dependency. +BENCHMARKS = \ + doc/bench/db_bench_sqlite3 \ + doc/bench/db_bench_tree_db + CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) LDFLAGS += $(PLATFORM_LDFLAGS) LIBS += $(PLATFORM_LIBS) -LIBOBJECTS = $(SOURCES:.cc=.o) -MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o) - -TESTUTIL = ./util/testutil.o -TESTHARNESS = ./util/testharness.o $(TESTUTIL) +SIMULATOR_OUTDIR=out-ios-x86 +DEVICE_OUTDIR=out-ios-arm -# Note: iOS should probably be using libtool, not ar. ifeq ($(PLATFORM), IOS) +# Note: iOS should probably be using libtool, not ar. AR=xcrun ar +SIMULATORSDK=$(shell xcrun -sdk iphonesimulator --show-sdk-path) +DEVICESDK=$(shell xcrun -sdk iphoneos --show-sdk-path) +DEVICE_CFLAGS = -isysroot "$(DEVICESDK)" -arch armv6 -arch armv7 -arch armv7s -arch arm64 +SIMULATOR_CFLAGS = -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64 +STATIC_OUTDIR=out-ios-universal +else +STATIC_OUTDIR=out-static +SHARED_OUTDIR=out-shared +STATIC_PROGRAMS := $(addprefix $(STATIC_OUTDIR)/, $(PROGNAMES)) +SHARED_PROGRAMS := $(addprefix $(SHARED_OUTDIR)/, db_bench) endif -TESTS = \ - arena_test \ - autocompact_test \ - bloom_test \ - c_test \ - cache_test \ - coding_test \ - corruption_test \ - crc32c_test \ - db_test \ - dbformat_test \ - env_test \ - filename_test \ - filter_block_test \ - hash_test \ - issue178_test \ - issue200_test \ - log_test \ - memenv_test \ - skiplist_test \ - table_test \ - version_edit_test \ - version_set_test \ - write_batch_test - -PROGRAMS = db_bench leveldbutil $(TESTS) -BENCHMARKS = db_bench_sqlite3 db_bench_tree_db - -LIBRARY = libleveldb.a -MEMENVLIBRARY = libmemenv.a +STATIC_LIBOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(SOURCES:.cc=.o)) +STATIC_MEMENVOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) + +DEVICE_LIBOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(SOURCES:.cc=.o)) +DEVICE_MEMENVOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) + +SIMULATOR_LIBOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(SOURCES:.cc=.o)) +SIMULATOR_MEMENVOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) + +SHARED_LIBOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(SOURCES:.cc=.o)) +SHARED_MEMENVOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) + +TESTUTIL := $(STATIC_OUTDIR)/util/testutil.o +TESTHARNESS := $(STATIC_OUTDIR)/util/testharness.o $(TESTUTIL) + +STATIC_TESTOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(TESTS))) +STATIC_UTILOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(UTILS))) +STATIC_ALLOBJS := $(STATIC_LIBOBJECTS) $(STATIC_MEMENVOBJECTS) $(STATIC_TESTOBJS) $(STATIC_UTILOBJS) $(TESTHARNESS) +DEVICE_ALLOBJS := $(DEVICE_LIBOBJECTS) $(DEVICE_MEMENVOBJECTS) +SIMULATOR_ALLOBJS := $(SIMULATOR_LIBOBJECTS) $(SIMULATOR_MEMENVOBJECTS) default: all # Should we build shared libraries? ifneq ($(PLATFORM_SHARED_EXT),) +# Many leveldb test apps use non-exported API's. Only build a subset for testing. +SHARED_ALLOBJS := $(SHARED_LIBOBJECTS) $(SHARED_MEMENVOBJECTS) $(TESTHARNESS) + ifneq ($(PLATFORM_SHARED_VERSIONED),true) -SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) -SHARED2 = $(SHARED1) -SHARED3 = $(SHARED1) -SHARED = $(SHARED1) +SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT) +SHARED_LIB2 = $(SHARED_LIB1) +SHARED_LIB3 = $(SHARED_LIB1) +SHARED_LIBS = $(SHARED_LIB1) +SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a else # Update db.h if you change these. -SHARED_MAJOR = 1 -SHARED_MINOR = 18 -SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) -SHARED2 = $(SHARED1).$(SHARED_MAJOR) -SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) -SHARED = $(SHARED1) $(SHARED2) $(SHARED3) -$(SHARED1): $(SHARED3) - ln -fs $(SHARED3) $(SHARED1) -$(SHARED2): $(SHARED3) - ln -fs $(SHARED3) $(SHARED2) +SHARED_VERSION_MAJOR = 1 +SHARED_VERSION_MINOR = 19 +SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT) +SHARED_LIB2 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR) +SHARED_LIB3 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR).$(SHARED_VERSION_MINOR) +SHARED_LIBS = $(SHARED_OUTDIR)/$(SHARED_LIB1) $(SHARED_OUTDIR)/$(SHARED_LIB2) $(SHARED_OUTDIR)/$(SHARED_LIB3) +$(SHARED_OUTDIR)/$(SHARED_LIB1): $(SHARED_OUTDIR)/$(SHARED_LIB3) + ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB1) +$(SHARED_OUTDIR)/$(SHARED_LIB2): $(SHARED_OUTDIR)/$(SHARED_LIB3) + ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB2) +SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a endif -$(SHARED3): - $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3) $(LIBS) +$(SHARED_OUTDIR)/$(SHARED_LIB3): $(SHARED_LIBOBJECTS) + $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED_LIB2) $(SHARED_LIBOBJECTS) -o $(SHARED_OUTDIR)/$(SHARED_LIB3) $(LIBS) endif # PLATFORM_SHARED_EXT -all: $(SHARED) $(LIBRARY) +all: $(SHARED_LIBS) $(SHARED_PROGRAMS) $(STATIC_OUTDIR)/libleveldb.a $(STATIC_OUTDIR)/libmemenv.a $(STATIC_PROGRAMS) -check: all $(PROGRAMS) $(TESTS) - for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done +check: $(STATIC_PROGRAMS) + for t in $(notdir $(TESTS)); do echo "***** Running $$t"; $(STATIC_OUTDIR)/$$t || exit 1; done clean: - -rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk - -rm -rf ios-x86/* ios-arm/* + -rm -rf out-static out-shared out-ios-x86 out-ios-arm out-ios-universal + -rm -f build_config.mk + -rm -rf ios-x86 ios-arm -$(LIBRARY): $(LIBOBJECTS) - rm -f $@ - $(AR) -rs $@ $(LIBOBJECTS) +$(STATIC_OUTDIR): + mkdir $@ -db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) $(LDFLAGS) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS) +$(STATIC_OUTDIR)/db: | $(STATIC_OUTDIR) + mkdir $@ -db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) $(LDFLAGS) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS) +$(STATIC_OUTDIR)/helpers/memenv: | $(STATIC_OUTDIR) + mkdir -p $@ -db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) +$(STATIC_OUTDIR)/port: | $(STATIC_OUTDIR) + mkdir $@ -leveldbutil: db/leveldb_main.o $(LIBOBJECTS) - $(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS) +$(STATIC_OUTDIR)/table: | $(STATIC_OUTDIR) + mkdir $@ -arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(STATIC_OUTDIR)/util: | $(STATIC_OUTDIR) + mkdir $@ -autocompact_test: db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +.PHONY: STATIC_OBJDIRS +STATIC_OBJDIRS: \ + $(STATIC_OUTDIR)/db \ + $(STATIC_OUTDIR)/port \ + $(STATIC_OUTDIR)/table \ + $(STATIC_OUTDIR)/util \ + $(STATIC_OUTDIR)/helpers/memenv -bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR): + mkdir $@ -c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/db: | $(SHARED_OUTDIR) + mkdir $@ -cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/helpers/memenv: | $(SHARED_OUTDIR) + mkdir -p $@ -coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/port: | $(SHARED_OUTDIR) + mkdir $@ -corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/table: | $(SHARED_OUTDIR) + mkdir $@ -crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SHARED_OUTDIR)/util: | $(SHARED_OUTDIR) + mkdir $@ -db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +.PHONY: SHARED_OBJDIRS +SHARED_OBJDIRS: \ + $(SHARED_OUTDIR)/db \ + $(SHARED_OUTDIR)/port \ + $(SHARED_OUTDIR)/table \ + $(SHARED_OUTDIR)/util \ + $(SHARED_OUTDIR)/helpers/memenv -dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR): + mkdir $@ -env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/db: | $(DEVICE_OUTDIR) + mkdir $@ -filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/helpers/memenv: | $(DEVICE_OUTDIR) + mkdir -p $@ -filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/port: | $(DEVICE_OUTDIR) + mkdir $@ -hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/table: | $(DEVICE_OUTDIR) + mkdir $@ -issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(DEVICE_OUTDIR)/util: | $(DEVICE_OUTDIR) + mkdir $@ -issue200_test: issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +.PHONY: DEVICE_OBJDIRS +DEVICE_OBJDIRS: \ + $(DEVICE_OUTDIR)/db \ + $(DEVICE_OUTDIR)/port \ + $(DEVICE_OUTDIR)/table \ + $(DEVICE_OUTDIR)/util \ + $(DEVICE_OUTDIR)/helpers/memenv -log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR): + mkdir $@ -table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/db: | $(SIMULATOR_OUTDIR) + mkdir $@ -skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/helpers/memenv: | $(SIMULATOR_OUTDIR) + mkdir -p $@ -version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/port: | $(SIMULATOR_OUTDIR) + mkdir $@ -version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/table: | $(SIMULATOR_OUTDIR) + mkdir $@ -write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(SIMULATOR_OUTDIR)/util: | $(SIMULATOR_OUTDIR) + mkdir $@ -$(MEMENVLIBRARY) : $(MEMENVOBJECTS) - rm -f $@ - $(AR) -rs $@ $(MEMENVOBJECTS) +.PHONY: SIMULATOR_OBJDIRS +SIMULATOR_OBJDIRS: \ + $(SIMULATOR_OUTDIR)/db \ + $(SIMULATOR_OUTDIR)/port \ + $(SIMULATOR_OUTDIR)/table \ + $(SIMULATOR_OUTDIR)/util \ + $(SIMULATOR_OUTDIR)/helpers/memenv -memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) - $(CXX) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LIBS) +$(STATIC_ALLOBJS): | STATIC_OBJDIRS +$(DEVICE_ALLOBJS): | DEVICE_OBJDIRS +$(SIMULATOR_ALLOBJS): | SIMULATOR_OBJDIRS +$(SHARED_ALLOBJS): | SHARED_OBJDIRS ifeq ($(PLATFORM), IOS) -# For iOS, create universal object files to be used on both the simulator and +$(DEVICE_OUTDIR)/libleveldb.a: $(DEVICE_LIBOBJECTS) + rm -f $@ + $(AR) -rs $@ $(DEVICE_LIBOBJECTS) + +$(SIMULATOR_OUTDIR)/libleveldb.a: $(SIMULATOR_LIBOBJECTS) + rm -f $@ + $(AR) -rs $@ $(SIMULATOR_LIBOBJECTS) + +$(DEVICE_OUTDIR)/libmemenv.a: $(DEVICE_MEMENVOBJECTS) + rm -f $@ + $(AR) -rs $@ $(DEVICE_MEMENVOBJECTS) + +$(SIMULATOR_OUTDIR)/libmemenv.a: $(SIMULATOR_MEMENVOBJECTS) + rm -f $@ + $(AR) -rs $@ $(SIMULATOR_MEMENVOBJECTS) + +# For iOS, create universal object libraries to be used on both the simulator and # a device. -PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms -SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer -DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer -IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString) -IOSARCH=-arch armv6 -arch armv7 -arch armv7s -arch arm64 - -.cc.o: - mkdir -p ios-x86/$(dir $@) - xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ - mkdir -p ios-arm/$(dir $@) - xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ - xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@ - -.c.o: - mkdir -p ios-x86/$(dir $@) - xcrun -sdk iphonesimulator $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ - mkdir -p ios-arm/$(dir $@) - xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ - xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@ +$(STATIC_OUTDIR)/libleveldb.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a + lipo -create $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a -output $@ +$(STATIC_OUTDIR)/libmemenv.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a + lipo -create $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a -output $@ else -.cc.o: +$(STATIC_OUTDIR)/libleveldb.a:$(STATIC_LIBOBJECTS) + rm -f $@ + $(AR) -rs $@ $(STATIC_LIBOBJECTS) + +$(STATIC_OUTDIR)/libmemenv.a:$(STATIC_MEMENVOBJECTS) + rm -f $@ + $(AR) -rs $@ $(STATIC_MEMENVOBJECTS) +endif + +$(SHARED_MEMENVLIB):$(SHARED_MEMENVOBJECTS) + rm -f $@ + $(AR) -rs $@ $(SHARED_MEMENVOBJECTS) + +$(STATIC_OUTDIR)/db_bench:db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/db_bench_sqlite3:doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) + $(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS) + +$(STATIC_OUTDIR)/db_bench_tree_db:doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) + $(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) + +$(STATIC_OUTDIR)/leveldbutil:db/leveldbutil.cc $(STATIC_LIBOBJECTS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/leveldbutil.cc $(STATIC_LIBOBJECTS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/arena_test:util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/autocompact_test:db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/bloom_test:util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/c_test:$(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/cache_test:util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/coding_test:util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/corruption_test:db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/crc32c_test:util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/db_test:db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/dbformat_test:db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/env_test:util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/fault_injection_test:db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/filename_test:db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/filter_block_test:table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/hash_test:util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/issue178_test:issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/issue200_test:issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/log_test:db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/recovery_test:db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/table_test:table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/skiplist_test:db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/version_edit_test:db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/version_set_test:db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/write_batch_test:db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + +$(STATIC_OUTDIR)/memenv_test:$(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) + $(XCRUN) $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) -o $@ $(LIBS) + +$(SHARED_OUTDIR)/db_bench:$(SHARED_OUTDIR)/db/db_bench.o $(SHARED_LIBS) $(TESTUTIL) + $(XCRUN) $(CXX) $(LDFLAGS) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SHARED_OUTDIR)/db/db_bench.o $(TESTUTIL) $(SHARED_OUTDIR)/$(SHARED_LIB3) -o $@ $(LIBS) + +.PHONY: run-shared +run-shared: $(SHARED_OUTDIR)/db_bench + LD_LIBRARY_PATH=$(SHARED_OUTDIR) $(SHARED_OUTDIR)/db_bench + +$(SIMULATOR_OUTDIR)/%.o: %.cc + xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@ + +$(DEVICE_OUTDIR)/%.o: %.cc + xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) $(DEVICE_CFLAGS) -c $< -o $@ + +$(SIMULATOR_OUTDIR)/%.o: %.c + xcrun -sdk iphonesimulator $(CC) $(CFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@ + +$(DEVICE_OUTDIR)/%.o: %.c + xcrun -sdk iphoneos $(CC) $(CFLAGS) $(DEVICE_CFLAGS) -c $< -o $@ + +$(STATIC_OUTDIR)/%.o: %.cc $(CXX) $(CXXFLAGS) -c $< -o $@ -.c.o: +$(STATIC_OUTDIR)/%.o: %.c $(CC) $(CFLAGS) -c $< -o $@ -endif + +$(SHARED_OUTDIR)/%.o: %.cc + $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@ + +$(SHARED_OUTDIR)/%.o: %.c + $(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@ diff --git a/src/leveldb/README b/src/leveldb/README deleted file mode 100644 index 3618adeeed..0000000000 --- a/src/leveldb/README +++ /dev/null @@ -1,51 +0,0 @@ -leveldb: A key-value store -Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) - -The code under this directory implements a system for maintaining a -persistent key/value store. - -See doc/index.html for more explanation. -See doc/impl.html for a brief overview of the implementation. - -The public interface is in include/*.h. Callers should not include or -rely on the details of any other header files in this package. Those -internal APIs may be changed without warning. - -Guide to header files: - -include/db.h - Main interface to the DB: Start here - -include/options.h - Control over the behavior of an entire database, and also - control over the behavior of individual reads and writes. - -include/comparator.h - Abstraction for user-specified comparison function. If you want - just bytewise comparison of keys, you can use the default comparator, - but clients can write their own comparator implementations if they - want custom ordering (e.g. to handle different character - encodings, etc.) - -include/iterator.h - Interface for iterating over data. You can get an iterator - from a DB object. - -include/write_batch.h - Interface for atomically applying multiple updates to a database. - -include/slice.h - A simple module for maintaining a pointer and a length into some - other byte array. - -include/status.h - Status is returned from many of the public interfaces and is used - to report success and various kinds of errors. - -include/env.h - Abstraction of the OS environment. A posix implementation of - this interface is in util/env_posix.cc - -include/table.h -include/table_builder.h - Lower-level modules that most clients probably won't use directly diff --git a/src/leveldb/README.md b/src/leveldb/README.md index 480affb5ca..c75b185e0e 100644 --- a/src/leveldb/README.md +++ b/src/leveldb/README.md @@ -1,5 +1,7 @@ **LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.** +[![Build Status](https://travis-ci.org/google/leveldb.svg?branch=master)](https://travis-ci.org/google/leveldb) + Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) # Features @@ -10,9 +12,11 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) * Multiple changes can be made in one atomic batch. * Users can create a transient snapshot to get a consistent view of data. * Forward and backward iteration is supported over the data. - * Data is automatically compressed using the [Snappy compression library](http://code.google.com/p/snappy). + * Data is automatically compressed using the [Snappy compression library](http://google.github.io/snappy/). * External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions. - * [Detailed documentation](http://htmlpreview.github.io/?https://github.com/google/leveldb/blob/master/doc/index.html) about how to use the library is included with the source code. + +# Documentation + [LevelDB library documentation](https://rawgit.com/google/leveldb/master/doc/index.html) is online and bundled with the source code. # Limitations @@ -20,6 +24,37 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) * Only a single process (possibly multi-threaded) can access a particular database at a time. * There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library. +# Contributing to the leveldb Project +The leveldb project welcomes contributions. leveldb's primary goal is to be +a reliable and fast key/value store. Changes that are in line with the +features/limitations outlined above, and meet the requirements below, +will be considered. + +Contribution requirements: + +1. **POSIX only**. We _generally_ will only accept changes that are both + compiled, and tested on a POSIX platform - usually Linux. Very small + changes will sometimes be accepted, but consider that more of an + exception than the rule. + +2. **Stable API**. We strive very hard to maintain a stable API. Changes that + require changes for projects using leveldb _might_ be rejected without + sufficient benefit to the project. + +3. **Tests**: All changes must be accompanied by a new (or changed) test, or + a sufficient explanation as to why a new (or changed) test is not required. + +## Submitting a Pull Request +Before any pull request will be accepted the author must first sign a +Contributor License Agreement (CLA) at https://cla.developers.google.com/. + +In order to keep the commit timeline linear +[squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits) +your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase) +on google/leveldb/master. This keeps the commit timeline linear and more easily sync'ed +with the internal repository at Google. More information at GitHub's +[About Git rebase](https://help.github.com/articles/about-git-rebase/) page. + # Performance Here is a performance report (with explanations) from the run of the diff --git a/src/leveldb/build_detect_platform b/src/leveldb/build_detect_platform index a1101c1bda..d7edab1d87 100755 --- a/src/leveldb/build_detect_platform +++ b/src/leveldb/build_detect_platform @@ -175,7 +175,7 @@ DIRS="$PREFIX/db $PREFIX/util $PREFIX/table" set -f # temporarily disable globbing so that our patterns aren't expanded PRUNE_TEST="-name *test*.cc -prune" PRUNE_BENCH="-name *_bench.cc -prune" -PRUNE_TOOL="-name leveldb_main.cc -prune" +PRUNE_TOOL="-name leveldbutil.cc -prune" PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "` set +f # re-enable globbing diff --git a/src/leveldb/db/corruption_test.cc b/src/leveldb/db/corruption_test.cc index 96afc68913..37a484d25f 100644 --- a/src/leveldb/db/corruption_test.cc +++ b/src/leveldb/db/corruption_test.cc @@ -36,7 +36,7 @@ class CorruptionTest { tiny_cache_ = NewLRUCache(100); options_.env = &env_; options_.block_cache = tiny_cache_; - dbname_ = test::TmpDir() + "/db_test"; + dbname_ = test::TmpDir() + "/corruption_test"; DestroyDB(dbname_, options_); db_ = NULL; diff --git a/src/leveldb/db/db_bench.cc b/src/leveldb/db/db_bench.cc index 705a170aae..7a0f5e08cd 100644 --- a/src/leveldb/db/db_bench.cc +++ b/src/leveldb/db/db_bench.cc @@ -33,6 +33,7 @@ // readmissing -- read N missing keys in random order // readhot -- read N times in random order from 1% section of DB // seekrandom -- N random seeks +// open -- cost of opening a DB // crc32c -- repeated crc32c of 4K of data // acquireload -- load N*1000 times // Meta operations: @@ -99,6 +100,9 @@ static int FLAGS_bloom_bits = -1; // benchmark will fail. static bool FLAGS_use_existing_db = false; +// If true, reuse existing log/MANIFEST files when re-opening a database. +static bool FLAGS_reuse_logs = false; + // Use the db with the following name. static const char* FLAGS_db = NULL; @@ -138,6 +142,7 @@ class RandomGenerator { } }; +#if defined(__linux) static Slice TrimSpace(Slice s) { size_t start = 0; while (start < s.size() && isspace(s[start])) { @@ -149,6 +154,7 @@ static Slice TrimSpace(Slice s) { } return Slice(s.data() + start, limit - start); } +#endif static void AppendWithSpace(std::string* str, Slice msg) { if (msg.empty()) return; @@ -442,7 +448,11 @@ class Benchmark { bool fresh_db = false; int num_threads = FLAGS_threads; - if (name == Slice("fillseq")) { + if (name == Slice("open")) { + method = &Benchmark::OpenBench; + num_ /= 10000; + if (num_ < 1) num_ = 1; + } else if (name == Slice("fillseq")) { fresh_db = true; method = &Benchmark::WriteSeq; } else if (name == Slice("fillbatch")) { @@ -695,6 +705,7 @@ class Benchmark { options.write_buffer_size = FLAGS_write_buffer_size; options.max_open_files = FLAGS_open_files; options.filter_policy = filter_policy_; + options.reuse_logs = FLAGS_reuse_logs; Status s = DB::Open(options, FLAGS_db, &db_); if (!s.ok()) { fprintf(stderr, "open error: %s\n", s.ToString().c_str()); @@ -702,6 +713,14 @@ class Benchmark { } } + void OpenBench(ThreadState* thread) { + for (int i = 0; i < num_; i++) { + delete db_; + Open(); + thread->stats.FinishedSingleOp(); + } + } + void WriteSeq(ThreadState* thread) { DoWrite(thread, true); } @@ -941,6 +960,9 @@ int main(int argc, char** argv) { } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && (n == 0 || n == 1)) { FLAGS_use_existing_db = n; + } else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_reuse_logs = n; } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { FLAGS_num = n; } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc index 49b95953b4..60f4e66e55 100644 --- a/src/leveldb/db/db_impl.cc +++ b/src/leveldb/db/db_impl.cc @@ -125,7 +125,7 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) db_lock_(NULL), shutting_down_(NULL), bg_cv_(&mutex_), - mem_(new MemTable(internal_comparator_)), + mem_(NULL), imm_(NULL), logfile_(NULL), logfile_number_(0), @@ -134,7 +134,6 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) tmp_batch_(new WriteBatch), bg_compaction_scheduled_(false), manual_compaction_(NULL) { - mem_->Ref(); has_imm_.Release_Store(NULL); // Reserve ten files or so for other uses and give the rest to TableCache. @@ -271,7 +270,7 @@ void DBImpl::DeleteObsoleteFiles() { } } -Status DBImpl::Recover(VersionEdit* edit) { +Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { mutex_.AssertHeld(); // Ignore error from CreateDir since the creation of the DB is @@ -301,66 +300,69 @@ Status DBImpl::Recover(VersionEdit* edit) { } } - s = versions_->Recover(); - if (s.ok()) { - SequenceNumber max_sequence(0); - - // Recover from all newer log files than the ones named in the - // descriptor (new log files may have been added by the previous - // incarnation without registering them in the descriptor). - // - // Note that PrevLogNumber() is no longer used, but we pay - // attention to it in case we are recovering a database - // produced by an older version of leveldb. - const uint64_t min_log = versions_->LogNumber(); - const uint64_t prev_log = versions_->PrevLogNumber(); - std::vector<std::string> filenames; - s = env_->GetChildren(dbname_, &filenames); + s = versions_->Recover(save_manifest); + if (!s.ok()) { + return s; + } + SequenceNumber max_sequence(0); + + // Recover from all newer log files than the ones named in the + // descriptor (new log files may have been added by the previous + // incarnation without registering them in the descriptor). + // + // Note that PrevLogNumber() is no longer used, but we pay + // attention to it in case we are recovering a database + // produced by an older version of leveldb. + const uint64_t min_log = versions_->LogNumber(); + const uint64_t prev_log = versions_->PrevLogNumber(); + std::vector<std::string> filenames; + s = env_->GetChildren(dbname_, &filenames); + if (!s.ok()) { + return s; + } + std::set<uint64_t> expected; + versions_->AddLiveFiles(&expected); + uint64_t number; + FileType type; + std::vector<uint64_t> logs; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type)) { + expected.erase(number); + if (type == kLogFile && ((number >= min_log) || (number == prev_log))) + logs.push_back(number); + } + } + if (!expected.empty()) { + char buf[50]; + snprintf(buf, sizeof(buf), "%d missing files; e.g.", + static_cast<int>(expected.size())); + return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); + } + + // Recover in the order in which the logs were generated + std::sort(logs.begin(), logs.end()); + for (size_t i = 0; i < logs.size(); i++) { + s = RecoverLogFile(logs[i], (i == logs.size() - 1), save_manifest, edit, + &max_sequence); if (!s.ok()) { return s; } - std::set<uint64_t> expected; - versions_->AddLiveFiles(&expected); - uint64_t number; - FileType type; - std::vector<uint64_t> logs; - for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type)) { - expected.erase(number); - if (type == kLogFile && ((number >= min_log) || (number == prev_log))) - logs.push_back(number); - } - } - if (!expected.empty()) { - char buf[50]; - snprintf(buf, sizeof(buf), "%d missing files; e.g.", - static_cast<int>(expected.size())); - return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); - } - - // Recover in the order in which the logs were generated - std::sort(logs.begin(), logs.end()); - for (size_t i = 0; i < logs.size(); i++) { - s = RecoverLogFile(logs[i], edit, &max_sequence); - // The previous incarnation may not have written any MANIFEST - // records after allocating this log number. So we manually - // update the file number allocation counter in VersionSet. - versions_->MarkFileNumberUsed(logs[i]); - } + // The previous incarnation may not have written any MANIFEST + // records after allocating this log number. So we manually + // update the file number allocation counter in VersionSet. + versions_->MarkFileNumberUsed(logs[i]); + } - if (s.ok()) { - if (versions_->LastSequence() < max_sequence) { - versions_->SetLastSequence(max_sequence); - } - } + if (versions_->LastSequence() < max_sequence) { + versions_->SetLastSequence(max_sequence); } - return s; + return Status::OK(); } -Status DBImpl::RecoverLogFile(uint64_t log_number, - VersionEdit* edit, +Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, + bool* save_manifest, VersionEdit* edit, SequenceNumber* max_sequence) { struct LogReporter : public log::Reader::Reporter { Env* env; @@ -405,6 +407,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, std::string scratch; Slice record; WriteBatch batch; + int compactions = 0; MemTable* mem = NULL; while (reader.ReadRecord(&record, &scratch) && status.ok()) { @@ -432,25 +435,52 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, } if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) { + compactions++; + *save_manifest = true; status = WriteLevel0Table(mem, edit, NULL); + mem->Unref(); + mem = NULL; if (!status.ok()) { // Reflect errors immediately so that conditions like full // file-systems cause the DB::Open() to fail. break; } - mem->Unref(); - mem = NULL; } } - if (status.ok() && mem != NULL) { - status = WriteLevel0Table(mem, edit, NULL); - // Reflect errors immediately so that conditions like full - // file-systems cause the DB::Open() to fail. + delete file; + + // See if we should keep reusing the last log file. + if (status.ok() && options_.reuse_logs && last_log && compactions == 0) { + assert(logfile_ == NULL); + assert(log_ == NULL); + assert(mem_ == NULL); + uint64_t lfile_size; + if (env_->GetFileSize(fname, &lfile_size).ok() && + env_->NewAppendableFile(fname, &logfile_).ok()) { + Log(options_.info_log, "Reusing old log %s \n", fname.c_str()); + log_ = new log::Writer(logfile_, lfile_size); + logfile_number_ = log_number; + if (mem != NULL) { + mem_ = mem; + mem = NULL; + } else { + // mem can be NULL if lognum exists but was empty. + mem_ = new MemTable(internal_comparator_); + mem_->Ref(); + } + } + } + + if (mem != NULL) { + // mem did not get reused; compact it. + if (status.ok()) { + *save_manifest = true; + status = WriteLevel0Table(mem, edit, NULL); + } + mem->Unref(); } - if (mem != NULL) mem->Unref(); - delete file; return status; } @@ -821,8 +851,9 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, delete iter; if (s.ok()) { Log(options_.info_log, - "Generated table #%llu: %lld keys, %lld bytes", + "Generated table #%llu@%d: %lld keys, %lld bytes", (unsigned long long) output_number, + compact->compaction->level(), (unsigned long long) current_entries, (unsigned long long) current_bytes); } @@ -1395,6 +1426,19 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) { } else if (in == "sstables") { *value = versions_->current()->DebugString(); return true; + } else if (in == "approximate-memory-usage") { + size_t total_usage = options_.block_cache->TotalCharge(); + if (mem_) { + total_usage += mem_->ApproximateMemoryUsage(); + } + if (imm_) { + total_usage += imm_->ApproximateMemoryUsage(); + } + char buf[50]; + snprintf(buf, sizeof(buf), "%llu", + static_cast<unsigned long long>(total_usage)); + value->append(buf); + return true; } return false; @@ -1449,8 +1493,11 @@ Status DB::Open(const Options& options, const std::string& dbname, DBImpl* impl = new DBImpl(options, dbname); impl->mutex_.Lock(); VersionEdit edit; - Status s = impl->Recover(&edit); // Handles create_if_missing, error_if_exists - if (s.ok()) { + // Recover handles create_if_missing, error_if_exists + bool save_manifest = false; + Status s = impl->Recover(&edit, &save_manifest); + if (s.ok() && impl->mem_ == NULL) { + // Create new log and a corresponding memtable. uint64_t new_log_number = impl->versions_->NewFileNumber(); WritableFile* lfile; s = options.env->NewWritableFile(LogFileName(dbname, new_log_number), @@ -1460,15 +1507,22 @@ Status DB::Open(const Options& options, const std::string& dbname, impl->logfile_ = lfile; impl->logfile_number_ = new_log_number; impl->log_ = new log::Writer(lfile); - s = impl->versions_->LogAndApply(&edit, &impl->mutex_); - } - if (s.ok()) { - impl->DeleteObsoleteFiles(); - impl->MaybeScheduleCompaction(); + impl->mem_ = new MemTable(impl->internal_comparator_); + impl->mem_->Ref(); } } + if (s.ok() && save_manifest) { + edit.SetPrevLogNumber(0); // No older logs needed after recovery. + edit.SetLogNumber(impl->logfile_number_); + s = impl->versions_->LogAndApply(&edit, &impl->mutex_); + } + if (s.ok()) { + impl->DeleteObsoleteFiles(); + impl->MaybeScheduleCompaction(); + } impl->mutex_.Unlock(); if (s.ok()) { + assert(impl->mem_ != NULL); *dbptr = impl; } else { delete impl; diff --git a/src/leveldb/db/db_impl.h b/src/leveldb/db/db_impl.h index cfc998164a..8ff323e728 100644 --- a/src/leveldb/db/db_impl.h +++ b/src/leveldb/db/db_impl.h @@ -78,7 +78,8 @@ class DBImpl : public DB { // Recover the descriptor from persistent storage. May do a significant // amount of work to recover recently logged updates. Any changes to // be made to the descriptor are added to *edit. - Status Recover(VersionEdit* edit) EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Status Recover(VersionEdit* edit, bool* save_manifest) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); void MaybeIgnoreError(Status* s) const; @@ -90,9 +91,8 @@ class DBImpl : public DB { // Errors are recorded in bg_error_. void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_); - Status RecoverLogFile(uint64_t log_number, - VersionEdit* edit, - SequenceNumber* max_sequence) + Status RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest, + VersionEdit* edit, SequenceNumber* max_sequence) EXCLUSIVE_LOCKS_REQUIRED(mutex_); Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base) diff --git a/src/leveldb/db/db_test.cc b/src/leveldb/db/db_test.cc index 0fed9137d5..a0b08bc19c 100644 --- a/src/leveldb/db/db_test.cc +++ b/src/leveldb/db/db_test.cc @@ -193,6 +193,7 @@ class DBTest { // Sequence of option configurations to try enum OptionConfig { kDefault, + kReuse, kFilter, kUncompressed, kEnd @@ -237,7 +238,11 @@ class DBTest { // Return the current option configuration. Options CurrentOptions() { Options options; + options.reuse_logs = false; switch (option_config_) { + case kReuse: + options.reuse_logs = true; + break; case kFilter: options.filter_policy = filter_policy_; break; @@ -558,6 +563,17 @@ TEST(DBTest, GetFromVersions) { } while (ChangeOptions()); } +TEST(DBTest, GetMemUsage) { + do { + ASSERT_OK(Put("foo", "v1")); + std::string val; + ASSERT_TRUE(db_->GetProperty("leveldb.approximate-memory-usage", &val)); + int mem_usage = atoi(val.c_str()); + ASSERT_GT(mem_usage, 0); + ASSERT_LT(mem_usage, 5*1024*1024); + } while (ChangeOptions()); +} + TEST(DBTest, GetSnapshot) { do { // Try with both a short key and a long key @@ -1080,6 +1096,14 @@ TEST(DBTest, ApproximateSizes) { // 0 because GetApproximateSizes() does not account for memtable space ASSERT_TRUE(Between(Size("", Key(50)), 0, 0)); + if (options.reuse_logs) { + // Recovery will reuse memtable, and GetApproximateSizes() does not + // account for memtable usage; + Reopen(&options); + ASSERT_TRUE(Between(Size("", Key(50)), 0, 0)); + continue; + } + // Check sizes across recovery by reopening a few times for (int run = 0; run < 3; run++) { Reopen(&options); @@ -1123,6 +1147,11 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) { ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000))); ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000))); + if (options.reuse_logs) { + // Need to force a memtable compaction since recovery does not do so. + ASSERT_OK(dbfull()->TEST_CompactMemTable()); + } + // Check sizes across recovery by reopening a few times for (int run = 0; run < 3; run++) { Reopen(&options); @@ -2084,7 +2113,8 @@ void BM_LogAndApply(int iters, int num_base_files) { InternalKeyComparator cmp(BytewiseComparator()); Options options; VersionSet vset(dbname, &options, NULL, &cmp); - ASSERT_OK(vset.Recover()); + bool save_manifest; + ASSERT_OK(vset.Recover(&save_manifest)); VersionEdit vbase; uint64_t fnum = 1; for (int i = 0; i < num_base_files; i++) { diff --git a/src/leveldb/db/fault_injection_test.cc b/src/leveldb/db/fault_injection_test.cc new file mode 100644 index 0000000000..875dfe81ee --- /dev/null +++ b/src/leveldb/db/fault_injection_test.cc @@ -0,0 +1,554 @@ +// Copyright 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// This test uses a custom Env to keep track of the state of a filesystem as of +// the last "sync". It then checks for data loss errors by purposely dropping +// file data (or entire files) not protected by a "sync". + +#include "leveldb/db.h" + +#include <map> +#include <set> +#include "db/db_impl.h" +#include "db/filename.h" +#include "db/log_format.h" +#include "db/version_set.h" +#include "leveldb/cache.h" +#include "leveldb/env.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" +#include "util/mutexlock.h" +#include "util/testharness.h" +#include "util/testutil.h" + +namespace leveldb { + +static const int kValueSize = 1000; +static const int kMaxNumValues = 2000; +static const size_t kNumIterations = 3; + +class FaultInjectionTestEnv; + +namespace { + +// Assume a filename, and not a directory name like "/foo/bar/" +static std::string GetDirName(const std::string filename) { + size_t found = filename.find_last_of("/\\"); + if (found == std::string::npos) { + return ""; + } else { + return filename.substr(0, found); + } +} + +Status SyncDir(const std::string& dir) { + // As this is a test it isn't required to *actually* sync this directory. + return Status::OK(); +} + +// A basic file truncation function suitable for this test. +Status Truncate(const std::string& filename, uint64_t length) { + leveldb::Env* env = leveldb::Env::Default(); + + SequentialFile* orig_file; + Status s = env->NewSequentialFile(filename, &orig_file); + if (!s.ok()) + return s; + + char* scratch = new char[length]; + leveldb::Slice result; + s = orig_file->Read(length, &result, scratch); + delete orig_file; + if (s.ok()) { + std::string tmp_name = GetDirName(filename) + "/truncate.tmp"; + WritableFile* tmp_file; + s = env->NewWritableFile(tmp_name, &tmp_file); + if (s.ok()) { + s = tmp_file->Append(result); + delete tmp_file; + if (s.ok()) { + s = env->RenameFile(tmp_name, filename); + } else { + env->DeleteFile(tmp_name); + } + } + } + + delete[] scratch; + + return s; +} + +struct FileState { + std::string filename_; + ssize_t pos_; + ssize_t pos_at_last_sync_; + ssize_t pos_at_last_flush_; + + FileState(const std::string& filename) + : filename_(filename), + pos_(-1), + pos_at_last_sync_(-1), + pos_at_last_flush_(-1) { } + + FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} + + bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; } + + Status DropUnsyncedData() const; +}; + +} // anonymous namespace + +// A wrapper around WritableFile which informs another Env whenever this file +// is written to or sync'ed. +class TestWritableFile : public WritableFile { + public: + TestWritableFile(const FileState& state, + WritableFile* f, + FaultInjectionTestEnv* env); + virtual ~TestWritableFile(); + virtual Status Append(const Slice& data); + virtual Status Close(); + virtual Status Flush(); + virtual Status Sync(); + + private: + FileState state_; + WritableFile* target_; + bool writable_file_opened_; + FaultInjectionTestEnv* env_; + + Status SyncParent(); +}; + +class FaultInjectionTestEnv : public EnvWrapper { + public: + FaultInjectionTestEnv() : EnvWrapper(Env::Default()), filesystem_active_(true) {} + virtual ~FaultInjectionTestEnv() { } + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result); + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result); + virtual Status DeleteFile(const std::string& f); + virtual Status RenameFile(const std::string& s, const std::string& t); + + void WritableFileClosed(const FileState& state); + Status DropUnsyncedFileData(); + Status DeleteFilesCreatedAfterLastDirSync(); + void DirWasSynced(); + bool IsFileCreatedSinceLastDirSync(const std::string& filename); + void ResetState(); + void UntrackFile(const std::string& f); + // Setting the filesystem to inactive is the test equivalent to simulating a + // system reset. Setting to inactive will freeze our saved filesystem state so + // that it will stop being recorded. It can then be reset back to the state at + // the time of the reset. + bool IsFilesystemActive() const { return filesystem_active_; } + void SetFilesystemActive(bool active) { filesystem_active_ = active; } + + private: + port::Mutex mutex_; + std::map<std::string, FileState> db_file_state_; + std::set<std::string> new_files_since_last_dir_sync_; + bool filesystem_active_; // Record flushes, syncs, writes +}; + +TestWritableFile::TestWritableFile(const FileState& state, + WritableFile* f, + FaultInjectionTestEnv* env) + : state_(state), + target_(f), + writable_file_opened_(true), + env_(env) { + assert(f != NULL); +} + +TestWritableFile::~TestWritableFile() { + if (writable_file_opened_) { + Close(); + } + delete target_; +} + +Status TestWritableFile::Append(const Slice& data) { + Status s = target_->Append(data); + if (s.ok() && env_->IsFilesystemActive()) { + state_.pos_ += data.size(); + } + return s; +} + +Status TestWritableFile::Close() { + writable_file_opened_ = false; + Status s = target_->Close(); + if (s.ok()) { + env_->WritableFileClosed(state_); + } + return s; +} + +Status TestWritableFile::Flush() { + Status s = target_->Flush(); + if (s.ok() && env_->IsFilesystemActive()) { + state_.pos_at_last_flush_ = state_.pos_; + } + return s; +} + +Status TestWritableFile::SyncParent() { + Status s = SyncDir(GetDirName(state_.filename_)); + if (s.ok()) { + env_->DirWasSynced(); + } + return s; +} + +Status TestWritableFile::Sync() { + if (!env_->IsFilesystemActive()) { + return Status::OK(); + } + // Ensure new files referred to by the manifest are in the filesystem. + Status s = target_->Sync(); + if (s.ok()) { + state_.pos_at_last_sync_ = state_.pos_; + } + if (env_->IsFileCreatedSinceLastDirSync(state_.filename_)) { + Status ps = SyncParent(); + if (s.ok() && !ps.ok()) { + s = ps; + } + } + return s; +} + +Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname, + WritableFile** result) { + WritableFile* actual_writable_file; + Status s = target()->NewWritableFile(fname, &actual_writable_file); + if (s.ok()) { + FileState state(fname); + state.pos_ = 0; + *result = new TestWritableFile(state, actual_writable_file, this); + // NewWritableFile doesn't append to files, so if the same file is + // opened again then it will be truncated - so forget our saved + // state. + UntrackFile(fname); + MutexLock l(&mutex_); + new_files_since_last_dir_sync_.insert(fname); + } + return s; +} + +Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname, + WritableFile** result) { + WritableFile* actual_writable_file; + Status s = target()->NewAppendableFile(fname, &actual_writable_file); + if (s.ok()) { + FileState state(fname); + state.pos_ = 0; + { + MutexLock l(&mutex_); + if (db_file_state_.count(fname) == 0) { + new_files_since_last_dir_sync_.insert(fname); + } else { + state = db_file_state_[fname]; + } + } + *result = new TestWritableFile(state, actual_writable_file, this); + } + return s; +} + +Status FaultInjectionTestEnv::DropUnsyncedFileData() { + Status s; + MutexLock l(&mutex_); + for (std::map<std::string, FileState>::const_iterator it = + db_file_state_.begin(); + s.ok() && it != db_file_state_.end(); ++it) { + const FileState& state = it->second; + if (!state.IsFullySynced()) { + s = state.DropUnsyncedData(); + } + } + return s; +} + +void FaultInjectionTestEnv::DirWasSynced() { + MutexLock l(&mutex_); + new_files_since_last_dir_sync_.clear(); +} + +bool FaultInjectionTestEnv::IsFileCreatedSinceLastDirSync( + const std::string& filename) { + MutexLock l(&mutex_); + return new_files_since_last_dir_sync_.find(filename) != + new_files_since_last_dir_sync_.end(); +} + +void FaultInjectionTestEnv::UntrackFile(const std::string& f) { + MutexLock l(&mutex_); + db_file_state_.erase(f); + new_files_since_last_dir_sync_.erase(f); +} + +Status FaultInjectionTestEnv::DeleteFile(const std::string& f) { + Status s = EnvWrapper::DeleteFile(f); + ASSERT_OK(s); + if (s.ok()) { + UntrackFile(f); + } + return s; +} + +Status FaultInjectionTestEnv::RenameFile(const std::string& s, + const std::string& t) { + Status ret = EnvWrapper::RenameFile(s, t); + + if (ret.ok()) { + MutexLock l(&mutex_); + if (db_file_state_.find(s) != db_file_state_.end()) { + db_file_state_[t] = db_file_state_[s]; + db_file_state_.erase(s); + } + + if (new_files_since_last_dir_sync_.erase(s) != 0) { + assert(new_files_since_last_dir_sync_.find(t) == + new_files_since_last_dir_sync_.end()); + new_files_since_last_dir_sync_.insert(t); + } + } + + return ret; +} + +void FaultInjectionTestEnv::ResetState() { + // Since we are not destroying the database, the existing files + // should keep their recorded synced/flushed state. Therefore + // we do not reset db_file_state_ and new_files_since_last_dir_sync_. + MutexLock l(&mutex_); + SetFilesystemActive(true); +} + +Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() { + // Because DeleteFile access this container make a copy to avoid deadlock + mutex_.Lock(); + std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(), + new_files_since_last_dir_sync_.end()); + mutex_.Unlock(); + Status s; + std::set<std::string>::const_iterator it; + for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) { + s = DeleteFile(*it); + } + return s; +} + +void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { + MutexLock l(&mutex_); + db_file_state_[state.filename_] = state; +} + +Status FileState::DropUnsyncedData() const { + ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; + return Truncate(filename_, sync_pos); +} + +class FaultInjectionTest { + public: + enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR }; + enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES }; + + FaultInjectionTestEnv* env_; + std::string dbname_; + Cache* tiny_cache_; + Options options_; + DB* db_; + + FaultInjectionTest() + : env_(new FaultInjectionTestEnv), + tiny_cache_(NewLRUCache(100)), + db_(NULL) { + dbname_ = test::TmpDir() + "/fault_test"; + DestroyDB(dbname_, Options()); // Destroy any db from earlier run + options_.reuse_logs = true; + options_.env = env_; + options_.paranoid_checks = true; + options_.block_cache = tiny_cache_; + options_.create_if_missing = true; + } + + ~FaultInjectionTest() { + CloseDB(); + DestroyDB(dbname_, Options()); + delete tiny_cache_; + delete env_; + } + + void ReuseLogs(bool reuse) { + options_.reuse_logs = reuse; + } + + void Build(int start_idx, int num_vals) { + std::string key_space, value_space; + WriteBatch batch; + for (int i = start_idx; i < start_idx + num_vals; i++) { + Slice key = Key(i, &key_space); + batch.Clear(); + batch.Put(key, Value(i, &value_space)); + WriteOptions options; + ASSERT_OK(db_->Write(options, &batch)); + } + } + + Status ReadValue(int i, std::string* val) const { + std::string key_space, value_space; + Slice key = Key(i, &key_space); + Value(i, &value_space); + ReadOptions options; + return db_->Get(options, key, val); + } + + Status Verify(int start_idx, int num_vals, + ExpectedVerifResult expected) const { + std::string val; + std::string value_space; + Status s; + for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) { + Value(i, &value_space); + s = ReadValue(i, &val); + if (expected == VAL_EXPECT_NO_ERROR) { + if (s.ok()) { + ASSERT_EQ(value_space, val); + } + } else if (s.ok()) { + fprintf(stderr, "Expected an error at %d, but was OK\n", i); + s = Status::IOError(dbname_, "Expected value error:"); + } else { + s = Status::OK(); // An expected error + } + } + return s; + } + + // Return the ith key + Slice Key(int i, std::string* storage) const { + char buf[100]; + snprintf(buf, sizeof(buf), "%016d", i); + storage->assign(buf, strlen(buf)); + return Slice(*storage); + } + + // Return the value to associate with the specified key + Slice Value(int k, std::string* storage) const { + Random r(k); + return test::RandomString(&r, kValueSize, storage); + } + + Status OpenDB() { + delete db_; + db_ = NULL; + env_->ResetState(); + return DB::Open(options_, dbname_, &db_); + } + + void CloseDB() { + delete db_; + db_ = NULL; + } + + void DeleteAllData() { + Iterator* iter = db_->NewIterator(ReadOptions()); + WriteOptions options; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(db_->Delete(WriteOptions(), iter->key())); + } + + delete iter; + } + + void ResetDBState(ResetMethod reset_method) { + switch (reset_method) { + case RESET_DROP_UNSYNCED_DATA: + ASSERT_OK(env_->DropUnsyncedFileData()); + break; + case RESET_DELETE_UNSYNCED_FILES: + ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync()); + break; + default: + assert(false); + } + } + + void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) { + DeleteAllData(); + Build(0, num_pre_sync); + db_->CompactRange(NULL, NULL); + Build(num_pre_sync, num_post_sync); + } + + void PartialCompactTestReopenWithFault(ResetMethod reset_method, + int num_pre_sync, + int num_post_sync) { + env_->SetFilesystemActive(false); + CloseDB(); + ResetDBState(reset_method); + ASSERT_OK(OpenDB()); + ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR)); + ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR)); + } + + void NoWriteTestPreFault() { + } + + void NoWriteTestReopenWithFault(ResetMethod reset_method) { + CloseDB(); + ResetDBState(reset_method); + ASSERT_OK(OpenDB()); + } + + void DoTest() { + Random rnd(0); + ASSERT_OK(OpenDB()); + for (size_t idx = 0; idx < kNumIterations; idx++) { + int num_pre_sync = rnd.Uniform(kMaxNumValues); + int num_post_sync = rnd.Uniform(kMaxNumValues); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, + num_pre_sync, + num_post_sync); + + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + // No new files created so we expect all values since no files will be + // dropped. + PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES, + num_pre_sync + num_post_sync, + 0); + + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES); + } + } +}; + +TEST(FaultInjectionTest, FaultTestNoLogReuse) { + ReuseLogs(false); + DoTest(); +} + +TEST(FaultInjectionTest, FaultTestWithLogReuse) { + ReuseLogs(true); + DoTest(); +} + +} // namespace leveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/src/leveldb/db/leveldb_main.cc b/src/leveldb/db/leveldbutil.cc index 9f4b7dd70c..9f4b7dd70c 100644 --- a/src/leveldb/db/leveldb_main.cc +++ b/src/leveldb/db/leveldbutil.cc diff --git a/src/leveldb/db/log_reader.cc b/src/leveldb/db/log_reader.cc index e44b66c85b..a6d304545d 100644 --- a/src/leveldb/db/log_reader.cc +++ b/src/leveldb/db/log_reader.cc @@ -25,7 +25,8 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, eof_(false), last_record_offset_(0), end_of_buffer_offset_(0), - initial_offset_(initial_offset) { + initial_offset_(initial_offset), + resyncing_(initial_offset > 0) { } Reader::~Reader() { @@ -72,8 +73,25 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { Slice fragment; while (true) { - uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size(); const unsigned int record_type = ReadPhysicalRecord(&fragment); + + // ReadPhysicalRecord may have only had an empty trailer remaining in its + // internal buffer. Calculate the offset of the next physical record now + // that it has returned, properly accounting for its header size. + uint64_t physical_record_offset = + end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size(); + + if (resyncing_) { + if (record_type == kMiddleType) { + continue; + } else if (record_type == kLastType) { + resyncing_ = false; + continue; + } else { + resyncing_ = false; + } + } + switch (record_type) { case kFullType: if (in_fragmented_record) { diff --git a/src/leveldb/db/log_reader.h b/src/leveldb/db/log_reader.h index 6aff791716..8389d61f8f 100644 --- a/src/leveldb/db/log_reader.h +++ b/src/leveldb/db/log_reader.h @@ -73,6 +73,11 @@ class Reader { // Offset at which to start looking for the first record to return uint64_t const initial_offset_; + // True if we are resynchronizing after a seek (initial_offset_ > 0). In + // particular, a run of kMiddleType and kLastType records can be silently + // skipped in this mode + bool resyncing_; + // Extend record types with the following special values enum { kEof = kMaxRecordType + 1, diff --git a/src/leveldb/db/log_test.cc b/src/leveldb/db/log_test.cc index dcf0562652..48a5928657 100644 --- a/src/leveldb/db/log_test.cc +++ b/src/leveldb/db/log_test.cc @@ -79,7 +79,7 @@ class LogTest { virtual Status Skip(uint64_t n) { if (n > contents_.size()) { contents_.clear(); - return Status::NotFound("in-memory file skipepd past end"); + return Status::NotFound("in-memory file skipped past end"); } contents_.remove_prefix(n); @@ -104,23 +104,34 @@ class LogTest { StringSource source_; ReportCollector report_; bool reading_; - Writer writer_; - Reader reader_; + Writer* writer_; + Reader* reader_; // Record metadata for testing initial offset functionality static size_t initial_offset_record_sizes_[]; static uint64_t initial_offset_last_record_offsets_[]; + static int num_initial_offset_records_; public: LogTest() : reading_(false), - writer_(&dest_), - reader_(&source_, &report_, true/*checksum*/, - 0/*initial_offset*/) { + writer_(new Writer(&dest_)), + reader_(new Reader(&source_, &report_, true/*checksum*/, + 0/*initial_offset*/)) { + } + + ~LogTest() { + delete writer_; + delete reader_; + } + + void ReopenForAppend() { + delete writer_; + writer_ = new Writer(&dest_, dest_.contents_.size()); } void Write(const std::string& msg) { ASSERT_TRUE(!reading_) << "Write() after starting to read"; - writer_.AddRecord(Slice(msg)); + writer_->AddRecord(Slice(msg)); } size_t WrittenBytes() const { @@ -134,7 +145,7 @@ class LogTest { } std::string scratch; Slice record; - if (reader_.ReadRecord(&record, &scratch)) { + if (reader_->ReadRecord(&record, &scratch)) { return record.ToString(); } else { return "EOF"; @@ -182,13 +193,18 @@ class LogTest { } void WriteInitialOffsetLog() { - for (int i = 0; i < 4; i++) { + for (int i = 0; i < num_initial_offset_records_; i++) { std::string record(initial_offset_record_sizes_[i], static_cast<char>('a' + i)); Write(record); } } + void StartReadingAt(uint64_t initial_offset) { + delete reader_; + reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset); + } + void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) { WriteInitialOffsetLog(); reading_ = true; @@ -208,32 +224,48 @@ class LogTest { source_.contents_ = Slice(dest_.contents_); Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, initial_offset); - Slice record; - std::string scratch; - ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch)); - ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset], - record.size()); - ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset], - offset_reader->LastRecordOffset()); - ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]); + + // Read all records from expected_record_offset through the last one. + ASSERT_LT(expected_record_offset, num_initial_offset_records_); + for (; expected_record_offset < num_initial_offset_records_; + ++expected_record_offset) { + Slice record; + std::string scratch; + ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch)); + ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset], + record.size()); + ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset], + offset_reader->LastRecordOffset()); + ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]); + } delete offset_reader; } - }; size_t LogTest::initial_offset_record_sizes_[] = {10000, // Two sizable records in first block 10000, 2 * log::kBlockSize - 1000, // Span three blocks - 1}; + 1, + 13716, // Consume all but two bytes of block 3. + log::kBlockSize - kHeaderSize, // Consume the entirety of block 4. + }; uint64_t LogTest::initial_offset_last_record_offsets_[] = {0, kHeaderSize + 10000, 2 * (kHeaderSize + 10000), 2 * (kHeaderSize + 10000) + - (2 * log::kBlockSize - 1000) + 3 * kHeaderSize}; + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, + 2 * (kHeaderSize + 10000) + + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize + + kHeaderSize + 1, + 3 * log::kBlockSize, + }; +// LogTest::initial_offset_last_record_offsets_ must be defined before this. +int LogTest::num_initial_offset_records_ = + sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t); TEST(LogTest, Empty) { ASSERT_EQ("EOF", Read()); @@ -318,6 +350,15 @@ TEST(LogTest, AlignedEof) { ASSERT_EQ("EOF", Read()); } +TEST(LogTest, OpenForAppend) { + Write("hello"); + ReopenForAppend(); + Write("world"); + ASSERT_EQ("hello", Read()); + ASSERT_EQ("world", Read()); + ASSERT_EQ("EOF", Read()); +} + TEST(LogTest, RandomRead) { const int N = 500; Random write_rnd(301); @@ -445,6 +486,22 @@ TEST(LogTest, PartialLastIsIgnored) { ASSERT_EQ(0, DroppedBytes()); } +TEST(LogTest, SkipIntoMultiRecord) { + // Consider a fragmented record: + // first(R1), middle(R1), last(R1), first(R2) + // If initial_offset points to a record after first(R1) but before first(R2) + // incomplete fragment errors are not actual errors, and must be suppressed + // until a new first or full record is encountered. + Write(BigString("foo", 3*kBlockSize)); + Write("correct"); + StartReadingAt(kBlockSize); + + ASSERT_EQ("correct", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("EOF", Read()); +} + TEST(LogTest, ErrorJoinsRecords) { // Consider two fragmented records: // first(R1) last(R1) first(R2) last(R2) @@ -514,6 +571,10 @@ TEST(LogTest, ReadFourthStart) { 3); } +TEST(LogTest, ReadInitialOffsetIntoBlockPadding) { + CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5); +} + TEST(LogTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); } diff --git a/src/leveldb/db/log_writer.cc b/src/leveldb/db/log_writer.cc index 2da99ac088..74a03270da 100644 --- a/src/leveldb/db/log_writer.cc +++ b/src/leveldb/db/log_writer.cc @@ -12,15 +12,24 @@ namespace leveldb { namespace log { -Writer::Writer(WritableFile* dest) - : dest_(dest), - block_offset_(0) { +static void InitTypeCrc(uint32_t* type_crc) { for (int i = 0; i <= kMaxRecordType; i++) { char t = static_cast<char>(i); - type_crc_[i] = crc32c::Value(&t, 1); + type_crc[i] = crc32c::Value(&t, 1); } } +Writer::Writer(WritableFile* dest) + : dest_(dest), + block_offset_(0) { + InitTypeCrc(type_crc_); +} + +Writer::Writer(WritableFile* dest, uint64_t dest_length) + : dest_(dest), block_offset_(dest_length % kBlockSize) { + InitTypeCrc(type_crc_); +} + Writer::~Writer() { } diff --git a/src/leveldb/db/log_writer.h b/src/leveldb/db/log_writer.h index a3a954d967..9e7cc4705b 100644 --- a/src/leveldb/db/log_writer.h +++ b/src/leveldb/db/log_writer.h @@ -22,6 +22,12 @@ class Writer { // "*dest" must be initially empty. // "*dest" must remain live while this Writer is in use. explicit Writer(WritableFile* dest); + + // Create a writer that will append data to "*dest". + // "*dest" must have initial length "dest_length". + // "*dest" must remain live while this Writer is in use. + Writer(WritableFile* dest, uint64_t dest_length); + ~Writer(); Status AddRecord(const Slice& slice); diff --git a/src/leveldb/db/memtable.h b/src/leveldb/db/memtable.h index 92e90bb099..9f41567cde 100644 --- a/src/leveldb/db/memtable.h +++ b/src/leveldb/db/memtable.h @@ -36,10 +36,7 @@ class MemTable { } // Returns an estimate of the number of bytes of data in use by this - // data structure. - // - // REQUIRES: external synchronization to prevent simultaneous - // operations on the same MemTable. + // data structure. It is safe to call when MemTable is being modified. size_t ApproximateMemoryUsage(); // Return an iterator that yields the contents of the memtable. diff --git a/src/leveldb/db/recovery_test.cc b/src/leveldb/db/recovery_test.cc new file mode 100644 index 0000000000..9596f4288a --- /dev/null +++ b/src/leveldb/db/recovery_test.cc @@ -0,0 +1,324 @@ +// Copyright (c) 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_impl.h" +#include "db/filename.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" +#include "util/testharness.h" +#include "util/testutil.h" + +namespace leveldb { + +class RecoveryTest { + public: + RecoveryTest() : env_(Env::Default()), db_(NULL) { + dbname_ = test::TmpDir() + "/recovery_test"; + DestroyDB(dbname_, Options()); + Open(); + } + + ~RecoveryTest() { + Close(); + DestroyDB(dbname_, Options()); + } + + DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); } + Env* env() const { return env_; } + + bool CanAppend() { + WritableFile* tmp; + Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp); + delete tmp; + if (s.IsNotSupportedError()) { + return false; + } else { + return true; + } + } + + void Close() { + delete db_; + db_ = NULL; + } + + void Open(Options* options = NULL) { + Close(); + Options opts; + if (options != NULL) { + opts = *options; + } else { + opts.reuse_logs = true; // TODO(sanjay): test both ways + opts.create_if_missing = true; + } + if (opts.env == NULL) { + opts.env = env_; + } + ASSERT_OK(DB::Open(opts, dbname_, &db_)); + ASSERT_EQ(1, NumLogs()); + } + + Status Put(const std::string& k, const std::string& v) { + return db_->Put(WriteOptions(), k, v); + } + + std::string Get(const std::string& k, const Snapshot* snapshot = NULL) { + std::string result; + Status s = db_->Get(ReadOptions(), k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + std::string ManifestFileName() { + std::string current; + ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), ¤t)); + size_t len = current.size(); + if (len > 0 && current[len-1] == '\n') { + current.resize(len - 1); + } + return dbname_ + "/" + current; + } + + std::string LogName(uint64_t number) { + return LogFileName(dbname_, number); + } + + size_t DeleteLogFiles() { + std::vector<uint64_t> logs = GetFiles(kLogFile); + for (size_t i = 0; i < logs.size(); i++) { + ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]); + } + return logs.size(); + } + + uint64_t FirstLogFile() { + return GetFiles(kLogFile)[0]; + } + + std::vector<uint64_t> GetFiles(FileType t) { + std::vector<std::string> filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + std::vector<uint64_t> result; + for (size_t i = 0; i < filenames.size(); i++) { + uint64_t number; + FileType type; + if (ParseFileName(filenames[i], &number, &type) && type == t) { + result.push_back(number); + } + } + return result; + } + + int NumLogs() { + return GetFiles(kLogFile).size(); + } + + int NumTables() { + return GetFiles(kTableFile).size(); + } + + uint64_t FileSize(const std::string& fname) { + uint64_t result; + ASSERT_OK(env_->GetFileSize(fname, &result)) << fname; + return result; + } + + void CompactMemTable() { + dbfull()->TEST_CompactMemTable(); + } + + // Directly construct a log file that sets key to val. + void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) { + std::string fname = LogFileName(dbname_, lognum); + WritableFile* file; + ASSERT_OK(env_->NewWritableFile(fname, &file)); + log::Writer writer(file); + WriteBatch batch; + batch.Put(key, val); + WriteBatchInternal::SetSequence(&batch, seq); + ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); + ASSERT_OK(file->Flush()); + delete file; + } + + private: + std::string dbname_; + Env* env_; + DB* db_; +}; + +TEST(RecoveryTest, ManifestReused) { + if (!CanAppend()) { + fprintf(stderr, "skipping test because env does not support appending\n"); + return; + } + ASSERT_OK(Put("foo", "bar")); + Close(); + std::string old_manifest = ManifestFileName(); + Open(); + ASSERT_EQ(old_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); + Open(); + ASSERT_EQ(old_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); +} + +TEST(RecoveryTest, LargeManifestCompacted) { + if (!CanAppend()) { + fprintf(stderr, "skipping test because env does not support appending\n"); + return; + } + ASSERT_OK(Put("foo", "bar")); + Close(); + std::string old_manifest = ManifestFileName(); + + // Pad with zeroes to make manifest file very big. + { + uint64_t len = FileSize(old_manifest); + WritableFile* file; + ASSERT_OK(env()->NewAppendableFile(old_manifest, &file)); + std::string zeroes(3*1048576 - static_cast<size_t>(len), 0); + ASSERT_OK(file->Append(zeroes)); + ASSERT_OK(file->Flush()); + delete file; + } + + Open(); + std::string new_manifest = ManifestFileName(); + ASSERT_NE(old_manifest, new_manifest); + ASSERT_GT(10000, FileSize(new_manifest)); + ASSERT_EQ("bar", Get("foo")); + + Open(); + ASSERT_EQ(new_manifest, ManifestFileName()); + ASSERT_EQ("bar", Get("foo")); +} + +TEST(RecoveryTest, NoLogFiles) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ(1, DeleteLogFiles()); + Open(); + ASSERT_EQ("NOT_FOUND", Get("foo")); + Open(); + ASSERT_EQ("NOT_FOUND", Get("foo")); +} + +TEST(RecoveryTest, LogFileReuse) { + if (!CanAppend()) { + fprintf(stderr, "skipping test because env does not support appending\n"); + return; + } + for (int i = 0; i < 2; i++) { + ASSERT_OK(Put("foo", "bar")); + if (i == 0) { + // Compact to ensure current log is empty + CompactMemTable(); + } + Close(); + ASSERT_EQ(1, NumLogs()); + uint64_t number = FirstLogFile(); + if (i == 0) { + ASSERT_EQ(0, FileSize(LogName(number))); + } else { + ASSERT_LT(0, FileSize(LogName(number))); + } + Open(); + ASSERT_EQ(1, NumLogs()); + ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; + ASSERT_EQ("bar", Get("foo")); + Open(); + ASSERT_EQ(1, NumLogs()); + ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; + ASSERT_EQ("bar", Get("foo")); + } +} + +TEST(RecoveryTest, MultipleMemTables) { + // Make a large log. + const int kNum = 1000; + for (int i = 0; i < kNum; i++) { + char buf[100]; + snprintf(buf, sizeof(buf), "%050d", i); + ASSERT_OK(Put(buf, buf)); + } + ASSERT_EQ(0, NumTables()); + Close(); + ASSERT_EQ(0, NumTables()); + ASSERT_EQ(1, NumLogs()); + uint64_t old_log_file = FirstLogFile(); + + // Force creation of multiple memtables by reducing the write buffer size. + Options opt; + opt.reuse_logs = true; + opt.write_buffer_size = (kNum*100) / 2; + Open(&opt); + ASSERT_LE(2, NumTables()); + ASSERT_EQ(1, NumLogs()); + ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log"; + for (int i = 0; i < kNum; i++) { + char buf[100]; + snprintf(buf, sizeof(buf), "%050d", i); + ASSERT_EQ(buf, Get(buf)); + } +} + +TEST(RecoveryTest, MultipleLogFiles) { + ASSERT_OK(Put("foo", "bar")); + Close(); + ASSERT_EQ(1, NumLogs()); + + // Make a bunch of uncompacted log files. + uint64_t old_log = FirstLogFile(); + MakeLogFile(old_log+1, 1000, "hello", "world"); + MakeLogFile(old_log+2, 1001, "hi", "there"); + MakeLogFile(old_log+3, 1002, "foo", "bar2"); + + // Recover and check that all log files were processed. + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + uint64_t new_log = FirstLogFile(); + ASSERT_LE(old_log+3, new_log); + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); + + // Test that previous recovery produced recoverable state. + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + if (CanAppend()) { + ASSERT_EQ(new_log, FirstLogFile()); + } + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); + + // Check that introducing an older log file does not cause it to be re-read. + Close(); + MakeLogFile(old_log+1, 2000, "hello", "stale write"); + Open(); + ASSERT_LE(1, NumTables()); + ASSERT_EQ(1, NumLogs()); + if (CanAppend()) { + ASSERT_EQ(new_log, FirstLogFile()); + } + ASSERT_EQ("bar2", Get("foo")); + ASSERT_EQ("world", Get("hello")); + ASSERT_EQ("there", Get("hi")); +} + +} // namespace leveldb + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/src/leveldb/db/skiplist.h b/src/leveldb/db/skiplist.h index ed8b092203..8bd77764d8 100644 --- a/src/leveldb/db/skiplist.h +++ b/src/leveldb/db/skiplist.h @@ -1,10 +1,10 @@ -#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_ -#define STORAGE_LEVELDB_DB_SKIPLIST_H_ - // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -// + +#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_ +#define STORAGE_LEVELDB_DB_SKIPLIST_H_ + // Thread safety // ------------- // diff --git a/src/leveldb/db/skiplist_test.cc b/src/leveldb/db/skiplist_test.cc index c78f4b4fb1..aee1461e1b 100644 --- a/src/leveldb/db/skiplist_test.cc +++ b/src/leveldb/db/skiplist_test.cc @@ -250,7 +250,7 @@ class ConcurrentTest { // Note that generation 0 is never inserted, so it is ok if // <*,0,*> is missing. ASSERT_TRUE((gen(pos) == 0) || - (gen(pos) > initial_state.Get(key(pos))) + (gen(pos) > static_cast<Key>(initial_state.Get(key(pos)))) ) << "key: " << key(pos) << "; gen: " << gen(pos) << "; initgen: " diff --git a/src/leveldb/db/snapshot.h b/src/leveldb/db/snapshot.h index e7f8fd2c37..6ed413c42d 100644 --- a/src/leveldb/db/snapshot.h +++ b/src/leveldb/db/snapshot.h @@ -5,6 +5,7 @@ #ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_ #define STORAGE_LEVELDB_DB_SNAPSHOT_H_ +#include "db/dbformat.h" #include "leveldb/db.h" namespace leveldb { diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index aa83df55e4..a5e0f77a6a 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -893,7 +893,7 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { return s; } -Status VersionSet::Recover() { +Status VersionSet::Recover(bool *save_manifest) { struct LogReporter : public log::Reader::Reporter { Status* status; virtual void Corruption(size_t bytes, const Status& s) { @@ -1003,11 +1003,49 @@ Status VersionSet::Recover() { last_sequence_ = last_sequence; log_number_ = log_number; prev_log_number_ = prev_log_number; + + // See if we can reuse the existing MANIFEST file. + if (ReuseManifest(dscname, current)) { + // No need to save new manifest + } else { + *save_manifest = true; + } } return s; } +bool VersionSet::ReuseManifest(const std::string& dscname, + const std::string& dscbase) { + if (!options_->reuse_logs) { + return false; + } + FileType manifest_type; + uint64_t manifest_number; + uint64_t manifest_size; + if (!ParseFileName(dscbase, &manifest_number, &manifest_type) || + manifest_type != kDescriptorFile || + !env_->GetFileSize(dscname, &manifest_size).ok() || + // Make new compacted MANIFEST if old one is too big + manifest_size >= kTargetFileSize) { + return false; + } + + assert(descriptor_file_ == NULL); + assert(descriptor_log_ == NULL); + Status r = env_->NewAppendableFile(dscname, &descriptor_file_); + if (!r.ok()) { + Log(options_->info_log, "Reuse MANIFEST: %s\n", r.ToString().c_str()); + assert(descriptor_file_ == NULL); + return false; + } + + Log(options_->info_log, "Reusing MANIFEST %s\n", dscname.c_str()); + descriptor_log_ = new log::Writer(descriptor_file_, manifest_size); + manifest_file_number_ = manifest_number; + return true; +} + void VersionSet::MarkFileNumberUsed(uint64_t number) { if (next_file_number_ <= number) { next_file_number_ = number + 1; diff --git a/src/leveldb/db/version_set.h b/src/leveldb/db/version_set.h index 8dc14b8e01..1dec745673 100644 --- a/src/leveldb/db/version_set.h +++ b/src/leveldb/db/version_set.h @@ -179,7 +179,7 @@ class VersionSet { EXCLUSIVE_LOCKS_REQUIRED(mu); // Recover the last saved descriptor from persistent storage. - Status Recover(); + Status Recover(bool *save_manifest); // Return the current version. Version* current() const { return current_; } @@ -274,6 +274,8 @@ class VersionSet { friend class Compaction; friend class Version; + bool ReuseManifest(const std::string& dscname, const std::string& dscbase); + void Finalize(Version* v); void GetRange(const std::vector<FileMetaData*>& inputs, diff --git a/src/leveldb/db/write_batch_internal.h b/src/leveldb/db/write_batch_internal.h index 310a3c8912..9448ef7b21 100644 --- a/src/leveldb/db/write_batch_internal.h +++ b/src/leveldb/db/write_batch_internal.h @@ -5,6 +5,7 @@ #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ +#include "db/dbformat.h" #include "leveldb/write_batch.h" namespace leveldb { diff --git a/src/leveldb/doc/index.html b/src/leveldb/doc/index.html index 3ed0ed9d9e..2155192581 100644 --- a/src/leveldb/doc/index.html +++ b/src/leveldb/doc/index.html @@ -22,7 +22,7 @@ directory. The following example shows how to open a database, creating it if necessary: <p> <pre> - #include <assert> + #include <cassert> #include "leveldb/db.h" leveldb::DB* db; diff --git a/src/leveldb/helpers/memenv/memenv.cc b/src/leveldb/helpers/memenv/memenv.cc index 43ef2e0729..9a98884daf 100644 --- a/src/leveldb/helpers/memenv/memenv.cc +++ b/src/leveldb/helpers/memenv/memenv.cc @@ -277,6 +277,19 @@ class InMemoryEnv : public EnvWrapper { return Status::OK(); } + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result) { + MutexLock lock(&mutex_); + FileState** sptr = &file_map_[fname]; + FileState* file = *sptr; + if (file == NULL) { + file = new FileState(); + file->Ref(); + } + *result = new WritableFileImpl(file); + return Status::OK(); + } + virtual bool FileExists(const std::string& fname) { MutexLock lock(&mutex_); return file_map_.find(fname) != file_map_.end(); diff --git a/src/leveldb/helpers/memenv/memenv_test.cc b/src/leveldb/helpers/memenv/memenv_test.cc index a44310fed8..5cff77613f 100644 --- a/src/leveldb/helpers/memenv/memenv_test.cc +++ b/src/leveldb/helpers/memenv/memenv_test.cc @@ -40,6 +40,8 @@ TEST(MemEnvTest, Basics) { // Create a file. ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(0, file_size); delete writable_file; // Check that the file exists. @@ -55,9 +57,16 @@ TEST(MemEnvTest, Basics) { ASSERT_OK(writable_file->Append("abc")); delete writable_file; - // Check for expected size. + // Check that append works. + ASSERT_OK(env_->NewAppendableFile("/dir/f", &writable_file)); ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); ASSERT_EQ(3, file_size); + ASSERT_OK(writable_file->Append("hello")); + delete writable_file; + + // Check for expected size. + ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(8, file_size); // Check that renaming works. ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); @@ -65,7 +74,7 @@ TEST(MemEnvTest, Basics) { ASSERT_TRUE(!env_->FileExists("/dir/f")); ASSERT_TRUE(env_->FileExists("/dir/g")); ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); - ASSERT_EQ(3, file_size); + ASSERT_EQ(8, file_size); // Check that opening non-existent file fails. SequentialFile* seq_file; diff --git a/src/leveldb/include/leveldb/cache.h b/src/leveldb/include/leveldb/cache.h index 1a201e5e0a..6819d5bc49 100644 --- a/src/leveldb/include/leveldb/cache.h +++ b/src/leveldb/include/leveldb/cache.h @@ -81,6 +81,17 @@ class Cache { // its cache keys. virtual uint64_t NewId() = 0; + // Remove all cache entries that are not actively in use. Memory-constrained + // applications may wish to call this method to reduce memory usage. + // Default implementation of Prune() does nothing. Subclasses are strongly + // encouraged to override the default implementation. A future release of + // leveldb may change Prune() to a pure abstract method. + virtual void Prune() {} + + // Return an estimate of the combined charges of all elements stored in the + // cache. + virtual size_t TotalCharge() const = 0; + private: void LRU_Remove(Handle* e); void LRU_Append(Handle* e); diff --git a/src/leveldb/include/leveldb/db.h b/src/leveldb/include/leveldb/db.h index 4c169bf22e..9752cbad51 100644 --- a/src/leveldb/include/leveldb/db.h +++ b/src/leveldb/include/leveldb/db.h @@ -14,7 +14,7 @@ namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 18; +static const int kMinorVersion = 19; struct Options; struct ReadOptions; @@ -115,6 +115,8 @@ class DB { // about the internal operation of the DB. // "leveldb.sstables" - returns a multi-line string that describes all // of the sstables that make up the db contents. + // "leveldb.approximate-memory-usage" - returns the approximate number of + // bytes of memory in use by the DB. virtual bool GetProperty(const Slice& property, std::string* value) = 0; // For each i in [0,n-1], store in "sizes[i]", the approximate diff --git a/src/leveldb/include/leveldb/env.h b/src/leveldb/include/leveldb/env.h index f709514da6..99b6c21414 100644 --- a/src/leveldb/include/leveldb/env.h +++ b/src/leveldb/include/leveldb/env.h @@ -69,6 +69,21 @@ class Env { virtual Status NewWritableFile(const std::string& fname, WritableFile** result) = 0; + // Create an object that either appends to an existing file, or + // writes to a new file (if the file does not exist to begin with). + // On success, stores a pointer to the new file in *result and + // returns OK. On failure stores NULL in *result and returns + // non-OK. + // + // The returned file will only be accessed by one thread at a time. + // + // May return an IsNotSupportedError error if this Env does + // not allow appending to an existing file. Users of Env (including + // the leveldb implementation) must be prepared to deal with + // an Env that does not support appending. + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result); + // Returns true iff the named file exists. virtual bool FileExists(const std::string& fname) = 0; @@ -289,6 +304,9 @@ class EnvWrapper : public Env { Status NewWritableFile(const std::string& f, WritableFile** r) { return target_->NewWritableFile(f, r); } + Status NewAppendableFile(const std::string& f, WritableFile** r) { + return target_->NewAppendableFile(f, r); + } bool FileExists(const std::string& f) { return target_->FileExists(f); } Status GetChildren(const std::string& dir, std::vector<std::string>* r) { return target_->GetChildren(dir, r); diff --git a/src/leveldb/include/leveldb/iterator.h b/src/leveldb/include/leveldb/iterator.h index 76aced04bd..da631ed9d8 100644 --- a/src/leveldb/include/leveldb/iterator.h +++ b/src/leveldb/include/leveldb/iterator.h @@ -37,7 +37,7 @@ class Iterator { // Valid() after this call iff the source is not empty. virtual void SeekToLast() = 0; - // Position at the first key in the source that at or past target + // Position at the first key in the source that is at or past target. // The iterator is Valid() after this call iff the source contains // an entry that comes at or past target. virtual void Seek(const Slice& target) = 0; diff --git a/src/leveldb/include/leveldb/options.h b/src/leveldb/include/leveldb/options.h index 7c9b973454..83a1ef39a4 100644 --- a/src/leveldb/include/leveldb/options.h +++ b/src/leveldb/include/leveldb/options.h @@ -128,6 +128,12 @@ struct Options { // efficiently detect that and will switch to uncompressed mode. CompressionType compression; + // EXPERIMENTAL: If true, append to existing MANIFEST and log files + // when a database is opened. This can significantly speed up open. + // + // Default: currently false, but may become true later. + bool reuse_logs; + // If non-NULL, use the specified filter policy to reduce disk reads. // Many applications will benefit from passing the result of // NewBloomFilterPolicy() here. diff --git a/src/leveldb/include/leveldb/status.h b/src/leveldb/include/leveldb/status.h index 11dbd4b47e..d9575f9753 100644 --- a/src/leveldb/include/leveldb/status.h +++ b/src/leveldb/include/leveldb/status.h @@ -60,6 +60,12 @@ class Status { // Returns true iff the status indicates an IOError. bool IsIOError() const { return code() == kIOError; } + // Returns true iff the status indicates a NotSupportedError. + bool IsNotSupportedError() const { return code() == kNotSupported; } + + // Returns true iff the status indicates an InvalidArgument. + bool IsInvalidArgument() const { return code() == kInvalidArgument; } + // Return a string representation of this status suitable for printing. // Returns the string "OK" for success. std::string ToString() const; diff --git a/src/leveldb/port/atomic_pointer.h b/src/leveldb/port/atomic_pointer.h index 9bf091f757..1c4c7aafc6 100644 --- a/src/leveldb/port/atomic_pointer.h +++ b/src/leveldb/port/atomic_pointer.h @@ -35,8 +35,12 @@ #define ARCH_CPU_X86_FAMILY 1 #elif defined(__ARMEL__) #define ARCH_CPU_ARM_FAMILY 1 +#elif defined(__aarch64__) +#define ARCH_CPU_ARM64_FAMILY 1 #elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__) #define ARCH_CPU_PPC_FAMILY 1 +#elif defined(__mips__) +#define ARCH_CPU_MIPS_FAMILY 1 #endif namespace leveldb { @@ -92,6 +96,13 @@ inline void MemoryBarrier() { } #define LEVELDB_HAVE_MEMORY_BARRIER +// ARM64 +#elif defined(ARCH_CPU_ARM64_FAMILY) +inline void MemoryBarrier() { + asm volatile("dmb sy" : : : "memory"); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + // PPC #elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__) inline void MemoryBarrier() { @@ -101,6 +112,13 @@ inline void MemoryBarrier() { } #define LEVELDB_HAVE_MEMORY_BARRIER +// MIPS +#elif defined(ARCH_CPU_MIPS_FAMILY) && defined(__GNUC__) +inline void MemoryBarrier() { + __asm__ __volatile__("sync" : : : "memory"); +} +#define LEVELDB_HAVE_MEMORY_BARRIER + #endif // AtomicPointer built using platform-specific MemoryBarrier() @@ -215,6 +233,7 @@ class AtomicPointer { #undef LEVELDB_HAVE_MEMORY_BARRIER #undef ARCH_CPU_X86_FAMILY #undef ARCH_CPU_ARM_FAMILY +#undef ARCH_CPU_ARM64_FAMILY #undef ARCH_CPU_PPC_FAMILY } // namespace port diff --git a/src/leveldb/port/port_posix.cc b/src/leveldb/port/port_posix.cc index 5ba127a5b9..30e8007ae3 100644 --- a/src/leveldb/port/port_posix.cc +++ b/src/leveldb/port/port_posix.cc @@ -7,7 +7,6 @@ #include <cstdlib> #include <stdio.h> #include <string.h> -#include "util/logging.h" namespace leveldb { namespace port { diff --git a/src/leveldb/table/filter_block.cc b/src/leveldb/table/filter_block.cc index 203e15c8bc..4e78b954f8 100644 --- a/src/leveldb/table/filter_block.cc +++ b/src/leveldb/table/filter_block.cc @@ -68,7 +68,7 @@ void FilterBlockBuilder::GenerateFilter() { // Generate filter for current set of keys and append to result_. filter_offsets_.push_back(result_.size()); - policy_->CreateFilter(&tmp_keys_[0], num_keys, &result_); + policy_->CreateFilter(&tmp_keys_[0], static_cast<int>(num_keys), &result_); tmp_keys_.clear(); keys_.clear(); @@ -97,7 +97,7 @@ bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) { if (index < num_) { uint32_t start = DecodeFixed32(offset_ + index*4); uint32_t limit = DecodeFixed32(offset_ + index*4 + 4); - if (start <= limit && limit <= (offset_ - data_)) { + if (start <= limit && limit <= static_cast<size_t>(offset_ - data_)) { Slice filter = Slice(data_ + start, limit - start); return policy_->KeyMayMatch(key, filter); } else if (start == limit) { diff --git a/src/leveldb/table/format.cc b/src/leveldb/table/format.cc index aa63144c9e..24e4e02445 100644 --- a/src/leveldb/table/format.cc +++ b/src/leveldb/table/format.cc @@ -30,15 +30,14 @@ Status BlockHandle::DecodeFrom(Slice* input) { } void Footer::EncodeTo(std::string* dst) const { -#ifndef NDEBUG const size_t original_size = dst->size(); -#endif metaindex_handle_.EncodeTo(dst); index_handle_.EncodeTo(dst); dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber & 0xffffffffu)); PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber >> 32)); assert(dst->size() == original_size + kEncodedLength); + (void)original_size; // Disable unused variable warning. } Status Footer::DecodeFrom(Slice* input) { diff --git a/src/leveldb/table/iterator_wrapper.h b/src/leveldb/table/iterator_wrapper.h index 9e16b3dbed..f410c3fabe 100644 --- a/src/leveldb/table/iterator_wrapper.h +++ b/src/leveldb/table/iterator_wrapper.h @@ -5,6 +5,9 @@ #ifndef STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ #define STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ +#include "leveldb/iterator.h" +#include "leveldb/slice.h" + namespace leveldb { // A internal wrapper class with an interface similar to Iterator that diff --git a/src/leveldb/table/table.cc b/src/leveldb/table/table.cc index dff8a82590..decf8082cc 100644 --- a/src/leveldb/table/table.cc +++ b/src/leveldb/table/table.cc @@ -82,7 +82,7 @@ Status Table::Open(const Options& options, *table = new Table(rep); (*table)->ReadMeta(footer); } else { - if (index_block) delete index_block; + delete index_block; } return s; diff --git a/src/leveldb/table/table_test.cc b/src/leveldb/table/table_test.cc index c723bf84cf..abf6e246ff 100644 --- a/src/leveldb/table/table_test.cc +++ b/src/leveldb/table/table_test.cc @@ -853,12 +853,20 @@ TEST(TableTest, ApproximateOffsetOfCompressed) { options.compression = kSnappyCompression; c.Finish(options, &keys, &kvmap); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 2000, 3000)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 2000, 3000)); - ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 4000, 6000)); + // Expected upper and lower bounds of space used by compressible strings. + static const int kSlop = 1000; // Compressor effectiveness varies. + const int expected = 2500; // 10000 * compression ratio (0.25) + const int min_z = expected - kSlop; + const int max_z = expected + kSlop; + + ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, kSlop)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, kSlop)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, kSlop)); + // Have now emitted a large compressible string, so adjust expected offset. + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), min_z, max_z)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), min_z, max_z)); + // Have now emitted two large compressible strings, so adjust expected offset. + ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 2 * min_z, 2 * max_z)); } } // namespace leveldb diff --git a/src/leveldb/util/arena.cc b/src/leveldb/util/arena.cc index 9367f71492..74078213ee 100644 --- a/src/leveldb/util/arena.cc +++ b/src/leveldb/util/arena.cc @@ -9,8 +9,7 @@ namespace leveldb { static const int kBlockSize = 4096; -Arena::Arena() { - blocks_memory_ = 0; +Arena::Arena() : memory_usage_(0) { alloc_ptr_ = NULL; // First allocation will allocate a block alloc_bytes_remaining_ = 0; } @@ -60,8 +59,9 @@ char* Arena::AllocateAligned(size_t bytes) { char* Arena::AllocateNewBlock(size_t block_bytes) { char* result = new char[block_bytes]; - blocks_memory_ += block_bytes; blocks_.push_back(result); + memory_usage_.NoBarrier_Store( + reinterpret_cast<void*>(MemoryUsage() + block_bytes + sizeof(char*))); return result; } diff --git a/src/leveldb/util/arena.h b/src/leveldb/util/arena.h index 73bbf1cb9b..48bab33741 100644 --- a/src/leveldb/util/arena.h +++ b/src/leveldb/util/arena.h @@ -9,6 +9,7 @@ #include <assert.h> #include <stddef.h> #include <stdint.h> +#include "port/port.h" namespace leveldb { @@ -24,10 +25,9 @@ class Arena { char* AllocateAligned(size_t bytes); // Returns an estimate of the total memory usage of data allocated - // by the arena (including space allocated but not yet used for user - // allocations). + // by the arena. size_t MemoryUsage() const { - return blocks_memory_ + blocks_.capacity() * sizeof(char*); + return reinterpret_cast<uintptr_t>(memory_usage_.NoBarrier_Load()); } private: @@ -41,8 +41,8 @@ class Arena { // Array of new[] allocated memory blocks std::vector<char*> blocks_; - // Bytes of memory in blocks allocated so far - size_t blocks_memory_; + // Total memory usage of the arena. + port::AtomicPointer memory_usage_; // No copying allowed Arena(const Arena&); diff --git a/src/leveldb/util/bloom.cc b/src/leveldb/util/bloom.cc index a27a2ace28..bf3e4ca6e9 100644 --- a/src/leveldb/util/bloom.cc +++ b/src/leveldb/util/bloom.cc @@ -47,7 +47,7 @@ class BloomFilterPolicy : public FilterPolicy { dst->resize(init_size + bytes, 0); dst->push_back(static_cast<char>(k_)); // Remember # of probes in filter char* array = &(*dst)[init_size]; - for (size_t i = 0; i < n; i++) { + for (int i = 0; i < n; i++) { // Use double-hashing to generate a sequence of hash values. // See analysis in [Kirsch,Mitzenmacher 2006]. uint32_t h = BloomHash(keys[i]); diff --git a/src/leveldb/util/bloom_test.cc b/src/leveldb/util/bloom_test.cc index 77fb1b3159..1b87a2be3f 100644 --- a/src/leveldb/util/bloom_test.cc +++ b/src/leveldb/util/bloom_test.cc @@ -46,7 +46,8 @@ class BloomTest { key_slices.push_back(Slice(keys_[i])); } filter_.clear(); - policy_->CreateFilter(&key_slices[0], key_slices.size(), &filter_); + policy_->CreateFilter(&key_slices[0], static_cast<int>(key_slices.size()), + &filter_); keys_.clear(); if (kVerbose >= 2) DumpFilter(); } diff --git a/src/leveldb/util/cache.cc b/src/leveldb/util/cache.cc index 8b197bc02a..ce46886171 100644 --- a/src/leveldb/util/cache.cc +++ b/src/leveldb/util/cache.cc @@ -19,6 +19,23 @@ Cache::~Cache() { namespace { // LRU cache implementation +// +// Cache entries have an "in_cache" boolean indicating whether the cache has a +// reference on the entry. The only ways that this can become false without the +// entry being passed to its "deleter" are via Erase(), via Insert() when +// an element with a duplicate key is inserted, or on destruction of the cache. +// +// The cache keeps two linked lists of items in the cache. All items in the +// cache are in one list or the other, and never both. Items still referenced +// by clients but erased from the cache are in neither list. The lists are: +// - in-use: contains the items currently referenced by clients, in no +// particular order. (This list is used for invariant checking. If we +// removed the check, elements that would otherwise be on this list could be +// left as disconnected singleton lists.) +// - LRU: contains the items not currently referenced by clients, in LRU order +// Elements are moved between these lists by the Ref() and Unref() methods, +// when they detect an element in the cache acquiring or losing its only +// external reference. // An entry is a variable length heap-allocated structure. Entries // are kept in a circular doubly linked list ordered by access time. @@ -30,7 +47,8 @@ struct LRUHandle { LRUHandle* prev; size_t charge; // TODO(opt): Only allow uint32_t? size_t key_length; - uint32_t refs; + bool in_cache; // Whether entry is in the cache. + uint32_t refs; // References, including cache reference, if present. uint32_t hash; // Hash of key(); used for fast sharding and comparisons char key_data[1]; // Beginning of key @@ -147,49 +165,77 @@ class LRUCache { Cache::Handle* Lookup(const Slice& key, uint32_t hash); void Release(Cache::Handle* handle); void Erase(const Slice& key, uint32_t hash); + void Prune(); + size_t TotalCharge() const { + MutexLock l(&mutex_); + return usage_; + } private: void LRU_Remove(LRUHandle* e); - void LRU_Append(LRUHandle* e); + void LRU_Append(LRUHandle*list, LRUHandle* e); + void Ref(LRUHandle* e); void Unref(LRUHandle* e); + bool FinishErase(LRUHandle* e); // Initialized before use. size_t capacity_; // mutex_ protects the following state. - port::Mutex mutex_; + mutable port::Mutex mutex_; size_t usage_; // Dummy head of LRU list. // lru.prev is newest entry, lru.next is oldest entry. + // Entries have refs==1 and in_cache==true. LRUHandle lru_; + // Dummy head of in-use list. + // Entries are in use by clients, and have refs >= 2 and in_cache==true. + LRUHandle in_use_; + HandleTable table_; }; LRUCache::LRUCache() : usage_(0) { - // Make empty circular linked list + // Make empty circular linked lists. lru_.next = &lru_; lru_.prev = &lru_; + in_use_.next = &in_use_; + in_use_.prev = &in_use_; } LRUCache::~LRUCache() { + assert(in_use_.next == &in_use_); // Error if caller has an unreleased handle for (LRUHandle* e = lru_.next; e != &lru_; ) { LRUHandle* next = e->next; - assert(e->refs == 1); // Error if caller has an unreleased handle + assert(e->in_cache); + e->in_cache = false; + assert(e->refs == 1); // Invariant of lru_ list. Unref(e); e = next; } } +void LRUCache::Ref(LRUHandle* e) { + if (e->refs == 1 && e->in_cache) { // If on lru_ list, move to in_use_ list. + LRU_Remove(e); + LRU_Append(&in_use_, e); + } + e->refs++; +} + void LRUCache::Unref(LRUHandle* e) { assert(e->refs > 0); e->refs--; - if (e->refs <= 0) { - usage_ -= e->charge; + if (e->refs == 0) { // Deallocate. + assert(!e->in_cache); (*e->deleter)(e->key(), e->value); free(e); + } else if (e->in_cache && e->refs == 1) { // No longer in use; move to lru_ list. + LRU_Remove(e); + LRU_Append(&lru_, e); } } @@ -198,10 +244,10 @@ void LRUCache::LRU_Remove(LRUHandle* e) { e->prev->next = e->next; } -void LRUCache::LRU_Append(LRUHandle* e) { - // Make "e" newest entry by inserting just before lru_ - e->next = &lru_; - e->prev = lru_.prev; +void LRUCache::LRU_Append(LRUHandle* list, LRUHandle* e) { + // Make "e" newest entry by inserting just before *list + e->next = list; + e->prev = list->prev; e->prev->next = e; e->next->prev = e; } @@ -210,9 +256,7 @@ Cache::Handle* LRUCache::Lookup(const Slice& key, uint32_t hash) { MutexLock l(&mutex_); LRUHandle* e = table_.Lookup(key, hash); if (e != NULL) { - e->refs++; - LRU_Remove(e); - LRU_Append(e); + Ref(e); } return reinterpret_cast<Cache::Handle*>(e); } @@ -234,34 +278,58 @@ Cache::Handle* LRUCache::Insert( e->charge = charge; e->key_length = key.size(); e->hash = hash; - e->refs = 2; // One from LRUCache, one for the returned handle + e->in_cache = false; + e->refs = 1; // for the returned handle. memcpy(e->key_data, key.data(), key.size()); - LRU_Append(e); - usage_ += charge; - LRUHandle* old = table_.Insert(e); - if (old != NULL) { - LRU_Remove(old); - Unref(old); - } + if (capacity_ > 0) { + e->refs++; // for the cache's reference. + e->in_cache = true; + LRU_Append(&in_use_, e); + usage_ += charge; + FinishErase(table_.Insert(e)); + } // else don't cache. (Tests use capacity_==0 to turn off caching.) while (usage_ > capacity_ && lru_.next != &lru_) { LRUHandle* old = lru_.next; - LRU_Remove(old); - table_.Remove(old->key(), old->hash); - Unref(old); + assert(old->refs == 1); + bool erased = FinishErase(table_.Remove(old->key(), old->hash)); + if (!erased) { // to avoid unused variable when compiled NDEBUG + assert(erased); + } } return reinterpret_cast<Cache::Handle*>(e); } -void LRUCache::Erase(const Slice& key, uint32_t hash) { - MutexLock l(&mutex_); - LRUHandle* e = table_.Remove(key, hash); +// If e != NULL, finish removing *e from the cache; it has already been removed +// from the hash table. Return whether e != NULL. Requires mutex_ held. +bool LRUCache::FinishErase(LRUHandle* e) { if (e != NULL) { + assert(e->in_cache); LRU_Remove(e); + e->in_cache = false; + usage_ -= e->charge; Unref(e); } + return e != NULL; +} + +void LRUCache::Erase(const Slice& key, uint32_t hash) { + MutexLock l(&mutex_); + FinishErase(table_.Remove(key, hash)); +} + +void LRUCache::Prune() { + MutexLock l(&mutex_); + while (lru_.next != &lru_) { + LRUHandle* e = lru_.next; + assert(e->refs == 1); + bool erased = FinishErase(table_.Remove(e->key(), e->hash)); + if (!erased) { // to avoid unused variable when compiled NDEBUG + assert(erased); + } + } } static const int kNumShardBits = 4; @@ -314,6 +382,18 @@ class ShardedLRUCache : public Cache { MutexLock l(&id_mutex_); return ++(last_id_); } + virtual void Prune() { + for (int s = 0; s < kNumShards; s++) { + shard_[s].Prune(); + } + } + virtual size_t TotalCharge() const { + size_t total = 0; + for (int s = 0; s < kNumShards; s++) { + total += shard_[s].TotalCharge(); + } + return total; + } }; } // end anonymous namespace diff --git a/src/leveldb/util/cache_test.cc b/src/leveldb/util/cache_test.cc index 43716715a8..468f7a6425 100644 --- a/src/leveldb/util/cache_test.cc +++ b/src/leveldb/util/cache_test.cc @@ -59,6 +59,11 @@ class CacheTest { &CacheTest::Deleter)); } + Cache::Handle* InsertAndReturnHandle(int key, int value, int charge = 1) { + return cache_->Insert(EncodeKey(key), EncodeValue(value), charge, + &CacheTest::Deleter); + } + void Erase(int key) { cache_->Erase(EncodeKey(key)); } @@ -135,8 +140,11 @@ TEST(CacheTest, EntriesArePinned) { TEST(CacheTest, EvictionPolicy) { Insert(100, 101); Insert(200, 201); + Insert(300, 301); + Cache::Handle* h = cache_->Lookup(EncodeKey(300)); - // Frequently used entry must be kept around + // Frequently used entry must be kept around, + // as must things that are still in use. for (int i = 0; i < kCacheSize + 100; i++) { Insert(1000+i, 2000+i); ASSERT_EQ(2000+i, Lookup(1000+i)); @@ -144,6 +152,25 @@ TEST(CacheTest, EvictionPolicy) { } ASSERT_EQ(101, Lookup(100)); ASSERT_EQ(-1, Lookup(200)); + ASSERT_EQ(301, Lookup(300)); + cache_->Release(h); +} + +TEST(CacheTest, UseExceedsCacheSize) { + // Overfill the cache, keeping handles on all inserted entries. + std::vector<Cache::Handle*> h; + for (int i = 0; i < kCacheSize + 100; i++) { + h.push_back(InsertAndReturnHandle(1000+i, 2000+i)); + } + + // Check that all the entries can be found in the cache. + for (int i = 0; i < h.size(); i++) { + ASSERT_EQ(2000+i, Lookup(1000+i)); + } + + for (int i = 0; i < h.size(); i++) { + cache_->Release(h[i]); + } } TEST(CacheTest, HeavyEntries) { @@ -179,6 +206,19 @@ TEST(CacheTest, NewId) { ASSERT_NE(a, b); } +TEST(CacheTest, Prune) { + Insert(1, 100); + Insert(2, 200); + + Cache::Handle* handle = cache_->Lookup(EncodeKey(1)); + ASSERT_TRUE(handle); + cache_->Prune(); + cache_->Release(handle); + + ASSERT_EQ(100, Lookup(1)); + ASSERT_EQ(-1, Lookup(2)); +} + } // namespace leveldb int main(int argc, char** argv) { diff --git a/src/leveldb/util/env.cc b/src/leveldb/util/env.cc index c2600e964a..c58a0821ef 100644 --- a/src/leveldb/util/env.cc +++ b/src/leveldb/util/env.cc @@ -9,6 +9,10 @@ namespace leveldb { Env::~Env() { } +Status Env::NewAppendableFile(const std::string& fname, WritableFile** result) { + return Status::NotSupported("NewAppendableFile", fname); +} + SequentialFile::~SequentialFile() { } diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index ba2667864a..e0fca52f46 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -351,6 +351,19 @@ class PosixEnv : public Env { return s; } + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result) { + Status s; + FILE* f = fopen(fname.c_str(), "a"); + if (f == NULL) { + *result = NULL; + s = IOError(fname, errno); + } else { + *result = new PosixWritableFile(fname, f); + } + return s; + } + virtual bool FileExists(const std::string& fname) { return access(fname.c_str(), F_OK) == 0; } diff --git a/src/leveldb/util/env_win.cc b/src/leveldb/util/env_win.cc index e11a96b791..b074b7579e 100644 --- a/src/leveldb/util/env_win.cc +++ b/src/leveldb/util/env_win.cc @@ -106,7 +106,7 @@ private: class Win32WritableFile : public WritableFile { public: - Win32WritableFile(const std::string& fname); + Win32WritableFile(const std::string& fname, bool append); ~Win32WritableFile(); virtual Status Append(const Slice& data); @@ -158,6 +158,8 @@ public: RandomAccessFile** result); virtual Status NewWritableFile(const std::string& fname, WritableFile** result); + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result); virtual bool FileExists(const std::string& fname); @@ -423,17 +425,23 @@ void Win32RandomAccessFile::_CleanUp() } } -Win32WritableFile::Win32WritableFile(const std::string& fname) +Win32WritableFile::Win32WritableFile(const std::string& fname, bool append) : filename_(fname) { std::wstring path; ToWidePath(fname, path); - DWORD Flag = PathFileExistsW(path.c_str()) ? OPEN_EXISTING : CREATE_ALWAYS; + // NewAppendableFile: append to an existing file, or create a new one + // if none exists - this is OPEN_ALWAYS behavior, with + // FILE_APPEND_DATA to avoid having to manually position the file + // pointer at the end of the file. + // NewWritableFile: create a new file, delete if it exists - this is + // CREATE_ALWAYS behavior. This file is used for writing only so + // use GENERIC_WRITE. _hFile = CreateFileW(path.c_str(), - GENERIC_READ | GENERIC_WRITE, + append ? FILE_APPEND_DATA : GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_DELETE|FILE_SHARE_WRITE, NULL, - Flag, + append ? OPEN_ALWAYS : CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); // CreateFileW returns INVALID_HANDLE_VALUE in case of error, always check isEnable() before use @@ -823,7 +831,9 @@ Status Win32Env::NewLogger( const std::string& fname, Logger** result ) { Status sRet; std::string path = fname; - Win32WritableFile* pMapFile = new Win32WritableFile(ModifyPath(path)); + // Logs are opened with write semantics, not with append semantics + // (see PosixEnv::NewLogger) + Win32WritableFile* pMapFile = new Win32WritableFile(ModifyPath(path), false); if(!pMapFile->isEnable()){ delete pMapFile; *result = NULL; @@ -837,7 +847,20 @@ Status Win32Env::NewWritableFile( const std::string& fname, WritableFile** resul { Status sRet; std::string path = fname; - Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path)); + Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path), false); + if(!pFile->isEnable()){ + *result = NULL; + sRet = Status::IOError(fname,Win32::GetLastErrSz()); + }else + *result = pFile; + return sRet; +} + +Status Win32Env::NewAppendableFile( const std::string& fname, WritableFile** result ) +{ + Status sRet; + std::string path = fname; + Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path), true); if(!pFile->isEnable()){ *result = NULL; sRet = Status::IOError(fname,Win32::GetLastErrSz()); diff --git a/src/leveldb/util/options.cc b/src/leveldb/util/options.cc index 76af5b9302..8b618fb1ae 100644 --- a/src/leveldb/util/options.cc +++ b/src/leveldb/util/options.cc @@ -22,8 +22,8 @@ Options::Options() block_size(4096), block_restart_interval(16), compression(kSnappyCompression), + reuse_logs(false), filter_policy(NULL) { } - } // namespace leveldb diff --git a/src/leveldb/util/testutil.h b/src/leveldb/util/testutil.h index adad3fc1ea..d7e4583702 100644 --- a/src/leveldb/util/testutil.h +++ b/src/leveldb/util/testutil.h @@ -45,6 +45,16 @@ class ErrorEnv : public EnvWrapper { } return target()->NewWritableFile(fname, result); } + + virtual Status NewAppendableFile(const std::string& fname, + WritableFile** result) { + if (writable_file_error_) { + ++num_writable_file_errors_; + *result = NULL; + return Status::IOError(fname, "fake error"); + } + return target()->NewAppendableFile(fname, result); + } }; } // namespace test diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 5a7f6aa750..74da788ca8 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -50,6 +50,7 @@ struct IteratorComparator }; struct COrphanTx { + // When modifying, adapt the copy of this definition in tests/DoS_tests. CTransaction tx; NodeId fromPeer; int64_t nTimeExpire; @@ -1517,7 +1518,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } CBlock block; - assert(ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())); + bool ret = ReadBlockFromDisk(block, it->second, chainparams.GetConsensus()); + assert(ret); BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { @@ -1594,8 +1596,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, deque<COutPoint> vWorkQueue; vector<uint256> vEraseQueue; - CTransaction tx; - vRecv >> tx; + CTransaction tx(deserialize, vRecv); CInv inv(MSG_TX, tx.GetHash()); pfrom->AddInventoryKnown(inv); @@ -2730,7 +2731,8 @@ bool SendMessages(CNode* pto, CConnman& connman) vHeaders.front().GetHash().ToString(), pto->id); //TODO: Shouldn't need to reload block from disk, but requires refactor CBlock block; - assert(ReadBlockFromDisk(block, pBestIndex, consensusParams)); + bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); + assert(ret); CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); @@ -2995,7 +2997,8 @@ bool SendMessages(CNode* pto, CConnman& connman) CAmount currentFilter = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); int64_t timeNow = GetTimeMicros(); if (timeNow > pto->nextSendTimeFeeFilter) { - static FeeFilterRounder filterRounder(::minRelayTxFee); + static CFeeRate default_feerate(DEFAULT_MIN_RELAY_TX_FEE); + static FeeFilterRounder filterRounder(default_feerate); CAmount filterToSend = filterRounder.round(currentFilter); if (filterToSend != pto->lastSentFeeFilter) { connman.PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); diff --git a/src/netbase.cpp b/src/netbase.cpp index 9118584b80..da94fd4d13 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -292,7 +292,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5Init.push_back(0x01); // # METHODS vSocks5Init.push_back(0x00); // X'00' NO AUTHENTICATION REQUIRED } - ssize_t ret = send(hSocket, (const char*)begin_ptr(vSocks5Init), vSocks5Init.size(), MSG_NOSIGNAL); + ssize_t ret = send(hSocket, (const char*)vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5Init.size()) { CloseSocket(hSocket); return error("Error sending to proxy"); @@ -317,7 +317,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); vAuth.push_back(auth->password.size()); vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); - ret = send(hSocket, (const char*)begin_ptr(vAuth), vAuth.size(), MSG_NOSIGNAL); + ret = send(hSocket, (const char*)vAuth.data(), vAuth.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vAuth.size()) { CloseSocket(hSocket); return error("Error sending authentication to proxy"); @@ -347,7 +347,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); vSocks5.push_back((port >> 8) & 0xFF); vSocks5.push_back((port >> 0) & 0xFF); - ret = send(hSocket, (const char*)begin_ptr(vSocks5), vSocks5.size(), MSG_NOSIGNAL); + ret = send(hSocket, (const char*)vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5.size()) { CloseSocket(hSocket); return error("Error sending to proxy"); diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 91f4d29488..11d7eace55 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -62,9 +62,9 @@ uint256 CMutableTransaction::GetHash() const return SerializeHash(*this, SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); } -void CTransaction::UpdateHash() const +uint256 CTransaction::ComputeHash() const { - *const_cast<uint256*>(&hash) = SerializeHash(*this, SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); + return SerializeHash(*this, SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); } uint256 CTransaction::GetWitnessHash() const @@ -72,25 +72,10 @@ uint256 CTransaction::GetWitnessHash() const return SerializeHash(*this, SER_GETHASH, 0); } -CTransaction::CTransaction() : nVersion(CTransaction::CURRENT_VERSION), vin(), vout(), nLockTime(0) { } - -CTransaction::CTransaction(const CMutableTransaction &tx) : nVersion(tx.nVersion), vin(tx.vin), vout(tx.vout), wit(tx.wit), nLockTime(tx.nLockTime) { - UpdateHash(); -} - -CTransaction::CTransaction(CMutableTransaction &&tx) : nVersion(tx.nVersion), vin(std::move(tx.vin)), vout(std::move(tx.vout)), wit(std::move(tx.wit)), nLockTime(tx.nLockTime) { - UpdateHash(); -} - -CTransaction& CTransaction::operator=(const CTransaction &tx) { - *const_cast<int*>(&nVersion) = tx.nVersion; - *const_cast<std::vector<CTxIn>*>(&vin) = tx.vin; - *const_cast<std::vector<CTxOut>*>(&vout) = tx.vout; - *const_cast<CTxWitness*>(&wit) = tx.wit; - *const_cast<unsigned int*>(&nLockTime) = tx.nLockTime; - *const_cast<uint256*>(&hash) = tx.hash; - return *this; -} +/* For backward compatibility, the hash is initialized to 0. TODO: remove the need for this default constructor entirely. */ +CTransaction::CTransaction() : nVersion(CTransaction::CURRENT_VERSION), vin(), vout(), nLockTime(0), hash() {} +CTransaction::CTransaction(const CMutableTransaction &tx) : nVersion(tx.nVersion), vin(tx.vin), vout(tx.vout), wit(tx.wit), nLockTime(tx.nLockTime), hash(ComputeHash()) {} +CTransaction::CTransaction(CMutableTransaction &&tx) : nVersion(tx.nVersion), vin(std::move(tx.vin)), vout(std::move(tx.vout)), wit(std::move(tx.wit)), nLockTime(tx.nLockTime), hash(ComputeHash()) {} CAmount CTransaction::GetValueOut() const { diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 0fa85a1519..66fefafef5 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -286,73 +286,81 @@ struct CMutableTransaction; * - CTxWitness wit; * - uint32_t nLockTime */ -template<typename Stream, typename Operation, typename TxType> -inline void SerializeTransaction(TxType& tx, Stream& s, Operation ser_action) { +template<typename Stream, typename TxType> +inline void UnserializeTransaction(TxType& tx, Stream& s) { const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); - READWRITE(*const_cast<int32_t*>(&tx.nVersion)); + s >> tx.nVersion; unsigned char flags = 0; - if (ser_action.ForRead()) { - const_cast<std::vector<CTxIn>*>(&tx.vin)->clear(); - const_cast<std::vector<CTxOut>*>(&tx.vout)->clear(); - const_cast<CTxWitness*>(&tx.wit)->SetNull(); - /* Try to read the vin. In case the dummy is there, this will be read as an empty vector. */ - READWRITE(*const_cast<std::vector<CTxIn>*>(&tx.vin)); - if (tx.vin.size() == 0 && fAllowWitness) { - /* We read a dummy or an empty vin. */ - READWRITE(flags); - if (flags != 0) { - READWRITE(*const_cast<std::vector<CTxIn>*>(&tx.vin)); - READWRITE(*const_cast<std::vector<CTxOut>*>(&tx.vout)); - } - } else { - /* We read a non-empty vin. Assume a normal vout follows. */ - READWRITE(*const_cast<std::vector<CTxOut>*>(&tx.vout)); - } - if ((flags & 1) && fAllowWitness) { - /* The witness flag is present, and we support witnesses. */ - flags ^= 1; - const_cast<CTxWitness*>(&tx.wit)->vtxinwit.resize(tx.vin.size()); - READWRITE(tx.wit); - } - if (flags) { - /* Unknown flag in the serialization */ - throw std::ios_base::failure("Unknown transaction optional data"); + tx.vin.clear(); + tx.vout.clear(); + tx.wit.SetNull(); + /* Try to read the vin. In case the dummy is there, this will be read as an empty vector. */ + s >> tx.vin; + if (tx.vin.size() == 0 && fAllowWitness) { + /* We read a dummy or an empty vin. */ + s >> flags; + if (flags != 0) { + s >> tx.vin; + s >> tx.vout; } } else { - // Consistency check - assert(tx.wit.vtxinwit.size() <= tx.vin.size()); - if (fAllowWitness) { - /* Check whether witnesses need to be serialized. */ - if (!tx.wit.IsNull()) { - flags |= 1; - } - } - if (flags) { - /* Use extended format in case witnesses are to be serialized. */ - std::vector<CTxIn> vinDummy; - READWRITE(vinDummy); - READWRITE(flags); + /* We read a non-empty vin. Assume a normal vout follows. */ + s >> tx.vout; + } + if ((flags & 1) && fAllowWitness) { + /* The witness flag is present, and we support witnesses. */ + flags ^= 1; + tx.wit.vtxinwit.resize(tx.vin.size()); + s >> tx.wit; + } + if (flags) { + /* Unknown flag in the serialization */ + throw std::ios_base::failure("Unknown transaction optional data"); + } + s >> tx.nLockTime; +} + +template<typename Stream, typename TxType> +inline void SerializeTransaction(const TxType& tx, Stream& s) { + const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); + + s << tx.nVersion; + unsigned char flags = 0; + // Consistency check + assert(tx.wit.vtxinwit.size() <= tx.vin.size()); + if (fAllowWitness) { + /* Check whether witnesses need to be serialized. */ + if (!tx.wit.IsNull()) { + flags |= 1; } - READWRITE(*const_cast<std::vector<CTxIn>*>(&tx.vin)); - READWRITE(*const_cast<std::vector<CTxOut>*>(&tx.vout)); - if (flags & 1) { - const_cast<CTxWitness*>(&tx.wit)->vtxinwit.resize(tx.vin.size()); - READWRITE(tx.wit); + } + if (flags) { + /* Use extended format in case witnesses are to be serialized. */ + std::vector<CTxIn> vinDummy; + s << vinDummy; + s << flags; + } + s << tx.vin; + s << tx.vout; + if (flags & 1) { + for (size_t i = 0; i < tx.vin.size(); i++) { + if (i < tx.wit.vtxinwit.size()) { + s << tx.wit.vtxinwit[i]; + } else { + s << CTxInWitness(); + } } } - READWRITE(*const_cast<uint32_t*>(&tx.nLockTime)); + s << tx.nLockTime; } + /** The basic transaction that is broadcasted on the network and contained in * blocks. A transaction can contain multiple inputs and outputs. */ class CTransaction { -private: - /** Memory only. */ - const uint256 hash; - public: // Default transaction version. static const int32_t CURRENT_VERSION=1; @@ -374,6 +382,13 @@ public: CTxWitness wit; // Not const: can change without invalidating the txid cache const uint32_t nLockTime; +private: + /** Memory only. */ + const uint256 hash; + + uint256 ComputeHash() const; + +public: /** Construct a CTransaction that qualifies as IsNull() */ CTransaction(); @@ -381,18 +396,13 @@ public: CTransaction(const CMutableTransaction &tx); CTransaction(CMutableTransaction &&tx); - CTransaction& operator=(const CTransaction& tx); - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - SerializeTransaction(*this, s, ser_action); - if (ser_action.ForRead()) { - UpdateHash(); - } + template <typename Stream> + inline void Serialize(Stream& s) const { + SerializeTransaction(*this, s); } + /** This deserializing constructor is provided instead of an Unserialize method. + * Unserialize is not possible, since it would require overwriting const fields. */ template <typename Stream> CTransaction(deserialize_type, Stream& s) : CTransaction(CMutableTransaction(deserialize, s)) {} @@ -417,7 +427,7 @@ public: // Compute modified tx size for priority calculation (optionally given tx size) unsigned int CalculateModifiedSize(unsigned int nTxSize=0) const; - + /** * Get the total transaction size in bytes, including witness data. * "Total Size" defined in BIP141 and BIP144. @@ -441,8 +451,6 @@ public: } std::string ToString() const; - - void UpdateHash() const; }; /** A mutable version of CTransaction. */ @@ -457,11 +465,15 @@ struct CMutableTransaction CMutableTransaction(); CMutableTransaction(const CTransaction& tx); - ADD_SERIALIZE_METHODS; + template <typename Stream> + inline void Serialize(Stream& s) const { + SerializeTransaction(*this, s); + } - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - SerializeTransaction(*this, s, ser_action); + + template <typename Stream> + inline void Unserialize(Stream& s) { + UnserializeTransaction(*this, s); } template <typename Stream> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 6112a1d256..651ff84293 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -46,7 +46,6 @@ #include <QMenuBar> #include <QMessageBox> #include <QMimeData> -#include <QProgressBar> #include <QProgressDialog> #include <QSettings> #include <QShortcut> @@ -199,8 +198,8 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *_platformStyle, const NetworkStyle * unitDisplayControl = new UnitDisplayStatusBarControl(platformStyle); labelWalletEncryptionIcon = new QLabel(); labelWalletHDStatusIcon = new QLabel(); - connectionsControl = new NetworkToggleStatusBarControl(); - labelBlocksIcon = new QLabel(); + connectionsControl = new GUIUtil::ClickableLabel(); + labelBlocksIcon = new GUIUtil::ClickableLabel(); if(enableWallet) { frameBlocksLayout->addStretch(); @@ -244,10 +243,15 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *_platformStyle, const NetworkStyle * // Subscribe to notifications from core subscribeToCoreSignals(); + connect(connectionsControl, SIGNAL(clicked(QPoint)), this, SLOT(toggleNetworkActive())); + modalOverlay = new ModalOverlay(this->centralWidget()); #ifdef ENABLE_WALLET - if(enableWallet) + if(enableWallet) { connect(walletFrame, SIGNAL(requestedSyncWarningInfo()), this, SLOT(showModalOverlay())); + connect(labelBlocksIcon, SIGNAL(clicked(QPoint)), this, SLOT(showModalOverlay())); + connect(progressBar, SIGNAL(clicked(QPoint)), this, SLOT(showModalOverlay())); + } #endif } @@ -490,7 +494,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) } #endif // ENABLE_WALLET unitDisplayControl->setOptionsModel(_clientModel->getOptionsModel()); - connectionsControl->setClientModel(_clientModel); OptionsModel* optionsModel = _clientModel->getOptionsModel(); if(optionsModel) @@ -517,7 +520,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) walletFrame->setClientModel(nullptr); #endif // ENABLE_WALLET unitDisplayControl->setOptionsModel(nullptr); - connectionsControl->setClientModel(nullptr); } } @@ -1136,8 +1138,8 @@ void BitcoinGUI::setTrayIconVisible(bool fHideTrayIcon) void BitcoinGUI::showModalOverlay() { - if (modalOverlay) - modalOverlay->showHide(false, true); + if (modalOverlay && (progressBar->isVisible() || modalOverlay->isLayerVisible())) + modalOverlay->toggleVisibility(); } static bool ThreadSafeMessageBox(BitcoinGUI *gui, const std::string& message, const std::string& caption, unsigned int style) @@ -1171,6 +1173,13 @@ void BitcoinGUI::unsubscribeFromCoreSignals() uiInterface.ThreadSafeQuestion.disconnect(boost::bind(ThreadSafeMessageBox, this, _1, _3, _4)); } +void BitcoinGUI::toggleNetworkActive() +{ + if (clientModel) { + clientModel->setNetworkActive(!clientModel->getNetworkActive()); + } +} + UnitDisplayStatusBarControl::UnitDisplayStatusBarControl(const PlatformStyle *platformStyle) : optionsModel(0), menu(0) @@ -1244,16 +1253,3 @@ void UnitDisplayStatusBarControl::onMenuSelection(QAction* action) optionsModel->setDisplayUnit(action->data()); } } - -void NetworkToggleStatusBarControl::mousePressEvent(QMouseEvent *event) -{ - if (clientModel) { - clientModel->setNetworkActive(!clientModel->getNetworkActive()); - } -} - -/** Lets the control know about the Client Model */ -void NetworkToggleStatusBarControl::setClientModel(ClientModel *_clientModel) -{ - this->clientModel = _clientModel; -} diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 1b02e77fc4..59540bfe6b 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -26,7 +26,6 @@ class PlatformStyle; class RPCConsole; class SendCoinsRecipient; class UnitDisplayStatusBarControl; -class NetworkToggleStatusBarControl; class WalletFrame; class WalletModel; class HelpMessageDialog; @@ -86,7 +85,7 @@ private: UnitDisplayStatusBarControl *unitDisplayControl; QLabel *labelWalletEncryptionIcon; QLabel *labelWalletHDStatusIcon; - NetworkToggleStatusBarControl *connectionsControl; + QLabel *connectionsControl; QLabel *labelBlocksIcon; QLabel *progressBarLabel; QProgressBar *progressBar; @@ -238,6 +237,9 @@ private Q_SLOTS: /** When hideTrayIcon setting is changed in OptionsModel hide or show the icon accordingly. */ void setTrayIconVisible(bool); + /** Toggle networking */ + void toggleNetworkActive(); + void showModalOverlay(); }; @@ -270,17 +272,4 @@ private Q_SLOTS: void onMenuSelection(QAction* action); }; -class NetworkToggleStatusBarControl : public QLabel -{ - Q_OBJECT - -public: - void setClientModel(ClientModel *clientModel); -protected: - void mousePressEvent(QMouseEvent *event); - -private: - ClientModel *clientModel; -}; - #endif // BITCOIN_QT_BITCOINGUI_H diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 704e2e2569..4d34bf95e9 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -470,21 +470,21 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nQuantity++; // Amount - nAmount += out.tx->vout[out.i].nValue; + nAmount += out.tx->tx->vout[out.i].nValue; // Priority - dPriorityInputs += (double)out.tx->vout[out.i].nValue * (out.nDepth+1); + dPriorityInputs += (double)out.tx->tx->vout[out.i].nValue * (out.nDepth+1); // Bytes CTxDestination address; int witnessversion = 0; std::vector<unsigned char> witnessprogram; - if (out.tx->vout[out.i].scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) + if (out.tx->tx->vout[out.i].scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { nBytesInputs += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); fWitness = true; } - else if(ExtractDestination(out.tx->vout[out.i].scriptPubKey, address)) + else if(ExtractDestination(out.tx->tx->vout[out.i].scriptPubKey, address)) { CPubKey pubkey; CKeyID *keyid = boost::get<CKeyID>(&address); @@ -677,7 +677,7 @@ void CoinControlDialog::updateView() CAmount nSum = 0; int nChildren = 0; BOOST_FOREACH(const COutput& out, coins.second) { - nSum += out.tx->vout[out.i].nValue; + nSum += out.tx->tx->vout[out.i].nValue; nChildren++; CCoinControlWidgetItem *itemOutput; @@ -689,7 +689,7 @@ void CoinControlDialog::updateView() // address CTxDestination outputAddress; QString sAddress = ""; - if(ExtractDestination(out.tx->vout[out.i].scriptPubKey, outputAddress)) + if(ExtractDestination(out.tx->tx->vout[out.i].scriptPubKey, outputAddress)) { sAddress = QString::fromStdString(CBitcoinAddress(outputAddress).ToString()); @@ -714,8 +714,8 @@ void CoinControlDialog::updateView() } // amount - itemOutput->setText(COLUMN_AMOUNT, BitcoinUnits::format(nDisplayUnit, out.tx->vout[out.i].nValue)); - itemOutput->setData(COLUMN_AMOUNT, Qt::UserRole, QVariant((qlonglong)out.tx->vout[out.i].nValue)); // padding so that sorting works correctly + itemOutput->setText(COLUMN_AMOUNT, BitcoinUnits::format(nDisplayUnit, out.tx->tx->vout[out.i].nValue)); + itemOutput->setData(COLUMN_AMOUNT, Qt::UserRole, QVariant((qlonglong)out.tx->tx->vout[out.i].nValue)); // padding so that sorting works correctly // date itemOutput->setText(COLUMN_DATE, GUIUtil::dateTimeStr(out.tx->GetTxTime())); diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 23f80ebb86..8132e4fe0d 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -55,6 +55,7 @@ #include <QSettings> #include <QTextDocument> // for Qt::mightBeRichText #include <QThread> +#include <QMouseEvent> #if QT_VERSION < 0x050000 #include <QUrl> @@ -116,7 +117,7 @@ static std::string DummyAddress(const CChainParams ¶ms) std::vector<unsigned char> sourcedata = params.Base58Prefix(CChainParams::PUBKEY_ADDRESS); sourcedata.insert(sourcedata.end(), dummydata, dummydata + sizeof(dummydata)); for(int i=0; i<256; ++i) { // Try every trailing byte - std::string s = EncodeBase58(begin_ptr(sourcedata), end_ptr(sourcedata)); + std::string s = EncodeBase58(sourcedata.data(), sourcedata.data() + sourcedata.size()); if (!CBitcoinAddress(s).IsValid()) return s; sourcedata[sourcedata.size()-1] += 1; @@ -986,4 +987,15 @@ QString formateNiceTimeOffset(qint64 secs) } return timeBehindText; } + +void ClickableLabel::mouseReleaseEvent(QMouseEvent *event) +{ + Q_EMIT clicked(event->pos()); +} + +void ClickableProgressBar::mouseReleaseEvent(QMouseEvent *event) +{ + Q_EMIT clicked(event->pos()); +} + } // namespace GUIUtil diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 8f1f3fbb2c..4ea2aa36e7 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -14,6 +14,7 @@ #include <QProgressBar> #include <QString> #include <QTableView> +#include <QLabel> #include <boost/filesystem.hpp> @@ -201,18 +202,44 @@ namespace GUIUtil QString formateNiceTimeOffset(qint64 secs); + class ClickableLabel : public QLabel + { + Q_OBJECT + + Q_SIGNALS: + /** Emitted when the label is clicked. The relative mouse coordinates of the click are + * passed to the signal. + */ + void clicked(const QPoint& point); + protected: + void mouseReleaseEvent(QMouseEvent *event); + }; + + class ClickableProgressBar : public QProgressBar + { + Q_OBJECT + + Q_SIGNALS: + /** Emitted when the progressbar is clicked. The relative mouse coordinates of the click are + * passed to the signal. + */ + void clicked(const QPoint& point); + protected: + void mouseReleaseEvent(QMouseEvent *event); + }; + #if defined(Q_OS_MAC) && QT_VERSION >= 0x050000 // workaround for Qt OSX Bug: // https://bugreports.qt-project.org/browse/QTBUG-15631 // QProgressBar uses around 10% CPU even when app is in background - class ProgressBar : public QProgressBar + class ProgressBar : public ClickableProgressBar { bool event(QEvent *e) { return (e->type() != QEvent::StyleAnimationUpdate) ? QProgressBar::event(e) : false; } }; #else - typedef QProgressBar ProgressBar; + typedef ClickableProgressBar ProgressBar; #endif } // namespace GUIUtil diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 1a843a07ac..3e8282e63d 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -137,6 +137,13 @@ void ModalOverlay::tipUpdate(int count, const QDateTime& blockDate, double nVeri } } +void ModalOverlay::toggleVisibility() +{ + showHide(layerIsVisible, true); + if (!layerIsVisible) + userClosed = true; +} + void ModalOverlay::showHide(bool hide, bool userRequested) { if ( (layerIsVisible && !hide) || (!layerIsVisible && hide) || (!hide && userClosed && !userRequested)) diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index 66c0aa78cf..70d37b87ad 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -25,9 +25,11 @@ public Q_SLOTS: void tipUpdate(int count, const QDateTime& blockDate, double nVerificationProgress); void setKnownBestHeight(int count, const QDateTime& blockDate); + void toggleVisibility(); // will show or hide the modal layer void showHide(bool hide = false, bool userRequested = false); void closeClicked(); + bool isLayerVisible() { return layerIsVisible; } protected: bool eventFilter(QObject * obj, QEvent * ev); diff --git a/src/qt/paymentrequestplus.cpp b/src/qt/paymentrequestplus.cpp index 20e1f79ffa..82be4d831f 100644 --- a/src/qt/paymentrequestplus.cpp +++ b/src/qt/paymentrequestplus.cpp @@ -159,14 +159,24 @@ bool PaymentRequestPlus::getMerchant(X509_STORE* certStore, QString& merchant) c std::string data_to_verify; // Everything but the signature rcopy.SerializeToString(&data_to_verify); - EVP_MD_CTX ctx; +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + if (!ctx) throw SSLVerifyError("Error allocating OpenSSL context."); +#else + EVP_MD_CTX _ctx; + EVP_MD_CTX *ctx; + ctx = &_ctx; +#endif EVP_PKEY *pubkey = X509_get_pubkey(signing_cert); - EVP_MD_CTX_init(&ctx); - if (!EVP_VerifyInit_ex(&ctx, digestAlgorithm, NULL) || - !EVP_VerifyUpdate(&ctx, data_to_verify.data(), data_to_verify.size()) || - !EVP_VerifyFinal(&ctx, (const unsigned char*)paymentRequest.signature().data(), (unsigned int)paymentRequest.signature().size(), pubkey)) { + EVP_MD_CTX_init(ctx); + if (!EVP_VerifyInit_ex(ctx, digestAlgorithm, NULL) || + !EVP_VerifyUpdate(ctx, data_to_verify.data(), data_to_verify.size()) || + !EVP_VerifyFinal(ctx, (const unsigned char*)paymentRequest.signature().data(), (unsigned int)paymentRequest.signature().size(), pubkey)) { throw SSLVerifyError("Bad signature, invalid payment request."); } +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + EVP_MD_CTX_free(ctx); +#endif // OpenSSL API for getting human printable strings from certs is baroque. int textlen = X509_NAME_get_text_by_NID(certname, NID_commonName, NULL, 0); diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 520d229901..4ba2f2615f 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -515,7 +515,7 @@ void RPCConsole::setClientModel(ClientModel *model) // peer table signal handling - update peer details when new nodes are added to the model connect(model->getPeerTableModel(), SIGNAL(layoutChanged()), this, SLOT(peerLayoutChanged())); // peer table signal handling - cache selected node ids - connect(model->getPeerTableModel(), SIGNAL(layoutAboutToChange()), this, SLOT(peerLayoutAboutToChange())); + connect(model->getPeerTableModel(), SIGNAL(layoutAboutToBeChanged()), this, SLOT(peerLayoutAboutToChange())); // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); @@ -650,13 +650,18 @@ void RPCConsole::clear(bool clearHistory) "td.message { font-family: %1; font-size: %2; white-space:pre-wrap; } " "td.cmd-request { color: #006060; } " "td.cmd-error { color: red; } " + ".secwarning { color: red; }" "b { color: #006060; } " ).arg(fixedFontInfo.family(), QString("%1pt").arg(consoleFontSize)) ); message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + tr("Use up and down arrows to navigate history, and <b>Ctrl-L</b> to clear screen.") + "<br>" + - tr("Type <b>help</b> for an overview of available commands.")), true); + tr("Type <b>help</b> for an overview of available commands.")) + + "<br><span class=\"secwarning\">" + + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramification of a command.") + + "</span>", + true); } void RPCConsole::keyPressEvent(QKeyEvent *event) @@ -778,7 +783,6 @@ void RPCConsole::startExecutor() connect(this, SIGNAL(stopExecutor()), &thread, SLOT(quit())); // - queue executor for deletion (in execution thread) connect(&thread, SIGNAL(finished()), executor, SLOT(deleteLater()), Qt::DirectConnection); - connect(&thread, SIGNAL(finished()), this, SLOT(test()), Qt::DirectConnection); // Default implementation of QThread::run() simply spins up an event loop in the thread, // which is what we want. @@ -1008,11 +1012,11 @@ void RPCConsole::disconnectSelectedNode() return; // Get selected peer addresses - QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, 0); + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data(PeerTableModel::NetNodeId).toInt(); + NodeId id = nodes.at(i).data().toInt(); // Find the node, disconnect it and clear the selected node if(g_connman->DisconnectNode(id)) clearSelectedNode(); @@ -1025,11 +1029,11 @@ void RPCConsole::banSelectedNode(int bantime) return; // Get selected peer addresses - QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, 0); + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data(PeerTableModel::NetNodeId).toInt(); + NodeId id = nodes.at(i).data().toInt(); // Get currently selected peer address int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); @@ -1052,11 +1056,11 @@ void RPCConsole::unbanSelectedNode() return; // Get selected ban addresses - QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->banlistWidget, 0); + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->banlistWidget, BanTableModel::Address); for(int i = 0; i < nodes.count(); i++) { // Get currently selected ban address - QString strNode = nodes.at(i).data(BanTableModel::Address).toString(); + QString strNode = nodes.at(i).data().toString(); CSubNet possibleSubnet; LookupSubNet(strNode.toStdString().c_str(), possibleSubnet); @@ -1090,8 +1094,3 @@ void RPCConsole::setTabFocus(enum TabTypes tabType) { ui->tabWidget->setCurrentIndex(tabType); } - -void RPCConsole::on_toggleNetworkActiveButton_clicked() -{ - clientModel->setNetworkActive(!clientModel->getNetworkActive()); -} diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 344d5ecb98..ab8c3dc914 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -62,8 +62,6 @@ protected: private Q_SLOTS: void on_lineEdit_returnPressed(); void on_tabWidget_currentChanged(int index); - /** toggle network activity */ - void on_toggleNetworkActiveButton_clicked(); /** open the debug.log from the current datadir */ void on_openDebugLogfileButton_clicked(); /** change the time range of the network traffic graph */ diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 01f6ab867f..e7d03a6119 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -26,10 +26,10 @@ QString TransactionDesc::FormatTxStatus(const CWalletTx& wtx) AssertLockHeld(cs_main); if (!CheckFinalTx(wtx)) { - if (wtx.nLockTime < LOCKTIME_THRESHOLD) - return tr("Open for %n more block(s)", "", wtx.nLockTime - chainActive.Height()); + if (wtx.tx->nLockTime < LOCKTIME_THRESHOLD) + return tr("Open for %n more block(s)", "", wtx.tx->nLockTime - chainActive.Height()); else - return tr("Open until %1").arg(GUIUtil::dateTimeStr(wtx.nLockTime)); + return tr("Open until %1").arg(GUIUtil::dateTimeStr(wtx.tx->nLockTime)); } else { @@ -133,7 +133,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco // Coinbase // CAmount nUnmatured = 0; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) nUnmatured += wallet->GetCredit(txout, ISMINE_ALL); strHTML += "<b>" + tr("Credit") + ":</b> "; if (wtx.IsInMainChain()) @@ -152,14 +152,14 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco else { isminetype fAllFromMe = ISMINE_SPENDABLE; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { isminetype mine = wallet->IsMine(txin); if(fAllFromMe > mine) fAllFromMe = mine; } isminetype fAllToMe = ISMINE_SPENDABLE; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { isminetype mine = wallet->IsMine(txout); if(fAllToMe > mine) fAllToMe = mine; @@ -173,7 +173,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco // // Debit // - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { // Ignore change isminetype toSelf = wallet->IsMine(txout); @@ -212,7 +212,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco strHTML += "<b>" + tr("Total credit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, nValue) + "<br>"; } - CAmount nTxFee = nDebit - wtx.GetValueOut(); + CAmount nTxFee = nDebit - wtx.tx->GetValueOut(); if (nTxFee > 0) strHTML += "<b>" + tr("Transaction fee") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, -nTxFee) + "<br>"; } @@ -221,10 +221,10 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco // // Mixed debit transaction // - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) if (wallet->IsMine(txin)) strHTML += "<b>" + tr("Debit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, -wallet->GetDebit(txin, ISMINE_ALL)) + "<br>"; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) if (wallet->IsMine(txout)) strHTML += "<b>" + tr("Credit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, wallet->GetCredit(txout, ISMINE_ALL)) + "<br>"; } @@ -241,7 +241,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco strHTML += "<br><b>" + tr("Comment") + ":</b><br>" + GUIUtil::HtmlEscape(wtx.mapValue["comment"], true) + "<br>"; strHTML += "<b>" + tr("Transaction ID") + ":</b> " + rec->getTxID() + "<br>"; - strHTML += "<b>" + tr("Transaction total size") + ":</b> " + QString::number(wtx.GetTotalSize()) + " bytes<br>"; + strHTML += "<b>" + tr("Transaction total size") + ":</b> " + QString::number(wtx.tx->GetTotalSize()) + " bytes<br>"; strHTML += "<b>" + tr("Output index") + ":</b> " + QString::number(rec->getOutputIndex()) + "<br>"; // Message from normal bitcoin:URI (bitcoin:123...?message=example) @@ -276,20 +276,20 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco if (fDebug) { strHTML += "<hr><br>" + tr("Debug information") + "<br><br>"; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) if(wallet->IsMine(txin)) strHTML += "<b>" + tr("Debit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, -wallet->GetDebit(txin, ISMINE_ALL)) + "<br>"; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) if(wallet->IsMine(txout)) strHTML += "<b>" + tr("Credit") + ":</b> " + BitcoinUnits::formatHtmlWithUnit(unit, wallet->GetCredit(txout, ISMINE_ALL)) + "<br>"; strHTML += "<br><b>" + tr("Transaction") + ":</b><br>"; - strHTML += GUIUtil::HtmlEscape(wtx.ToString(), true); + strHTML += GUIUtil::HtmlEscape(wtx.tx->ToString(), true); strHTML += "<br><b>" + tr("Inputs") + ":</b>"; strHTML += "<ul>"; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { COutPoint prevout = txin.prevout; diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 7d7bce7be2..a1d1422317 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -47,7 +47,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet * // // Credit // - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { isminetype mine = wallet->IsMine(txout); if(mine) @@ -83,7 +83,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet * { bool involvesWatchAddress = false; isminetype fAllFromMe = ISMINE_SPENDABLE; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { isminetype mine = wallet->IsMine(txin); if(mine & ISMINE_WATCH_ONLY) involvesWatchAddress = true; @@ -91,7 +91,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet * } isminetype fAllToMe = ISMINE_SPENDABLE; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { isminetype mine = wallet->IsMine(txout); if(mine & ISMINE_WATCH_ONLY) involvesWatchAddress = true; @@ -112,11 +112,11 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet * // // Debit // - CAmount nTxFee = nDebit - wtx.GetValueOut(); + CAmount nTxFee = nDebit - wtx.tx->GetValueOut(); - for (unsigned int nOut = 0; nOut < wtx.vout.size(); nOut++) + for (unsigned int nOut = 0; nOut < wtx.tx->vout.size(); nOut++) { - const CTxOut& txout = wtx.vout[nOut]; + const CTxOut& txout = wtx.tx->vout[nOut]; TransactionRecord sub(hash, nTime); sub.idx = parts.size(); sub.involvesWatchAddress = involvesWatchAddress; @@ -190,15 +190,15 @@ void TransactionRecord::updateStatus(const CWalletTx &wtx) if (!CheckFinalTx(wtx)) { - if (wtx.nLockTime < LOCKTIME_THRESHOLD) + if (wtx.tx->nLockTime < LOCKTIME_THRESHOLD) { status.status = TransactionStatus::OpenUntilBlock; - status.open_for = wtx.nLockTime - chainActive.Height(); + status.open_for = wtx.tx->nLockTime - chainActive.Height(); } else { status.status = TransactionStatus::OpenUntilDate; - status.open_for = wtx.nLockTime; + status.open_for = wtx.tx->nLockTime; } } // For generated transactions, determine maturity diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index ab6c4b9274..afc72fae69 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -67,7 +67,7 @@ CAmount WalletModel::getBalance(const CCoinControl *coinControl) const wallet->AvailableCoins(vCoins, true, coinControl); BOOST_FOREACH(const COutput& out, vCoins) if(out.fSpendable) - nBalance += out.tx->vout[out.i].nValue; + nBalance += out.tx->tx->vout[out.i].nValue; return nBalance; } @@ -335,9 +335,8 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran if(!wallet->CommitTransaction(*newTx, *keyChange, g_connman.get(), state)) return SendCoinsReturn(TransactionCommitFailed, QString::fromStdString(state.GetRejectReason())); - CTransaction* t = (CTransaction*)newTx; CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << *t; + ssTx << *newTx->tx; transaction_array.append(&(ssTx[0]), ssTx.size()); } @@ -609,7 +608,7 @@ void WalletModel::listCoins(std::map<QString, std::vector<COutput> >& mapCoins) int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); if (nDepth < 0) continue; COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true, true); - if (outpoint.n < out.tx->vout.size() && wallet->IsMine(out.tx->vout[outpoint.n]) == ISMINE_SPENDABLE) + if (outpoint.n < out.tx->tx->vout.size() && wallet->IsMine(out.tx->tx->vout[outpoint.n]) == ISMINE_SPENDABLE) vCoins.push_back(out); } @@ -617,14 +616,14 @@ void WalletModel::listCoins(std::map<QString, std::vector<COutput> >& mapCoins) { COutput cout = out; - while (wallet->IsChange(cout.tx->vout[cout.i]) && cout.tx->vin.size() > 0 && wallet->IsMine(cout.tx->vin[0])) + while (wallet->IsChange(cout.tx->tx->vout[cout.i]) && cout.tx->tx->vin.size() > 0 && wallet->IsMine(cout.tx->tx->vin[0])) { - if (!wallet->mapWallet.count(cout.tx->vin[0].prevout.hash)) break; - cout = COutput(&wallet->mapWallet[cout.tx->vin[0].prevout.hash], cout.tx->vin[0].prevout.n, 0, true, true); + if (!wallet->mapWallet.count(cout.tx->tx->vin[0].prevout.hash)) break; + cout = COutput(&wallet->mapWallet[cout.tx->tx->vin[0].prevout.hash], cout.tx->tx->vin[0].prevout.n, 0, true, true); } CTxDestination address; - if(!out.fSpendable || !ExtractDestination(cout.tx->vout[cout.i].scriptPubKey, address)) + if(!out.fSpendable || !ExtractDestination(cout.tx->tx->vout[cout.i].scriptPubKey, address)) continue; mapCoins[QString::fromStdString(CBitcoinAddress(address).ToString())].push_back(out); } diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index fdec6a1c86..f9a95bed42 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -64,7 +64,7 @@ void WalletModelTransaction::reassignAmounts(int nChangePosRet) if (out.amount() <= 0) continue; if (i == nChangePosRet) i++; - subtotal += walletTransaction->vout[i].nValue; + subtotal += walletTransaction->tx->vout[i].nValue; i++; } rcp.amount = subtotal; @@ -73,7 +73,7 @@ void WalletModelTransaction::reassignAmounts(int nChangePosRet) { if (i == nChangePosRet) i++; - rcp.amount = walletTransaction->vout[i].nValue; + rcp.amount = walletTransaction->tx->vout[i].nValue; i++; } } diff --git a/src/random.cpp b/src/random.cpp index aa027e49c4..c2605b45bd 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -11,7 +11,6 @@ #include "compat.h" // for Windows API #include <wincrypt.h> #endif -#include "serialize.h" // for begin_ptr(vec) #include "util.h" // for LogPrint() #include "utilstrencodings.h" // for GetTime() @@ -72,15 +71,15 @@ static void RandAddSeedPerfmon() const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data while (true) { nSize = vData.size(); - ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", NULL, NULL, begin_ptr(vData), &nSize); + ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", NULL, NULL, vData.data(), &nSize); if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) break; vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially } RegCloseKey(HKEY_PERFORMANCE_DATA); if (ret == ERROR_SUCCESS) { - RAND_add(begin_ptr(vData), nSize, nSize / 100.0); - memory_cleanse(begin_ptr(vData), nSize); + RAND_add(vData.data(), nSize, nSize / 100.0); + memory_cleanse(vData.data(), nSize); LogPrint("rand", "%s: %lu bytes\n", __func__, nSize); } else { static bool warned = false; // Warn only once diff --git a/src/rest.cpp b/src/rest.cpp index 7d7a7d3225..6b6b7401f6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -228,7 +228,7 @@ static bool rest_block(HTTPRequest* req, return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; switch (rf) { @@ -363,12 +363,12 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) if (!ParseHashStr(hashStr, hash)) return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); - CTransaction tx; + CTransactionRef tx; uint256 hashBlock = uint256(); if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; switch (rf) { @@ -388,7 +388,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) case RF_JSON: { UniValue objTx(UniValue::VOBJ); - TxToJSON(tx, hashBlock, objTx); + TxToJSON(*tx, hashBlock, objTx); string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e6d80f06a3..1139c2aa5c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -751,7 +751,7 @@ UniValue getblock(const JSONRPCRequest& request) if (!fVerbose) { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); return strHex; diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index ce16870d3c..1229a00c5f 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -218,19 +218,19 @@ UniValue getrawtransaction(const JSONRPCRequest& request) } } - CTransaction tx; + CTransactionRef tx; uint256 hashBlock; if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); - string strHex = EncodeHexTx(tx); + string strHex = EncodeHexTx(*tx, RPCSerializationFlags()); if (!fVerbose) return strHex; UniValue result(UniValue::VOBJ); result.push_back(Pair("hex", strHex)); - TxToJSON(tx, hashBlock, result); + TxToJSON(*tx, hashBlock, result); return result; } @@ -289,7 +289,7 @@ UniValue gettxoutproof(const JSONRPCRequest& request) if (pblockindex == NULL) { - CTransaction tx; + CTransactionRef tx; if (!GetTransaction(oneTxid, tx, Params().GetConsensus(), hashBlock, false) || hashBlock.IsNull()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); if (!mapBlockIndex.count(hashBlock)) @@ -520,13 +520,13 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) LOCK(cs_main); RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VSTR)); - CTransaction tx; + CMutableTransaction mtx; - if (!DecodeHexTx(tx, request.params[0].get_str(), true)) + if (!DecodeHexTx(mtx, request.params[0].get_str(), true)) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); UniValue result(UniValue::VOBJ); - TxToJSON(tx, uint256(), result); + TxToJSON(CTransaction(std::move(mtx)), uint256(), result); return result; } @@ -883,9 +883,10 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VSTR)(UniValue::VBOOL)); // parse hex string from parameter - CTransaction tx; - if (!DecodeHexTx(tx, request.params[0].get_str())) + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, request.params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); + CTransaction tx(std::move(mtx)); uint256 hashTx = tx.GetHash(); bool fLimitFree = false; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 164e0f00e2..2122819398 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -497,4 +497,12 @@ void RPCRunLater(const std::string& name, boost::function<void(void)> func, int6 deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); } +int RPCSerializationFlags() +{ + int flag = 0; + if (GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) + flag |= SERIALIZE_TRANSACTION_NO_WITNESS; + return flag; +} + CRPCTable tableRPC; diff --git a/src/rpc/server.h b/src/rpc/server.h index c59886222c..4fac68a51f 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -19,6 +19,8 @@ #include <univalue.h> +static const unsigned int DEFAULT_RPC_SERIALIZE_VERSION = 1; + class CRPCCommand; namespace RPCServer @@ -198,4 +200,7 @@ void StopRPC(); std::string JSONRPCExecBatch(const UniValue& vReq); void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); +// Retrieves any serialization flags requested in command line argument +int RPCSerializationFlags(); + #endif // BITCOIN_RPCSERVER_H diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 069ac55bfb..036d6ca7bf 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -85,8 +85,7 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP } try { TxInputStream stream(SER_NETWORK, PROTOCOL_VERSION, txTo, txToLen); - CTransaction tx; - stream >> tx; + CTransaction tx(deserialize, stream); if (nIn >= tx.vin.size()) return set_error(err, bitcoinconsensus_ERR_TX_INDEX); if (GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION) != txToLen) diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index a6403f9363..1410d0b73f 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -856,15 +856,15 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, un valtype& vch = stacktop(-1); valtype vchHash((opcode == OP_RIPEMD160 || opcode == OP_SHA1 || opcode == OP_HASH160) ? 20 : 32); if (opcode == OP_RIPEMD160) - CRIPEMD160().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CRIPEMD160().Write(vch.data(), vch.size()).Finalize(vchHash.data()); else if (opcode == OP_SHA1) - CSHA1().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CSHA1().Write(vch.data(), vch.size()).Finalize(vchHash.data()); else if (opcode == OP_SHA256) - CSHA256().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CSHA256().Write(vch.data(), vch.size()).Finalize(vchHash.data()); else if (opcode == OP_HASH160) - CHash160().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CHash160().Write(vch.data(), vch.size()).Finalize(vchHash.data()); else if (opcode == OP_HASH256) - CHash256().Write(begin_ptr(vch), vch.size()).Finalize(begin_ptr(vchHash)); + CHash256().Write(vch.data(), vch.size()).Finalize(vchHash.data()); popstack(stack); stack.push_back(vchHash); } diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index efb277d347..87fea161ba 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -6,6 +6,7 @@ bench_schnorr_verify bench_recover bench_internal tests +exhaustive_tests gen_context *.exe *.so diff --git a/src/secp256k1/.travis.yml b/src/secp256k1/.travis.yml index 2c5c63adad..2439529242 100644 --- a/src/secp256k1/.travis.yml +++ b/src/secp256k1/.travis.yml @@ -11,7 +11,7 @@ cache: - src/java/guava/ env: global: - - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no STATICPRECOMPUTATION=yes ASM=no BUILD=check EXTRAFLAGS= HOST= ECDH=no schnorr=no RECOVERY=no EXPERIMENTAL=no + - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no STATICPRECOMPUTATION=yes ASM=no BUILD=check EXTRAFLAGS= HOST= ECDH=no RECOVERY=no EXPERIMENTAL=no - GUAVA_URL=https://search.maven.org/remotecontent?filepath=com/google/guava/guava/18.0/guava-18.0.jar GUAVA_JAR=src/java/guava/guava-18.0.jar matrix: - SCALAR=32bit RECOVERY=yes @@ -22,15 +22,14 @@ env: - FIELD=64bit ENDOMORPHISM=yes ECDH=yes EXPERIMENTAL=yes - FIELD=64bit ASM=x86_64 - FIELD=64bit ENDOMORPHISM=yes ASM=x86_64 - - FIELD=32bit SCHNORR=yes EXPERIMENTAL=yes - FIELD=32bit ENDOMORPHISM=yes - BIGNUM=no - - BIGNUM=no ENDOMORPHISM=yes SCHNORR=yes RECOVERY=yes EXPERIMENTAL=yes + - BIGNUM=no ENDOMORPHISM=yes RECOVERY=yes EXPERIMENTAL=yes - BIGNUM=no STATICPRECOMPUTATION=no - BUILD=distcheck - EXTRAFLAGS=CPPFLAGS=-DDETERMINISTIC - EXTRAFLAGS=CFLAGS=-O0 - - BUILD=check-java ECDH=yes SCHNORR=yes EXPERIMENTAL=yes + - BUILD=check-java ECDH=yes EXPERIMENTAL=yes matrix: fast_finish: true include: @@ -66,5 +65,5 @@ before_script: ./autogen.sh script: - if [ -n "$HOST" ]; then export USE_HOST="--host=$HOST"; fi - if [ "x$HOST" = "xi686-linux-gnu" ]; then export CC="$CC -m32"; fi - - ./configure --enable-experimental=$EXPERIMENTAL --enable-endomorphism=$ENDOMORPHISM --with-field=$FIELD --with-bignum=$BIGNUM --with-scalar=$SCALAR --enable-ecmult-static-precomputation=$STATICPRECOMPUTATION --enable-module-ecdh=$ECDH --enable-module-schnorr=$SCHNORR --enable-module-recovery=$RECOVERY $EXTRAFLAGS $USE_HOST && make -j2 $BUILD + - ./configure --enable-experimental=$EXPERIMENTAL --enable-endomorphism=$ENDOMORPHISM --with-field=$FIELD --with-bignum=$BIGNUM --with-scalar=$SCALAR --enable-ecmult-static-precomputation=$STATICPRECOMPUTATION --enable-module-ecdh=$ECDH --enable-module-recovery=$RECOVERY $EXTRAFLAGS $USE_HOST && make -j2 $BUILD os: linux diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 3d130bdcbd..e5657f7f31 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -12,9 +12,11 @@ noinst_HEADERS = noinst_HEADERS += src/scalar.h noinst_HEADERS += src/scalar_4x64.h noinst_HEADERS += src/scalar_8x32.h +noinst_HEADERS += src/scalar_low.h noinst_HEADERS += src/scalar_impl.h noinst_HEADERS += src/scalar_4x64_impl.h noinst_HEADERS += src/scalar_8x32_impl.h +noinst_HEADERS += src/scalar_low_impl.h noinst_HEADERS += src/group.h noinst_HEADERS += src/group_impl.h noinst_HEADERS += src/num_gmp.h @@ -87,13 +89,23 @@ bench_internal_LDADD = $(SECP_LIBS) $(COMMON_LIB) bench_internal_CPPFLAGS = -DSECP256K1_BUILD $(SECP_INCLUDES) endif +TESTS = if USE_TESTS noinst_PROGRAMS += tests tests_SOURCES = src/tests.c tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) tests_LDADD = $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) tests_LDFLAGS = -static -TESTS = tests +TESTS += tests +endif + +if USE_EXHAUSTIVE_TESTS +noinst_PROGRAMS += exhaustive_tests +exhaustive_tests_SOURCES = src/tests_exhaustive.c +exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src $(SECP_INCLUDES) +exhaustive_tests_LDADD = $(SECP_LIBS) +exhaustive_tests_LDFLAGS = -static +TESTS += exhaustive_tests endif JAVAROOT=src/java @@ -154,10 +166,6 @@ if ENABLE_MODULE_ECDH include src/modules/ecdh/Makefile.am.include endif -if ENABLE_MODULE_SCHNORR -include src/modules/schnorr/Makefile.am.include -endif - if ENABLE_MODULE_RECOVERY include src/modules/recovery/Makefile.am.include endif diff --git a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 index b25d8adb92..b74acb8c13 100644 --- a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 +++ b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 @@ -46,6 +46,10 @@ if test x"$has_libcrypto" = x"yes" && test x"$has_openssl_ec" = x; then ECDSA_sign(0, NULL, 0, NULL, NULL, eckey); ECDSA_verify(0, NULL, 0, NULL, 0, eckey); EC_KEY_free(eckey); + ECDSA_SIG *sig_openssl; + sig_openssl = ECDSA_SIG_new(); + (void)sig_openssl->r; + ECDSA_SIG_free(sig_openssl); ]])],[has_openssl_ec=yes],[has_openssl_ec=no]) AC_MSG_RESULT([$has_openssl_ec]) fi diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 0743c36690..ec50ffe3a2 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -104,6 +104,11 @@ AC_ARG_ENABLE(experimental, [use_experimental=$enableval], [use_experimental=no]) +AC_ARG_ENABLE(exhaustive_tests, + AS_HELP_STRING([--enable-exhaustive-tests],[compile exhaustive tests (default is yes)]), + [use_exhaustive_tests=$enableval], + [use_exhaustive_tests=yes]) + AC_ARG_ENABLE(endomorphism, AS_HELP_STRING([--enable-endomorphism],[enable endomorphism (default is no)]), [use_endomorphism=$enableval], @@ -119,11 +124,6 @@ AC_ARG_ENABLE(module_ecdh, [enable_module_ecdh=$enableval], [enable_module_ecdh=no]) -AC_ARG_ENABLE(module_schnorr, - AS_HELP_STRING([--enable-module-schnorr],[enable Schnorr signature module (experimental)]), - [enable_module_schnorr=$enableval], - [enable_module_schnorr=no]) - AC_ARG_ENABLE(module_recovery, AS_HELP_STRING([--enable-module-recovery],[enable ECDSA pubkey recovery module (default is no)]), [enable_module_recovery=$enableval], @@ -381,9 +381,6 @@ fi if test x"$use_jni" != x"no"; then AX_JNI_INCLUDE_DIR have_jni_dependencies=yes - if test x"$enable_module_schnorr" = x"no"; then - have_jni_dependencies=no - fi if test x"$enable_module_ecdh" = x"no"; then have_jni_dependencies=no fi @@ -392,7 +389,7 @@ if test x"$use_jni" != x"no"; then fi if test "x$have_jni_dependencies" = "xno"; then if test x"$use_jni" = x"yes"; then - AC_MSG_ERROR([jni support explicitly requested but headers/dependencies were not found. Enable ECDH and Schnorr and try again.]) + AC_MSG_ERROR([jni support explicitly requested but headers/dependencies were not found. Enable ECDH and try again.]) fi AC_MSG_WARN([jni headers/dependencies not found. jni support disabled]) use_jni=no @@ -413,7 +410,7 @@ if test x"$use_endomorphism" = x"yes"; then AC_DEFINE(USE_ENDOMORPHISM, 1, [Define this symbol to use endomorphism optimization]) fi -if test x"$use_ecmult_static_precomputation" = x"yes"; then +if test x"$set_precomp" = x"yes"; then AC_DEFINE(USE_ECMULT_STATIC_PRECOMPUTATION, 1, [Define this symbol to use a statically generated ecmult table]) fi @@ -421,10 +418,6 @@ if test x"$enable_module_ecdh" = x"yes"; then AC_DEFINE(ENABLE_MODULE_ECDH, 1, [Define this symbol to enable the ECDH module]) fi -if test x"$enable_module_schnorr" = x"yes"; then - AC_DEFINE(ENABLE_MODULE_SCHNORR, 1, [Define this symbol to enable the Schnorr signature module]) -fi - if test x"$enable_module_recovery" = x"yes"; then AC_DEFINE(ENABLE_MODULE_RECOVERY, 1, [Define this symbol to enable the ECDSA pubkey recovery module]) fi @@ -442,7 +435,6 @@ AC_MSG_NOTICE([Using bignum implementation: $set_bignum]) AC_MSG_NOTICE([Using scalar implementation: $set_scalar]) AC_MSG_NOTICE([Using endomorphism optimizations: $use_endomorphism]) AC_MSG_NOTICE([Building ECDH module: $enable_module_ecdh]) -AC_MSG_NOTICE([Building Schnorr signatures module: $enable_module_schnorr]) AC_MSG_NOTICE([Building ECDSA pubkey recovery module: $enable_module_recovery]) AC_MSG_NOTICE([Using jni: $use_jni]) @@ -451,12 +443,8 @@ if test x"$enable_experimental" = x"yes"; then AC_MSG_NOTICE([WARNING: experimental build]) AC_MSG_NOTICE([Experimental features do not have stable APIs or properties, and may not be safe for production use.]) AC_MSG_NOTICE([Building ECDH module: $enable_module_ecdh]) - AC_MSG_NOTICE([Building Schnorr signatures module: $enable_module_schnorr]) AC_MSG_NOTICE([******]) else - if test x"$enable_module_schnorr" = x"yes"; then - AC_MSG_ERROR([Schnorr signature module is experimental. Use --enable-experimental to allow.]) - fi if test x"$enable_module_ecdh" = x"yes"; then AC_MSG_ERROR([ECDH module is experimental. Use --enable-experimental to allow.]) fi @@ -473,10 +461,10 @@ AC_SUBST(SECP_LIBS) AC_SUBST(SECP_TEST_LIBS) AC_SUBST(SECP_TEST_INCLUDES) AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) +AM_CONDITIONAL([USE_EXHAUSTIVE_TESTS], [test x"$use_exhaustive_tests" != x"no"]) AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) -AM_CONDITIONAL([USE_ECMULT_STATIC_PRECOMPUTATION], [test x"$use_ecmult_static_precomputation" = x"yes"]) +AM_CONDITIONAL([USE_ECMULT_STATIC_PRECOMPUTATION], [test x"$set_precomp" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_ECDH], [test x"$enable_module_ecdh" = x"yes"]) -AM_CONDITIONAL([ENABLE_MODULE_SCHNORR], [test x"$enable_module_schnorr" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_RECOVERY], [test x"$enable_module_recovery" = x"yes"]) AM_CONDITIONAL([USE_JNI], [test x"$use_jni" == x"yes"]) AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$use_external_asm" = x"yes"]) diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index 7145dbcc54..f268e309d0 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -47,11 +47,8 @@ typedef struct secp256k1_context_struct secp256k1_context; * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is * however guaranteed to be 64 bytes in size, and can be safely copied/moved. - * If you need to convert to a format suitable for storage or transmission, use - * secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. - * - * Furthermore, it is guaranteed that identical public keys (ignoring - * compression) will have identical representation, so they can be memcmp'ed. + * If you need to convert to a format suitable for storage, transmission, or + * comparison, use secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. */ typedef struct { unsigned char data[64]; @@ -62,12 +59,9 @@ typedef struct { * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is * however guaranteed to be 64 bytes in size, and can be safely copied/moved. - * If you need to convert to a format suitable for storage or transmission, use - * the secp256k1_ecdsa_signature_serialize_* and + * If you need to convert to a format suitable for storage, transmission, or + * comparison, use the secp256k1_ecdsa_signature_serialize_* and * secp256k1_ecdsa_signature_serialize_* functions. - * - * Furthermore, it is guaranteed to identical signatures will have identical - * representation, so they can be memcmp'ed. */ typedef struct { unsigned char data[64]; diff --git a/src/secp256k1/include/secp256k1_schnorr.h b/src/secp256k1/include/secp256k1_schnorr.h deleted file mode 100644 index dc32fec1ea..0000000000 --- a/src/secp256k1/include/secp256k1_schnorr.h +++ /dev/null @@ -1,173 +0,0 @@ -#ifndef _SECP256K1_SCHNORR_ -# define _SECP256K1_SCHNORR_ - -# include "secp256k1.h" - -# ifdef __cplusplus -extern "C" { -# endif - -/** Create a signature using a custom EC-Schnorr-SHA256 construction. It - * produces non-malleable 64-byte signatures which support public key recovery - * batch validation, and multiparty signing. - * Returns: 1: signature created - * 0: the nonce generation function failed, or the private key was - * invalid. - * Args: ctx: pointer to a context object, initialized for signing - * (cannot be NULL) - * Out: sig64: pointer to a 64-byte array where the signature will be - * placed (cannot be NULL) - * In: msg32: the 32-byte message hash being signed (cannot be NULL) - * seckey: pointer to a 32-byte secret key (cannot be NULL) - * noncefp:pointer to a nonce generation function. If NULL, - * secp256k1_nonce_function_default is used - * ndata: pointer to arbitrary data used by the nonce generation - * function (can be NULL) - */ -SECP256K1_API int secp256k1_schnorr_sign( - const secp256k1_context* ctx, - unsigned char *sig64, - const unsigned char *msg32, - const unsigned char *seckey, - secp256k1_nonce_function noncefp, - const void *ndata -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); - -/** Verify a signature created by secp256k1_schnorr_sign. - * Returns: 1: correct signature - * 0: incorrect signature - * Args: ctx: a secp256k1 context object, initialized for verification. - * In: sig64: the 64-byte signature being verified (cannot be NULL) - * msg32: the 32-byte message hash being verified (cannot be NULL) - * pubkey: the public key to verify with (cannot be NULL) - */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorr_verify( - const secp256k1_context* ctx, - const unsigned char *sig64, - const unsigned char *msg32, - const secp256k1_pubkey *pubkey -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); - -/** Recover an EC public key from a Schnorr signature created using - * secp256k1_schnorr_sign. - * Returns: 1: public key successfully recovered (which guarantees a correct - * signature). - * 0: otherwise. - * Args: ctx: pointer to a context object, initialized for - * verification (cannot be NULL) - * Out: pubkey: pointer to a pubkey to set to the recovered public key - * (cannot be NULL). - * In: sig64: signature as 64 byte array (cannot be NULL) - * msg32: the 32-byte message hash assumed to be signed (cannot - * be NULL) - */ -SECP256K1_API int secp256k1_schnorr_recover( - const secp256k1_context* ctx, - secp256k1_pubkey *pubkey, - const unsigned char *sig64, - const unsigned char *msg32 -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); - -/** Generate a nonce pair deterministically for use with - * secp256k1_schnorr_partial_sign. - * Returns: 1: valid nonce pair was generated. - * 0: otherwise (nonce generation function failed) - * Args: ctx: pointer to a context object, initialized for signing - * (cannot be NULL) - * Out: pubnonce: public side of the nonce (cannot be NULL) - * privnonce32: private side of the nonce (32 byte) (cannot be NULL) - * In: msg32: the 32-byte message hash assumed to be signed (cannot - * be NULL) - * sec32: the 32-byte private key (cannot be NULL) - * noncefp: pointer to a nonce generation function. If NULL, - * secp256k1_nonce_function_default is used - * noncedata: pointer to arbitrary data used by the nonce generation - * function (can be NULL) - * - * Do not use the output as a private/public key pair for signing/validation. - */ -SECP256K1_API int secp256k1_schnorr_generate_nonce_pair( - const secp256k1_context* ctx, - secp256k1_pubkey *pubnonce, - unsigned char *privnonce32, - const unsigned char *msg32, - const unsigned char *sec32, - secp256k1_nonce_function noncefp, - const void* noncedata -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); - -/** Produce a partial Schnorr signature, which can be combined using - * secp256k1_schnorr_partial_combine, to end up with a full signature that is - * verifiable using secp256k1_schnorr_verify. - * Returns: 1: signature created successfully. - * 0: no valid signature exists with this combination of keys, nonces - * and message (chance around 1 in 2^128) - * -1: invalid private key, nonce, or public nonces. - * Args: ctx: pointer to context object, initialized for signing (cannot - * be NULL) - * Out: sig64: pointer to 64-byte array to put partial signature in - * In: msg32: pointer to 32-byte message to sign - * sec32: pointer to 32-byte private key - * pubnonce_others: pointer to pubkey containing the sum of the other's - * nonces (see secp256k1_ec_pubkey_combine) - * secnonce32: pointer to 32-byte array containing our nonce - * - * The intended procedure for creating a multiparty signature is: - * - Each signer S[i] with private key x[i] and public key Q[i] runs - * secp256k1_schnorr_generate_nonce_pair to produce a pair (k[i],R[i]) of - * private/public nonces. - * - All signers communicate their public nonces to each other (revealing your - * private nonce can lead to discovery of your private key, so it should be - * considered secret). - * - All signers combine all the public nonces they received (excluding their - * own) using secp256k1_ec_pubkey_combine to obtain an - * Rall[i] = sum(R[0..i-1,i+1..n]). - * - All signers produce a partial signature using - * secp256k1_schnorr_partial_sign, passing in their own private key x[i], - * their own private nonce k[i], and the sum of the others' public nonces - * Rall[i]. - * - All signers communicate their partial signatures to each other. - * - Someone combines all partial signatures using - * secp256k1_schnorr_partial_combine, to obtain a full signature. - * - The resulting signature is validatable using secp256k1_schnorr_verify, with - * public key equal to the result of secp256k1_ec_pubkey_combine of the - * signers' public keys (sum(Q[0..n])). - * - * Note that secp256k1_schnorr_partial_combine and secp256k1_ec_pubkey_combine - * function take their arguments in any order, and it is possible to - * pre-combine several inputs already with one call, and add more inputs later - * by calling the function again (they are commutative and associative). - */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorr_partial_sign( - const secp256k1_context* ctx, - unsigned char *sig64, - const unsigned char *msg32, - const unsigned char *sec32, - const secp256k1_pubkey *pubnonce_others, - const unsigned char *secnonce32 -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(6); - -/** Combine multiple Schnorr partial signatures. - * Returns: 1: the passed signatures were successfully combined. - * 0: the resulting signature is not valid (chance of 1 in 2^256) - * -1: some inputs were invalid, or the signatures were not created - * using the same set of nonces - * Args: ctx: pointer to a context object - * Out: sig64: pointer to a 64-byte array to place the combined signature - * (cannot be NULL) - * In: sig64sin: pointer to an array of n pointers to 64-byte input - * signatures - * n: the number of signatures to combine (at least 1) - */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorr_partial_combine( - const secp256k1_context* ctx, - unsigned char *sig64, - const unsigned char * const * sig64sin, - size_t n -) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); - -# ifdef __cplusplus -} -# endif - -#endif diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index d110b4bb1d..9a42e519bd 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -203,7 +203,9 @@ static int secp256k1_ecdsa_sig_serialize(unsigned char *sig, size_t *size, const static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const secp256k1_scalar *sigr, const secp256k1_scalar *sigs, const secp256k1_ge *pubkey, const secp256k1_scalar *message) { unsigned char c[32]; secp256k1_scalar sn, u1, u2; +#if !defined(EXHAUSTIVE_TEST_ORDER) secp256k1_fe xr; +#endif secp256k1_gej pubkeyj; secp256k1_gej pr; @@ -219,6 +221,21 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const if (secp256k1_gej_is_infinity(&pr)) { return 0; } + +#if defined(EXHAUSTIVE_TEST_ORDER) +{ + secp256k1_scalar computed_r; + int overflow = 0; + secp256k1_ge pr_ge; + secp256k1_ge_set_gej(&pr_ge, &pr); + secp256k1_fe_normalize(&pr_ge.x); + + secp256k1_fe_get_b32(c, &pr_ge.x); + secp256k1_scalar_set_b32(&computed_r, c, &overflow); + /* we fully expect overflow */ + return secp256k1_scalar_eq(sigr, &computed_r); +} +#else secp256k1_scalar_get_b32(c, sigr); secp256k1_fe_set_b32(&xr, c); @@ -252,6 +269,7 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const return 1; } return 0; +#endif } static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context *ctx, secp256k1_scalar *sigr, secp256k1_scalar *sigs, const secp256k1_scalar *seckey, const secp256k1_scalar *message, const secp256k1_scalar *nonce, int *recid) { diff --git a/src/secp256k1/src/ecmult_const_impl.h b/src/secp256k1/src/ecmult_const_impl.h index 7a6a25318c..0db314c48e 100644 --- a/src/secp256k1/src/ecmult_const_impl.h +++ b/src/secp256k1/src/ecmult_const_impl.h @@ -78,7 +78,7 @@ static int secp256k1_wnaf_const(int *wnaf, secp256k1_scalar s, int w) { /* Negative numbers will be negated to keep their bit representation below the maximum width */ flip = secp256k1_scalar_is_high(&s); /* We add 1 to even numbers, 2 to odd ones, noting that negation flips parity */ - bit = flip ^ (s.d[0] & 1); + bit = flip ^ !secp256k1_scalar_is_even(&s); /* We check for negative one, since adding 2 to it will cause an overflow */ secp256k1_scalar_negate(&neg_s, &s); not_neg_one = !secp256k1_scalar_is_one(&neg_s); diff --git a/src/secp256k1/src/ecmult_gen_impl.h b/src/secp256k1/src/ecmult_gen_impl.h index b63c4d8662..35f2546077 100644 --- a/src/secp256k1/src/ecmult_gen_impl.h +++ b/src/secp256k1/src/ecmult_gen_impl.h @@ -77,7 +77,7 @@ static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context *ctx secp256k1_gej_add_var(&numsbase, &numsbase, &nums_gej, NULL); } } - secp256k1_ge_set_all_gej_var(1024, prec, precj, cb); + secp256k1_ge_set_all_gej_var(prec, precj, 1024, cb); } for (j = 0; j < 64; j++) { for (i = 0; i < 16; i++) { diff --git a/src/secp256k1/src/ecmult_impl.h b/src/secp256k1/src/ecmult_impl.h index 81ae08e100..4e40104ad4 100644 --- a/src/secp256k1/src/ecmult_impl.h +++ b/src/secp256k1/src/ecmult_impl.h @@ -7,15 +7,29 @@ #ifndef _SECP256K1_ECMULT_IMPL_H_ #define _SECP256K1_ECMULT_IMPL_H_ +#include <string.h> + #include "group.h" #include "scalar.h" #include "ecmult.h" -#include <string.h> - +#if defined(EXHAUSTIVE_TEST_ORDER) +/* We need to lower these values for exhaustive tests because + * the tables cannot have infinities in them (this breaks the + * affine-isomorphism stuff which tracks z-ratios) */ +# if EXHAUSTIVE_TEST_ORDER > 128 +# define WINDOW_A 5 +# define WINDOW_G 8 +# elif EXHAUSTIVE_TEST_ORDER > 8 +# define WINDOW_A 4 +# define WINDOW_G 4 +# else +# define WINDOW_A 2 +# define WINDOW_G 2 +# endif +#else /* optimal for 128-bit and 256-bit exponents. */ #define WINDOW_A 5 - /** larger numbers may result in slightly better performance, at the cost of exponentially larger precomputed tables. */ #ifdef USE_ENDOMORPHISM @@ -25,6 +39,7 @@ /** One table for window size 16: 1.375 MiB. */ #define WINDOW_G 16 #endif +#endif /** The number of entries a table with precomputed multiples needs to have. */ #define ECMULT_TABLE_SIZE(w) (1 << ((w)-2)) @@ -103,7 +118,7 @@ static void secp256k1_ecmult_odd_multiples_table_storage_var(int n, secp256k1_ge /* Compute the odd multiples in Jacobian form. */ secp256k1_ecmult_odd_multiples_table(n, prej, zr, a); /* Convert them in batch to affine coordinates. */ - secp256k1_ge_set_table_gej_var(n, prea, prej, zr); + secp256k1_ge_set_table_gej_var(prea, prej, zr, n); /* Convert them to compact storage form. */ for (i = 0; i < n; i++) { secp256k1_ge_to_storage(&pre[i], &prea[i]); diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index c5ba074244..bbb1ee866c 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -30,6 +30,8 @@ #error "Please select field implementation" #endif +#include "util.h" + /** Normalize a field element. */ static void secp256k1_fe_normalize(secp256k1_fe *r); @@ -50,6 +52,9 @@ static int secp256k1_fe_normalizes_to_zero_var(secp256k1_fe *r); /** Set a field element equal to a small integer. Resulting field element is normalized. */ static void secp256k1_fe_set_int(secp256k1_fe *r, int a); +/** Sets a field element equal to zero, initializing all fields. */ +static void secp256k1_fe_clear(secp256k1_fe *a); + /** Verify whether a field element is zero. Requires the input to be normalized. */ static int secp256k1_fe_is_zero(const secp256k1_fe *a); @@ -110,7 +115,7 @@ static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *a); /** Calculate the (modular) inverses of a batch of field elements. Requires the inputs' magnitudes to be * at most 8. The output magnitudes are 1 (but not guaranteed to be normalized). The inputs and * outputs must not overlap in memory. */ -static void secp256k1_fe_inv_all_var(size_t len, secp256k1_fe *r, const secp256k1_fe *a); +static void secp256k1_fe_inv_all_var(secp256k1_fe *r, const secp256k1_fe *a, size_t len); /** Convert a field element to the storage type. */ static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a); diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 52cd902eb3..5127b279bc 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -260,7 +260,7 @@ static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *a) { #endif } -static void secp256k1_fe_inv_all_var(size_t len, secp256k1_fe *r, const secp256k1_fe *a) { +static void secp256k1_fe_inv_all_var(secp256k1_fe *r, const secp256k1_fe *a, size_t len) { secp256k1_fe u; size_t i; if (len < 1) { diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index d515716744..4957b248fe 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -65,12 +65,12 @@ static void secp256k1_ge_neg(secp256k1_ge *r, const secp256k1_ge *a); static void secp256k1_ge_set_gej(secp256k1_ge *r, secp256k1_gej *a); /** Set a batch of group elements equal to the inputs given in jacobian coordinates */ -static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_callback *cb); +static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a, size_t len, const secp256k1_callback *cb); /** Set a batch of group elements equal to the inputs given in jacobian * coordinates (with known z-ratios). zr must contain the known z-ratios such * that mul(a[i].z, zr[i+1]) == a[i+1].z. zr[0] is ignored. */ -static void secp256k1_ge_set_table_gej_var(size_t len, secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zr); +static void secp256k1_ge_set_table_gej_var(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zr, size_t len); /** Bring a batch inputs given in jacobian coordinates (with known z-ratios) to * the same global z "denominator". zr must contain the known z-ratios such diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 3e9c4c410d..2e192b62fd 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -11,6 +11,53 @@ #include "field.h" #include "group.h" +/* These points can be generated in sage as follows: + * + * 0. Setup a worksheet with the following parameters. + * b = 4 # whatever CURVE_B will be set to + * F = FiniteField (0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F) + * C = EllipticCurve ([F (0), F (b)]) + * + * 1. Determine all the small orders available to you. (If there are + * no satisfactory ones, go back and change b.) + * print C.order().factor(limit=1000) + * + * 2. Choose an order as one of the prime factors listed in the above step. + * (You can also multiply some to get a composite order, though the + * tests will crash trying to invert scalars during signing.) We take a + * random point and scale it to drop its order to the desired value. + * There is some probability this won't work; just try again. + * order = 199 + * P = C.random_point() + * P = (int(P.order()) / int(order)) * P + * assert(P.order() == order) + * + * 3. Print the values. You'll need to use a vim macro or something to + * split the hex output into 4-byte chunks. + * print "%x %x" % P.xy() + */ +#if defined(EXHAUSTIVE_TEST_ORDER) +# if EXHAUSTIVE_TEST_ORDER == 199 +const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST( + 0xFA7CC9A7, 0x0737F2DB, 0xA749DD39, 0x2B4FB069, + 0x3B017A7D, 0xA808C2F1, 0xFB12940C, 0x9EA66C18, + 0x78AC123A, 0x5ED8AEF3, 0x8732BC91, 0x1F3A2868, + 0x48DF246C, 0x808DAE72, 0xCFE52572, 0x7F0501ED +); + +const int CURVE_B = 4; +# elif EXHAUSTIVE_TEST_ORDER == 13 +const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST( + 0xedc60018, 0xa51a786b, 0x2ea91f4d, 0x4c9416c0, + 0x9de54c3b, 0xa1316554, 0x6cf4345c, 0x7277ef15, + 0x54cb1b6b, 0xdc8c1273, 0x087844ea, 0x43f4603e, + 0x0eaf9a43, 0xf6effe55, 0x939f806d, 0x37adf8ac +); +const int CURVE_B = 2; +# else +# error No known generator for the specified exhaustive test group order. +# endif +#else /** Generator for secp256k1, value 'g' defined in * "Standards for Efficient Cryptography" (SEC2) 2.7.1. */ @@ -21,8 +68,11 @@ static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST( 0xFD17B448UL, 0xA6855419UL, 0x9C47D08FUL, 0xFB10D4B8UL ); +const int CURVE_B = 7; +#endif + static void secp256k1_ge_set_gej_zinv(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zi) { - secp256k1_fe zi2; + secp256k1_fe zi2; secp256k1_fe zi3; secp256k1_fe_sqr(&zi2, zi); secp256k1_fe_mul(&zi3, &zi2, zi); @@ -76,7 +126,7 @@ static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a) { r->y = a->y; } -static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_callback *cb) { +static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a, size_t len, const secp256k1_callback *cb) { secp256k1_fe *az; secp256k1_fe *azi; size_t i; @@ -89,7 +139,7 @@ static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge *r, const secp } azi = (secp256k1_fe *)checked_malloc(cb, sizeof(secp256k1_fe) * count); - secp256k1_fe_inv_all_var(count, azi, az); + secp256k1_fe_inv_all_var(azi, az, count); free(az); count = 0; @@ -102,7 +152,7 @@ static void secp256k1_ge_set_all_gej_var(size_t len, secp256k1_ge *r, const secp free(azi); } -static void secp256k1_ge_set_table_gej_var(size_t len, secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zr) { +static void secp256k1_ge_set_table_gej_var(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zr, size_t len) { size_t i = len - 1; secp256k1_fe zi; @@ -145,9 +195,15 @@ static void secp256k1_ge_globalz_set_table_gej(size_t len, secp256k1_ge *r, secp static void secp256k1_gej_set_infinity(secp256k1_gej *r) { r->infinity = 1; - secp256k1_fe_set_int(&r->x, 0); - secp256k1_fe_set_int(&r->y, 0); - secp256k1_fe_set_int(&r->z, 0); + secp256k1_fe_clear(&r->x); + secp256k1_fe_clear(&r->y); + secp256k1_fe_clear(&r->z); +} + +static void secp256k1_ge_set_infinity(secp256k1_ge *r) { + r->infinity = 1; + secp256k1_fe_clear(&r->x); + secp256k1_fe_clear(&r->y); } static void secp256k1_gej_clear(secp256k1_gej *r) { @@ -169,7 +225,7 @@ static int secp256k1_ge_set_xquad(secp256k1_ge *r, const secp256k1_fe *x) { secp256k1_fe_sqr(&x2, x); secp256k1_fe_mul(&x3, x, &x2); r->infinity = 0; - secp256k1_fe_set_int(&c, 7); + secp256k1_fe_set_int(&c, CURVE_B); secp256k1_fe_add(&c, &x3); return secp256k1_fe_sqrt(&r->y, &c); } @@ -228,7 +284,7 @@ static int secp256k1_gej_is_valid_var(const secp256k1_gej *a) { secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x); secp256k1_fe_sqr(&z2, &a->z); secp256k1_fe_sqr(&z6, &z2); secp256k1_fe_mul(&z6, &z6, &z2); - secp256k1_fe_mul_int(&z6, 7); + secp256k1_fe_mul_int(&z6, CURVE_B); secp256k1_fe_add(&x3, &z6); secp256k1_fe_normalize_weak(&x3); return secp256k1_fe_equal_var(&y2, &x3); @@ -242,7 +298,7 @@ static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) { /* y^2 = x^3 + 7 */ secp256k1_fe_sqr(&y2, &a->y); secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x); - secp256k1_fe_set_int(&c, 7); + secp256k1_fe_set_int(&c, CURVE_B); secp256k1_fe_add(&x3, &c); secp256k1_fe_normalize_weak(&x3); return secp256k1_fe_equal_var(&y2, &x3); @@ -260,7 +316,7 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s /** For secp256k1, 2Q is infinity if and only if Q is infinity. This is because if 2Q = infinity, * Q must equal -Q, or that Q.y == -(Q.y), or Q.y is 0. For a point on y^2 = x^3 + 7 to have * y=0, x^3 must be -7 mod p. However, -7 has no cube root mod p. - * + * * Having said this, if this function receives a point on a sextic twist, e.g. by * a fault attack, it is possible for y to be 0. This happens for y^2 = x^3 + 6, * since -6 does have a cube root mod p. For this point, this function will not set diff --git a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java index be67048fbe..1c67802fba 100644 --- a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java +++ b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1.java @@ -32,7 +32,7 @@ import static org.bitcoin.NativeSecp256k1Util.*; * <p>You can find an example library that can be used for this at https://github.com/bitcoin/secp256k1</p> * * <p>To build secp256k1 for use with bitcoinj, run - * `./configure --enable-jni --enable-experimental --enable-module-schnorr --enable-module-ecdh` + * `./configure --enable-jni --enable-experimental --enable-module-ecdh` * and `make` then copy `.libs/libsecp256k1.so` to your system library path * or point the JVM to the folder containing it with -Djava.library.path * </p> @@ -417,36 +417,6 @@ public class NativeSecp256k1 { } } - public static byte[] schnorrSign(byte[] data, byte[] sec) throws AssertFailException { - Preconditions.checkArgument(data.length == 32 && sec.length <= 32); - - ByteBuffer byteBuff = nativeECDSABuffer.get(); - if (byteBuff == null) { - byteBuff = ByteBuffer.allocateDirect(32 + 32); - byteBuff.order(ByteOrder.nativeOrder()); - nativeECDSABuffer.set(byteBuff); - } - byteBuff.rewind(); - byteBuff.put(data); - byteBuff.put(sec); - - byte[][] retByteArray; - - r.lock(); - try { - retByteArray = secp256k1_schnorr_sign(byteBuff, Secp256k1Context.getContext()); - } finally { - r.unlock(); - } - - byte[] sigArr = retByteArray[0]; - int retVal = new BigInteger(new byte[] { retByteArray[1][0] }).intValue(); - - assertEquals(sigArr.length, 64, "Got bad signature length."); - - return retVal == 0 ? new byte[0] : sigArr; - } - private static native long secp256k1_ctx_clone(long context); private static native int secp256k1_context_randomize(ByteBuffer byteBuff, long context); @@ -471,8 +441,6 @@ public class NativeSecp256k1 { private static native byte[][] secp256k1_ec_pubkey_parse(ByteBuffer byteBuff, long context, int inputLen); - private static native byte[][] secp256k1_schnorr_sign(ByteBuffer byteBuff, long context); - private static native byte[][] secp256k1_ecdh(ByteBuffer byteBuff, long context, int inputLen); } diff --git a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java index f18ce95810..c00d08899b 100644 --- a/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java +++ b/src/secp256k1/src/java/org/bitcoin/NativeSecp256k1Test.java @@ -167,22 +167,6 @@ public class NativeSecp256k1Test { assertEquals( result, true, "testRandomize"); } - /** - * This tests signSchnorr() for a valid secretkey - */ - public static void testSchnorrSign() throws AssertFailException{ - - byte[] data = BaseEncoding.base16().lowerCase().decode("CF80CD8AED482D5D1527D7DC72FCEFF84E6326592848447D2DC0B0E87DFC9A90".toLowerCase()); //sha256hash of "testing" - byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); - - byte[] resultArr = NativeSecp256k1.schnorrSign(data, sec); - String sigString = javax.xml.bind.DatatypeConverter.printHexBinary(resultArr); - assertEquals( sigString, "C5E929AA058B982048760422D3B563749B7D0E50C5EBD8CD2FFC23214BD6A2F1B072C13880997EBA847CF20F2F90FCE07C1CA33A890A4127095A351127F8D95F" , "testSchnorrSign"); - } - - /** - * This tests signSchnorr() for a valid secretkey - */ public static void testCreateECDHSecret() throws AssertFailException{ byte[] sec = BaseEncoding.base16().lowerCase().decode("67E56582298859DDAE725F972992A07C6C4FB9F62A8FFF58CE3CA926A1063530".toLowerCase()); @@ -216,11 +200,6 @@ public class NativeSecp256k1Test { testSignPos(); testSignNeg(); - //Test Schnorr (partial support) //TODO - testSchnorrSign(); - //testSchnorrVerify - //testSchnorrRecovery - //Test privKeyTweakAdd() 1 testPrivKeyTweakAdd_1(); diff --git a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c index dba9524dd4..bcef7b32ce 100644 --- a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c +++ b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c @@ -5,7 +5,6 @@ #include "include/secp256k1.h" #include "include/secp256k1_ecdh.h" #include "include/secp256k1_recovery.h" -#include "include/secp256k1_schnorr.h" SECP256K1_API jlong JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ctx_1clone @@ -333,39 +332,6 @@ SECP256K1_API jlong JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdsa_1p return 0; } -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1schnorr_1sign - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l) -{ - secp256k1_context *ctx = (secp256k1_context*)(uintptr_t)ctx_l; - unsigned char* data = (unsigned char*) (*env)->GetDirectBufferAddress(env, byteBufferObject); - unsigned char* secKey = (unsigned char*) (data + 32); - - jobjectArray retArray; - jbyteArray sigArray, intsByteArray; - unsigned char intsarray[1]; - unsigned char sig[64]; - - int ret = secp256k1_schnorr_sign(ctx, sig, data, secKey, NULL, NULL); - - intsarray[0] = ret; - - retArray = (*env)->NewObjectArray(env, 2, - (*env)->FindClass(env, "[B"), - (*env)->NewByteArray(env, 1)); - - sigArray = (*env)->NewByteArray(env, 64); - (*env)->SetByteArrayRegion(env, sigArray, 0, 64, (jbyte*)sig); - (*env)->SetObjectArrayElement(env, retArray, 0, sigArray); - - intsByteArray = (*env)->NewByteArray(env, 1); - (*env)->SetByteArrayRegion(env, intsByteArray, 0, 1, (jbyte*)intsarray); - (*env)->SetObjectArrayElement(env, retArray, 1, intsByteArray); - - (void)classObject; - - return retArray; -} - SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1ecdh (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l, jint publen) { diff --git a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h index 4125a1f523..fe613c9e9e 100644 --- a/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h +++ b/src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h @@ -106,14 +106,6 @@ SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1e /* * Class: org_bitcoin_NativeSecp256k1 - * Method: secp256k1_schnorr_sign - * Signature: (Ljava/nio/ByteBuffer;JI)[[B - */ -SECP256K1_API jobjectArray JNICALL Java_org_bitcoin_NativeSecp256k1_secp256k1_1schnorr_1sign - (JNIEnv* env, jclass classObject, jobject byteBufferObject, jlong ctx_l); - -/* - * Class: org_bitcoin_NativeSecp256k1 * Method: secp256k1_ecdh * Signature: (Ljava/nio/ByteBuffer;JI)[[B */ diff --git a/src/secp256k1/src/modules/recovery/main_impl.h b/src/secp256k1/src/modules/recovery/main_impl.h index ec42f4bb6c..86f2f0cb2b 100644..100755 --- a/src/secp256k1/src/modules/recovery/main_impl.h +++ b/src/secp256k1/src/modules/recovery/main_impl.h @@ -138,16 +138,15 @@ int secp256k1_ecdsa_sign_recoverable(const secp256k1_context* ctx, secp256k1_ecd secp256k1_scalar_set_b32(&sec, seckey, &overflow); /* Fail if the secret key is invalid. */ if (!overflow && !secp256k1_scalar_is_zero(&sec)) { + unsigned char nonce32[32]; unsigned int count = 0; secp256k1_scalar_set_b32(&msg, msg32, NULL); while (1) { - unsigned char nonce32[32]; ret = noncefp(nonce32, msg32, seckey, NULL, (void*)noncedata, count); if (!ret) { break; } secp256k1_scalar_set_b32(&non, nonce32, &overflow); - memset(nonce32, 0, 32); if (!secp256k1_scalar_is_zero(&non) && !overflow) { if (secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &r, &s, &sec, &msg, &non, &recid)) { break; @@ -155,6 +154,7 @@ int secp256k1_ecdsa_sign_recoverable(const secp256k1_context* ctx, secp256k1_ecd } count++; } + memset(nonce32, 0, 32); secp256k1_scalar_clear(&msg); secp256k1_scalar_clear(&non); secp256k1_scalar_clear(&sec); diff --git a/src/secp256k1/src/modules/schnorr/Makefile.am.include b/src/secp256k1/src/modules/schnorr/Makefile.am.include deleted file mode 100644 index f1af8e8325..0000000000 --- a/src/secp256k1/src/modules/schnorr/Makefile.am.include +++ /dev/null @@ -1,10 +0,0 @@ -include_HEADERS += include/secp256k1_schnorr.h -noinst_HEADERS += src/modules/schnorr/main_impl.h -noinst_HEADERS += src/modules/schnorr/schnorr.h -noinst_HEADERS += src/modules/schnorr/schnorr_impl.h -noinst_HEADERS += src/modules/schnorr/tests_impl.h -if USE_BENCHMARK -noinst_PROGRAMS += bench_schnorr_verify -bench_schnorr_verify_SOURCES = src/bench_schnorr_verify.c -bench_schnorr_verify_LDADD = libsecp256k1.la $(SECP_LIBS) $(COMMON_LIB) -endif diff --git a/src/secp256k1/src/modules/schnorr/main_impl.h b/src/secp256k1/src/modules/schnorr/main_impl.h deleted file mode 100644 index fa176a1767..0000000000 --- a/src/secp256k1/src/modules/schnorr/main_impl.h +++ /dev/null @@ -1,164 +0,0 @@ -/********************************************************************** - * Copyright (c) 2014-2015 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php.* - **********************************************************************/ - -#ifndef SECP256K1_MODULE_SCHNORR_MAIN -#define SECP256K1_MODULE_SCHNORR_MAIN - -#include "include/secp256k1_schnorr.h" -#include "modules/schnorr/schnorr_impl.h" - -static void secp256k1_schnorr_msghash_sha256(unsigned char *h32, const unsigned char *r32, const unsigned char *msg32) { - secp256k1_sha256_t sha; - secp256k1_sha256_initialize(&sha); - secp256k1_sha256_write(&sha, r32, 32); - secp256k1_sha256_write(&sha, msg32, 32); - secp256k1_sha256_finalize(&sha, h32); -} - -static const unsigned char secp256k1_schnorr_algo16[17] = "Schnorr+SHA256 "; - -int secp256k1_schnorr_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const unsigned char *seckey, secp256k1_nonce_function noncefp, const void* noncedata) { - secp256k1_scalar sec, non; - int ret = 0; - int overflow = 0; - unsigned int count = 0; - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sig64 != NULL); - ARG_CHECK(seckey != NULL); - if (noncefp == NULL) { - noncefp = secp256k1_nonce_function_default; - } - - secp256k1_scalar_set_b32(&sec, seckey, NULL); - while (1) { - unsigned char nonce32[32]; - ret = noncefp(nonce32, msg32, seckey, secp256k1_schnorr_algo16, (void*)noncedata, count); - if (!ret) { - break; - } - secp256k1_scalar_set_b32(&non, nonce32, &overflow); - memset(nonce32, 0, 32); - if (!secp256k1_scalar_is_zero(&non) && !overflow) { - if (secp256k1_schnorr_sig_sign(&ctx->ecmult_gen_ctx, sig64, &sec, &non, NULL, secp256k1_schnorr_msghash_sha256, msg32)) { - break; - } - } - count++; - } - if (!ret) { - memset(sig64, 0, 64); - } - secp256k1_scalar_clear(&non); - secp256k1_scalar_clear(&sec); - return ret; -} - -int secp256k1_schnorr_verify(const secp256k1_context* ctx, const unsigned char *sig64, const unsigned char *msg32, const secp256k1_pubkey *pubkey) { - secp256k1_ge q; - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sig64 != NULL); - ARG_CHECK(pubkey != NULL); - - secp256k1_pubkey_load(ctx, &q, pubkey); - return secp256k1_schnorr_sig_verify(&ctx->ecmult_ctx, sig64, &q, secp256k1_schnorr_msghash_sha256, msg32); -} - -int secp256k1_schnorr_recover(const secp256k1_context* ctx, secp256k1_pubkey *pubkey, const unsigned char *sig64, const unsigned char *msg32) { - secp256k1_ge q; - - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sig64 != NULL); - ARG_CHECK(pubkey != NULL); - - if (secp256k1_schnorr_sig_recover(&ctx->ecmult_ctx, sig64, &q, secp256k1_schnorr_msghash_sha256, msg32)) { - secp256k1_pubkey_save(pubkey, &q); - return 1; - } else { - memset(pubkey, 0, sizeof(*pubkey)); - return 0; - } -} - -int secp256k1_schnorr_generate_nonce_pair(const secp256k1_context* ctx, secp256k1_pubkey *pubnonce, unsigned char *privnonce32, const unsigned char *sec32, const unsigned char *msg32, secp256k1_nonce_function noncefp, const void* noncedata) { - int count = 0; - int ret = 1; - secp256k1_gej Qj; - secp256k1_ge Q; - secp256k1_scalar sec; - - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sec32 != NULL); - ARG_CHECK(pubnonce != NULL); - ARG_CHECK(privnonce32 != NULL); - - if (noncefp == NULL) { - noncefp = secp256k1_nonce_function_default; - } - - do { - int overflow; - ret = noncefp(privnonce32, sec32, msg32, secp256k1_schnorr_algo16, (void*)noncedata, count++); - if (!ret) { - break; - } - secp256k1_scalar_set_b32(&sec, privnonce32, &overflow); - if (overflow || secp256k1_scalar_is_zero(&sec)) { - continue; - } - secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &Qj, &sec); - secp256k1_ge_set_gej(&Q, &Qj); - - secp256k1_pubkey_save(pubnonce, &Q); - break; - } while(1); - - secp256k1_scalar_clear(&sec); - if (!ret) { - memset(pubnonce, 0, sizeof(*pubnonce)); - } - return ret; -} - -int secp256k1_schnorr_partial_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const unsigned char *sec32, const secp256k1_pubkey *pubnonce_others, const unsigned char *secnonce32) { - int overflow = 0; - secp256k1_scalar sec, non; - secp256k1_ge pubnon; - VERIFY_CHECK(ctx != NULL); - ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); - ARG_CHECK(msg32 != NULL); - ARG_CHECK(sig64 != NULL); - ARG_CHECK(sec32 != NULL); - ARG_CHECK(secnonce32 != NULL); - ARG_CHECK(pubnonce_others != NULL); - - secp256k1_scalar_set_b32(&sec, sec32, &overflow); - if (overflow || secp256k1_scalar_is_zero(&sec)) { - return -1; - } - secp256k1_scalar_set_b32(&non, secnonce32, &overflow); - if (overflow || secp256k1_scalar_is_zero(&non)) { - return -1; - } - secp256k1_pubkey_load(ctx, &pubnon, pubnonce_others); - return secp256k1_schnorr_sig_sign(&ctx->ecmult_gen_ctx, sig64, &sec, &non, &pubnon, secp256k1_schnorr_msghash_sha256, msg32); -} - -int secp256k1_schnorr_partial_combine(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char * const *sig64sin, size_t n) { - ARG_CHECK(sig64 != NULL); - ARG_CHECK(n >= 1); - ARG_CHECK(sig64sin != NULL); - return secp256k1_schnorr_sig_combine(sig64, n, sig64sin); -} - -#endif diff --git a/src/secp256k1/src/modules/schnorr/schnorr.h b/src/secp256k1/src/modules/schnorr/schnorr.h deleted file mode 100644 index de18147bd5..0000000000 --- a/src/secp256k1/src/modules/schnorr/schnorr.h +++ /dev/null @@ -1,20 +0,0 @@ -/*********************************************************************** - * Copyright (c) 2014-2015 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php. * - ***********************************************************************/ - -#ifndef _SECP256K1_MODULE_SCHNORR_H_ -#define _SECP256K1_MODULE_SCHNORR_H_ - -#include "scalar.h" -#include "group.h" - -typedef void (*secp256k1_schnorr_msghash)(unsigned char *h32, const unsigned char *r32, const unsigned char *msg32); - -static int secp256k1_schnorr_sig_sign(const secp256k1_ecmult_gen_context* ctx, unsigned char *sig64, const secp256k1_scalar *key, const secp256k1_scalar *nonce, const secp256k1_ge *pubnonce, secp256k1_schnorr_msghash hash, const unsigned char *msg32); -static int secp256k1_schnorr_sig_verify(const secp256k1_ecmult_context* ctx, const unsigned char *sig64, const secp256k1_ge *pubkey, secp256k1_schnorr_msghash hash, const unsigned char *msg32); -static int secp256k1_schnorr_sig_recover(const secp256k1_ecmult_context* ctx, const unsigned char *sig64, secp256k1_ge *pubkey, secp256k1_schnorr_msghash hash, const unsigned char *msg32); -static int secp256k1_schnorr_sig_combine(unsigned char *sig64, size_t n, const unsigned char * const *sig64ins); - -#endif diff --git a/src/secp256k1/src/modules/schnorr/schnorr_impl.h b/src/secp256k1/src/modules/schnorr/schnorr_impl.h deleted file mode 100644 index e13ab6db7c..0000000000 --- a/src/secp256k1/src/modules/schnorr/schnorr_impl.h +++ /dev/null @@ -1,207 +0,0 @@ -/*********************************************************************** - * Copyright (c) 2014-2015 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php. * - ***********************************************************************/ - -#ifndef _SECP256K1_SCHNORR_IMPL_H_ -#define _SECP256K1_SCHNORR_IMPL_H_ - -#include <string.h> - -#include "schnorr.h" -#include "num.h" -#include "field.h" -#include "group.h" -#include "ecmult.h" -#include "ecmult_gen.h" - -/** - * Custom Schnorr-based signature scheme. They support multiparty signing, public key - * recovery and batch validation. - * - * Rationale for verifying R's y coordinate: - * In order to support batch validation and public key recovery, the full R point must - * be known to verifiers, rather than just its x coordinate. In order to not risk - * being more strict in batch validation than normal validation, validators must be - * required to reject signatures with incorrect y coordinate. This is only possible - * by including a (relatively slow) field inverse, or a field square root. However, - * batch validation offers potentially much higher benefits than this cost. - * - * Rationale for having an implicit y coordinate oddness: - * If we commit to having the full R point known to verifiers, there are two mechanism. - * Either include its oddness in the signature, or give it an implicit fixed value. - * As the R y coordinate can be flipped by a simple negation of the nonce, we choose the - * latter, as it comes with nearly zero impact on signing or validation performance, and - * saves a byte in the signature. - * - * Signing: - * Inputs: 32-byte message m, 32-byte scalar key x (!=0), 32-byte scalar nonce k (!=0) - * - * Compute point R = k * G. Reject nonce if R's y coordinate is odd (or negate nonce). - * Compute 32-byte r, the serialization of R's x coordinate. - * Compute scalar h = Hash(r || m). Reject nonce if h == 0 or h >= order. - * Compute scalar s = k - h * x. - * The signature is (r, s). - * - * - * Verification: - * Inputs: 32-byte message m, public key point Q, signature: (32-byte r, scalar s) - * - * Signature is invalid if s >= order. - * Signature is invalid if r >= p. - * Compute scalar h = Hash(r || m). Signature is invalid if h == 0 or h >= order. - * Option 1 (faster for single verification): - * Compute point R = h * Q + s * G. Signature is invalid if R is infinity or R's y coordinate is odd. - * Signature is valid if the serialization of R's x coordinate equals r. - * Option 2 (allows batch validation and pubkey recovery): - * Decompress x coordinate r into point R, with odd y coordinate. Fail if R is not on the curve. - * Signature is valid if R + h * Q + s * G == 0. - */ - -static int secp256k1_schnorr_sig_sign(const secp256k1_ecmult_gen_context* ctx, unsigned char *sig64, const secp256k1_scalar *key, const secp256k1_scalar *nonce, const secp256k1_ge *pubnonce, secp256k1_schnorr_msghash hash, const unsigned char *msg32) { - secp256k1_gej Rj; - secp256k1_ge Ra; - unsigned char h32[32]; - secp256k1_scalar h, s; - int overflow; - secp256k1_scalar n; - - if (secp256k1_scalar_is_zero(key) || secp256k1_scalar_is_zero(nonce)) { - return 0; - } - n = *nonce; - - secp256k1_ecmult_gen(ctx, &Rj, &n); - if (pubnonce != NULL) { - secp256k1_gej_add_ge(&Rj, &Rj, pubnonce); - } - secp256k1_ge_set_gej(&Ra, &Rj); - secp256k1_fe_normalize(&Ra.y); - if (secp256k1_fe_is_odd(&Ra.y)) { - /* R's y coordinate is odd, which is not allowed (see rationale above). - Force it to be even by negating the nonce. Note that this even works - for multiparty signing, as the R point is known to all participants, - which can all decide to flip the sign in unison, resulting in the - overall R point to be negated too. */ - secp256k1_scalar_negate(&n, &n); - } - secp256k1_fe_normalize(&Ra.x); - secp256k1_fe_get_b32(sig64, &Ra.x); - hash(h32, sig64, msg32); - overflow = 0; - secp256k1_scalar_set_b32(&h, h32, &overflow); - if (overflow || secp256k1_scalar_is_zero(&h)) { - secp256k1_scalar_clear(&n); - return 0; - } - secp256k1_scalar_mul(&s, &h, key); - secp256k1_scalar_negate(&s, &s); - secp256k1_scalar_add(&s, &s, &n); - secp256k1_scalar_clear(&n); - secp256k1_scalar_get_b32(sig64 + 32, &s); - return 1; -} - -static int secp256k1_schnorr_sig_verify(const secp256k1_ecmult_context* ctx, const unsigned char *sig64, const secp256k1_ge *pubkey, secp256k1_schnorr_msghash hash, const unsigned char *msg32) { - secp256k1_gej Qj, Rj; - secp256k1_ge Ra; - secp256k1_fe Rx; - secp256k1_scalar h, s; - unsigned char hh[32]; - int overflow; - - if (secp256k1_ge_is_infinity(pubkey)) { - return 0; - } - hash(hh, sig64, msg32); - overflow = 0; - secp256k1_scalar_set_b32(&h, hh, &overflow); - if (overflow || secp256k1_scalar_is_zero(&h)) { - return 0; - } - overflow = 0; - secp256k1_scalar_set_b32(&s, sig64 + 32, &overflow); - if (overflow) { - return 0; - } - if (!secp256k1_fe_set_b32(&Rx, sig64)) { - return 0; - } - secp256k1_gej_set_ge(&Qj, pubkey); - secp256k1_ecmult(ctx, &Rj, &Qj, &h, &s); - if (secp256k1_gej_is_infinity(&Rj)) { - return 0; - } - secp256k1_ge_set_gej_var(&Ra, &Rj); - secp256k1_fe_normalize_var(&Ra.y); - if (secp256k1_fe_is_odd(&Ra.y)) { - return 0; - } - return secp256k1_fe_equal_var(&Rx, &Ra.x); -} - -static int secp256k1_schnorr_sig_recover(const secp256k1_ecmult_context* ctx, const unsigned char *sig64, secp256k1_ge *pubkey, secp256k1_schnorr_msghash hash, const unsigned char *msg32) { - secp256k1_gej Qj, Rj; - secp256k1_ge Ra; - secp256k1_fe Rx; - secp256k1_scalar h, s; - unsigned char hh[32]; - int overflow; - - hash(hh, sig64, msg32); - overflow = 0; - secp256k1_scalar_set_b32(&h, hh, &overflow); - if (overflow || secp256k1_scalar_is_zero(&h)) { - return 0; - } - overflow = 0; - secp256k1_scalar_set_b32(&s, sig64 + 32, &overflow); - if (overflow) { - return 0; - } - if (!secp256k1_fe_set_b32(&Rx, sig64)) { - return 0; - } - if (!secp256k1_ge_set_xo_var(&Ra, &Rx, 0)) { - return 0; - } - secp256k1_gej_set_ge(&Rj, &Ra); - secp256k1_scalar_inverse_var(&h, &h); - secp256k1_scalar_negate(&s, &s); - secp256k1_scalar_mul(&s, &s, &h); - secp256k1_ecmult(ctx, &Qj, &Rj, &h, &s); - if (secp256k1_gej_is_infinity(&Qj)) { - return 0; - } - secp256k1_ge_set_gej(pubkey, &Qj); - return 1; -} - -static int secp256k1_schnorr_sig_combine(unsigned char *sig64, size_t n, const unsigned char * const *sig64ins) { - secp256k1_scalar s = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); - size_t i; - for (i = 0; i < n; i++) { - secp256k1_scalar si; - int overflow; - secp256k1_scalar_set_b32(&si, sig64ins[i] + 32, &overflow); - if (overflow) { - return -1; - } - if (i) { - if (memcmp(sig64ins[i - 1], sig64ins[i], 32) != 0) { - return -1; - } - } - secp256k1_scalar_add(&s, &s, &si); - } - if (secp256k1_scalar_is_zero(&s)) { - return 0; - } - memcpy(sig64, sig64ins[0], 32); - secp256k1_scalar_get_b32(sig64 + 32, &s); - secp256k1_scalar_clear(&s); - return 1; -} - -#endif diff --git a/src/secp256k1/src/modules/schnorr/tests_impl.h b/src/secp256k1/src/modules/schnorr/tests_impl.h deleted file mode 100644 index 5bd14a03e3..0000000000 --- a/src/secp256k1/src/modules/schnorr/tests_impl.h +++ /dev/null @@ -1,175 +0,0 @@ -/********************************************************************** - * Copyright (c) 2014-2015 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php.* - **********************************************************************/ - -#ifndef SECP256K1_MODULE_SCHNORR_TESTS -#define SECP256K1_MODULE_SCHNORR_TESTS - -#include "include/secp256k1_schnorr.h" - -void test_schnorr_end_to_end(void) { - unsigned char privkey[32]; - unsigned char message[32]; - unsigned char schnorr_signature[64]; - secp256k1_pubkey pubkey, recpubkey; - - /* Generate a random key and message. */ - { - secp256k1_scalar key; - random_scalar_order_test(&key); - secp256k1_scalar_get_b32(privkey, &key); - secp256k1_rand256_test(message); - } - - /* Construct and verify corresponding public key. */ - CHECK(secp256k1_ec_seckey_verify(ctx, privkey) == 1); - CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey, privkey) == 1); - - /* Schnorr sign. */ - CHECK(secp256k1_schnorr_sign(ctx, schnorr_signature, message, privkey, NULL, NULL) == 1); - CHECK(secp256k1_schnorr_verify(ctx, schnorr_signature, message, &pubkey) == 1); - CHECK(secp256k1_schnorr_recover(ctx, &recpubkey, schnorr_signature, message) == 1); - CHECK(memcmp(&pubkey, &recpubkey, sizeof(pubkey)) == 0); - /* Destroy signature and verify again. */ - schnorr_signature[secp256k1_rand_bits(6)] += 1 + secp256k1_rand_int(255); - CHECK(secp256k1_schnorr_verify(ctx, schnorr_signature, message, &pubkey) == 0); - CHECK(secp256k1_schnorr_recover(ctx, &recpubkey, schnorr_signature, message) != 1 || - memcmp(&pubkey, &recpubkey, sizeof(pubkey)) != 0); -} - -/** Horribly broken hash function. Do not use for anything but tests. */ -void test_schnorr_hash(unsigned char *h32, const unsigned char *r32, const unsigned char *msg32) { - int i; - for (i = 0; i < 32; i++) { - h32[i] = r32[i] ^ msg32[i]; - } -} - -void test_schnorr_sign_verify(void) { - unsigned char msg32[32]; - unsigned char sig64[3][64]; - secp256k1_gej pubkeyj[3]; - secp256k1_ge pubkey[3]; - secp256k1_scalar nonce[3], key[3]; - int i = 0; - int k; - - secp256k1_rand256_test(msg32); - - for (k = 0; k < 3; k++) { - random_scalar_order_test(&key[k]); - - do { - random_scalar_order_test(&nonce[k]); - if (secp256k1_schnorr_sig_sign(&ctx->ecmult_gen_ctx, sig64[k], &key[k], &nonce[k], NULL, &test_schnorr_hash, msg32)) { - break; - } - } while(1); - - secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pubkeyj[k], &key[k]); - secp256k1_ge_set_gej_var(&pubkey[k], &pubkeyj[k]); - CHECK(secp256k1_schnorr_sig_verify(&ctx->ecmult_ctx, sig64[k], &pubkey[k], &test_schnorr_hash, msg32)); - - for (i = 0; i < 4; i++) { - int pos = secp256k1_rand_bits(6); - int mod = 1 + secp256k1_rand_int(255); - sig64[k][pos] ^= mod; - CHECK(secp256k1_schnorr_sig_verify(&ctx->ecmult_ctx, sig64[k], &pubkey[k], &test_schnorr_hash, msg32) == 0); - sig64[k][pos] ^= mod; - } - } -} - -void test_schnorr_threshold(void) { - unsigned char msg[32]; - unsigned char sec[5][32]; - secp256k1_pubkey pub[5]; - unsigned char nonce[5][32]; - secp256k1_pubkey pubnonce[5]; - unsigned char sig[5][64]; - const unsigned char* sigs[5]; - unsigned char allsig[64]; - const secp256k1_pubkey* pubs[5]; - secp256k1_pubkey allpub; - int n, i; - int damage; - int ret = 0; - - damage = secp256k1_rand_bits(1) ? (1 + secp256k1_rand_int(4)) : 0; - secp256k1_rand256_test(msg); - n = 2 + secp256k1_rand_int(4); - for (i = 0; i < n; i++) { - do { - secp256k1_rand256_test(sec[i]); - } while (!secp256k1_ec_seckey_verify(ctx, sec[i])); - CHECK(secp256k1_ec_pubkey_create(ctx, &pub[i], sec[i])); - CHECK(secp256k1_schnorr_generate_nonce_pair(ctx, &pubnonce[i], nonce[i], msg, sec[i], NULL, NULL)); - pubs[i] = &pub[i]; - } - if (damage == 1) { - nonce[secp256k1_rand_int(n)][secp256k1_rand_int(32)] ^= 1 + secp256k1_rand_int(255); - } else if (damage == 2) { - sec[secp256k1_rand_int(n)][secp256k1_rand_int(32)] ^= 1 + secp256k1_rand_int(255); - } - for (i = 0; i < n; i++) { - secp256k1_pubkey allpubnonce; - const secp256k1_pubkey *pubnonces[4]; - int j; - for (j = 0; j < i; j++) { - pubnonces[j] = &pubnonce[j]; - } - for (j = i + 1; j < n; j++) { - pubnonces[j - 1] = &pubnonce[j]; - } - CHECK(secp256k1_ec_pubkey_combine(ctx, &allpubnonce, pubnonces, n - 1)); - ret |= (secp256k1_schnorr_partial_sign(ctx, sig[i], msg, sec[i], &allpubnonce, nonce[i]) != 1) * 1; - sigs[i] = sig[i]; - } - if (damage == 3) { - sig[secp256k1_rand_int(n)][secp256k1_rand_bits(6)] ^= 1 + secp256k1_rand_int(255); - } - ret |= (secp256k1_ec_pubkey_combine(ctx, &allpub, pubs, n) != 1) * 2; - if ((ret & 1) == 0) { - ret |= (secp256k1_schnorr_partial_combine(ctx, allsig, sigs, n) != 1) * 4; - } - if (damage == 4) { - allsig[secp256k1_rand_int(32)] ^= 1 + secp256k1_rand_int(255); - } - if ((ret & 7) == 0) { - ret |= (secp256k1_schnorr_verify(ctx, allsig, msg, &allpub) != 1) * 8; - } - CHECK((ret == 0) == (damage == 0)); -} - -void test_schnorr_recovery(void) { - unsigned char msg32[32]; - unsigned char sig64[64]; - secp256k1_ge Q; - - secp256k1_rand256_test(msg32); - secp256k1_rand256_test(sig64); - secp256k1_rand256_test(sig64 + 32); - if (secp256k1_schnorr_sig_recover(&ctx->ecmult_ctx, sig64, &Q, &test_schnorr_hash, msg32) == 1) { - CHECK(secp256k1_schnorr_sig_verify(&ctx->ecmult_ctx, sig64, &Q, &test_schnorr_hash, msg32) == 1); - } -} - -void run_schnorr_tests(void) { - int i; - for (i = 0; i < 32*count; i++) { - test_schnorr_end_to_end(); - } - for (i = 0; i < 32 * count; i++) { - test_schnorr_sign_verify(); - } - for (i = 0; i < 16 * count; i++) { - test_schnorr_recovery(); - } - for (i = 0; i < 10 * count; i++) { - test_schnorr_threshold(); - } -} - -#endif diff --git a/src/secp256k1/src/scalar.h b/src/secp256k1/src/scalar.h index b590ccd6dd..27e9d8375e 100644 --- a/src/secp256k1/src/scalar.h +++ b/src/secp256k1/src/scalar.h @@ -13,7 +13,9 @@ #include "libsecp256k1-config.h" #endif -#if defined(USE_SCALAR_4X64) +#if defined(EXHAUSTIVE_TEST_ORDER) +#include "scalar_low.h" +#elif defined(USE_SCALAR_4X64) #include "scalar_4x64.h" #elif defined(USE_SCALAR_8X32) #include "scalar_8x32.h" diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index aa2703dd23..56e7bd82af 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -282,8 +282,8 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "movq 56(%%rsi), %%r14\n" /* Initialize r8,r9,r10 */ "movq 0(%%rsi), %%r8\n" - "movq $0, %%r9\n" - "movq $0, %%r10\n" + "xorq %%r9, %%r9\n" + "xorq %%r10, %%r10\n" /* (r8,r9) += n0 * c0 */ "movq %8, %%rax\n" "mulq %%r11\n" @@ -291,7 +291,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq %%rdx, %%r9\n" /* extract m0 */ "movq %%r8, %q0\n" - "movq $0, %%r8\n" + "xorq %%r8, %%r8\n" /* (r9,r10) += l1 */ "addq 8(%%rsi), %%r9\n" "adcq $0, %%r10\n" @@ -309,7 +309,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r8\n" /* extract m1 */ "movq %%r9, %q1\n" - "movq $0, %%r9\n" + "xorq %%r9, %%r9\n" /* (r10,r8,r9) += l2 */ "addq 16(%%rsi), %%r10\n" "adcq $0, %%r8\n" @@ -332,7 +332,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r9\n" /* extract m2 */ "movq %%r10, %q2\n" - "movq $0, %%r10\n" + "xorq %%r10, %%r10\n" /* (r8,r9,r10) += l3 */ "addq 24(%%rsi), %%r8\n" "adcq $0, %%r9\n" @@ -355,7 +355,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r10\n" /* extract m3 */ "movq %%r8, %q3\n" - "movq $0, %%r8\n" + "xorq %%r8, %%r8\n" /* (r9,r10,r8) += n3 * c1 */ "movq %9, %%rax\n" "mulq %%r14\n" @@ -387,8 +387,8 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "movq %q11, %%r13\n" /* Initialize (r8,r9,r10) */ "movq %q5, %%r8\n" - "movq $0, %%r9\n" - "movq $0, %%r10\n" + "xorq %%r9, %%r9\n" + "xorq %%r10, %%r10\n" /* (r8,r9) += m4 * c0 */ "movq %12, %%rax\n" "mulq %%r11\n" @@ -396,7 +396,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq %%rdx, %%r9\n" /* extract p0 */ "movq %%r8, %q0\n" - "movq $0, %%r8\n" + "xorq %%r8, %%r8\n" /* (r9,r10) += m1 */ "addq %q6, %%r9\n" "adcq $0, %%r10\n" @@ -414,7 +414,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r8\n" /* extract p1 */ "movq %%r9, %q1\n" - "movq $0, %%r9\n" + "xorq %%r9, %%r9\n" /* (r10,r8,r9) += m2 */ "addq %q7, %%r10\n" "adcq $0, %%r8\n" @@ -472,7 +472,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "movq %%rax, 0(%q6)\n" /* Move to (r8,r9) */ "movq %%rdx, %%r8\n" - "movq $0, %%r9\n" + "xorq %%r9, %%r9\n" /* (r8,r9) += p1 */ "addq %q2, %%r8\n" "adcq $0, %%r9\n" @@ -483,7 +483,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq %%rdx, %%r9\n" /* Extract r1 */ "movq %%r8, 8(%q6)\n" - "movq $0, %%r8\n" + "xorq %%r8, %%r8\n" /* (r9,r8) += p4 */ "addq %%r10, %%r9\n" "adcq $0, %%r8\n" @@ -492,7 +492,7 @@ static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) "adcq $0, %%r8\n" /* Extract r2 */ "movq %%r9, 16(%q6)\n" - "movq $0, %%r9\n" + "xorq %%r9, %%r9\n" /* (r8,r9) += p3 */ "addq %q4, %%r8\n" "adcq $0, %%r9\n" diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index c5baf4df41..f5b2376407 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -14,7 +14,9 @@ #include "libsecp256k1-config.h" #endif -#if defined(USE_SCALAR_4X64) +#if defined(EXHAUSTIVE_TEST_ORDER) +#include "scalar_low_impl.h" +#elif defined(USE_SCALAR_4X64) #include "scalar_4x64_impl.h" #elif defined(USE_SCALAR_8X32) #include "scalar_8x32_impl.h" @@ -31,17 +33,37 @@ static void secp256k1_scalar_get_num(secp256k1_num *r, const secp256k1_scalar *a /** secp256k1 curve order, see secp256k1_ecdsa_const_order_as_fe in ecdsa_impl.h */ static void secp256k1_scalar_order_get_num(secp256k1_num *r) { +#if defined(EXHAUSTIVE_TEST_ORDER) + static const unsigned char order[32] = { + 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,EXHAUSTIVE_TEST_ORDER + }; +#else static const unsigned char order[32] = { 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF, 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFE, 0xBA,0xAE,0xDC,0xE6,0xAF,0x48,0xA0,0x3B, 0xBF,0xD2,0x5E,0x8C,0xD0,0x36,0x41,0x41 }; +#endif secp256k1_num_set_bin(r, order, 32); } #endif static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar *x) { +#if defined(EXHAUSTIVE_TEST_ORDER) + int i; + *r = 0; + for (i = 0; i < EXHAUSTIVE_TEST_ORDER; i++) + if ((i * *x) % EXHAUSTIVE_TEST_ORDER == 1) + *r = i; + /* If this VERIFY_CHECK triggers we were given a noninvertible scalar (and thus + * have a composite group order; fix it in exhaustive_tests.c). */ + VERIFY_CHECK(*r != 0); +} +#else secp256k1_scalar *t; int i; /* First compute x ^ (2^N - 1) for some values of N. */ @@ -233,9 +255,9 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar } SECP256K1_INLINE static int secp256k1_scalar_is_even(const secp256k1_scalar *a) { - /* d[0] is present and is the lowest word for all representations */ return !(a->d[0] & 1); } +#endif static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_scalar *x) { #if defined(USE_SCALAR_INV_BUILTIN) @@ -259,6 +281,18 @@ static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_sc } #ifdef USE_ENDOMORPHISM +#if defined(EXHAUSTIVE_TEST_ORDER) +/** + * Find k1 and k2 given k, such that k1 + k2 * lambda == k mod n; unlike in the + * full case we don't bother making k1 and k2 be small, we just want them to be + * nontrivial to get full test coverage for the exhaustive tests. We therefore + * (arbitrarily) set k2 = k + 5 and k1 = k - k2 * lambda. + */ +static void secp256k1_scalar_split_lambda(secp256k1_scalar *r1, secp256k1_scalar *r2, const secp256k1_scalar *a) { + *r2 = (*a + 5) % EXHAUSTIVE_TEST_ORDER; + *r1 = (*a + (EXHAUSTIVE_TEST_ORDER - *r2) * EXHAUSTIVE_TEST_LAMBDA) % EXHAUSTIVE_TEST_ORDER; +} +#else /** * The Secp256k1 curve has an endomorphism, where lambda * (x, y) = (beta * x, y), where * lambda is {0x53,0x63,0xad,0x4c,0xc0,0x5c,0x30,0xe0,0xa5,0x26,0x1c,0x02,0x88,0x12,0x64,0x5a, @@ -331,5 +365,6 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar *r1, secp256k1_scalar secp256k1_scalar_add(r1, r1, a); } #endif +#endif #endif diff --git a/src/secp256k1/src/scalar_low.h b/src/secp256k1/src/scalar_low.h new file mode 100644 index 0000000000..5574c44c7a --- /dev/null +++ b/src/secp256k1/src/scalar_low.h @@ -0,0 +1,15 @@ +/********************************************************************** + * Copyright (c) 2015 Andrew Poelstra * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _SECP256K1_SCALAR_REPR_ +#define _SECP256K1_SCALAR_REPR_ + +#include <stdint.h> + +/** A scalar modulo the group order of the secp256k1 curve. */ +typedef uint32_t secp256k1_scalar; + +#endif diff --git a/src/secp256k1/src/scalar_low_impl.h b/src/secp256k1/src/scalar_low_impl.h new file mode 100644 index 0000000000..4f94441f49 --- /dev/null +++ b/src/secp256k1/src/scalar_low_impl.h @@ -0,0 +1,114 @@ +/********************************************************************** + * Copyright (c) 2015 Andrew Poelstra * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _SECP256K1_SCALAR_REPR_IMPL_H_ +#define _SECP256K1_SCALAR_REPR_IMPL_H_ + +#include "scalar.h" + +#include <string.h> + +SECP256K1_INLINE static int secp256k1_scalar_is_even(const secp256k1_scalar *a) { + return !(*a & 1); +} + +SECP256K1_INLINE static void secp256k1_scalar_clear(secp256k1_scalar *r) { *r = 0; } +SECP256K1_INLINE static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsigned int v) { *r = v; } + +SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { + if (offset < 32) + return ((*a >> offset) & ((((uint32_t)1) << count) - 1)); + else + return 0; +} + +SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits_var(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { + return secp256k1_scalar_get_bits(a, offset, count); +} + +SECP256K1_INLINE static int secp256k1_scalar_check_overflow(const secp256k1_scalar *a) { return *a >= EXHAUSTIVE_TEST_ORDER; } + +static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { + *r = (*a + *b) % EXHAUSTIVE_TEST_ORDER; + return *r < *b; +} + +static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { + if (flag && bit < 32) + *r += (1 << bit); +#ifdef VERIFY + VERIFY_CHECK(secp256k1_scalar_check_overflow(r) == 0); +#endif +} + +static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b32, int *overflow) { + const int base = 0x100 % EXHAUSTIVE_TEST_ORDER; + int i; + *r = 0; + for (i = 0; i < 32; i++) { + *r = ((*r * base) + b32[i]) % EXHAUSTIVE_TEST_ORDER; + } + /* just deny overflow, it basically always happens */ + if (overflow) *overflow = 0; +} + +static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* a) { + memset(bin, 0, 32); + bin[28] = *a >> 24; bin[29] = *a >> 16; bin[30] = *a >> 8; bin[31] = *a; +} + +SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) { + return *a == 0; +} + +static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar *a) { + if (*a == 0) { + *r = 0; + } else { + *r = EXHAUSTIVE_TEST_ORDER - *a; + } +} + +SECP256K1_INLINE static int secp256k1_scalar_is_one(const secp256k1_scalar *a) { + return *a == 1; +} + +static int secp256k1_scalar_is_high(const secp256k1_scalar *a) { + return *a > EXHAUSTIVE_TEST_ORDER / 2; +} + +static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { + if (flag) secp256k1_scalar_negate(r, r); + return flag ? -1 : 1; +} + +static void secp256k1_scalar_mul(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { + *r = (*a * *b) % EXHAUSTIVE_TEST_ORDER; +} + +static int secp256k1_scalar_shr_int(secp256k1_scalar *r, int n) { + int ret; + VERIFY_CHECK(n > 0); + VERIFY_CHECK(n < 16); + ret = *r & ((1 << n) - 1); + *r >>= n; + return ret; +} + +static void secp256k1_scalar_sqr(secp256k1_scalar *r, const secp256k1_scalar *a) { + *r = (*a * *a) % EXHAUSTIVE_TEST_ORDER; +} + +static void secp256k1_scalar_split_128(secp256k1_scalar *r1, secp256k1_scalar *r2, const secp256k1_scalar *a) { + *r1 = *a; + *r2 = 0; +} + +SECP256K1_INLINE static int secp256k1_scalar_eq(const secp256k1_scalar *a, const secp256k1_scalar *b) { + return *a == *b; +} + +#endif diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index 7973d60c36..fb8b882faa 100644..100755 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -359,16 +359,15 @@ int secp256k1_ecdsa_sign(const secp256k1_context* ctx, secp256k1_ecdsa_signature secp256k1_scalar_set_b32(&sec, seckey, &overflow); /* Fail if the secret key is invalid. */ if (!overflow && !secp256k1_scalar_is_zero(&sec)) { + unsigned char nonce32[32]; unsigned int count = 0; secp256k1_scalar_set_b32(&msg, msg32, NULL); while (1) { - unsigned char nonce32[32]; ret = noncefp(nonce32, msg32, seckey, NULL, (void*)noncedata, count); if (!ret) { break; } secp256k1_scalar_set_b32(&non, nonce32, &overflow); - memset(nonce32, 0, 32); if (!overflow && !secp256k1_scalar_is_zero(&non)) { if (secp256k1_ecdsa_sig_sign(&ctx->ecmult_gen_ctx, &r, &s, &sec, &msg, &non, NULL)) { break; @@ -376,6 +375,7 @@ int secp256k1_ecdsa_sign(const secp256k1_context* ctx, secp256k1_ecdsa_signature } count++; } + memset(nonce32, 0, 32); secp256k1_scalar_clear(&msg); secp256k1_scalar_clear(&non); secp256k1_scalar_clear(&sec); diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index b32cb90813..9ae7d30281 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -520,7 +520,7 @@ void test_num_mod(void) { secp256k1_num order, n; /* check that 0 mod anything is 0 */ - random_scalar_order_test(&s); + random_scalar_order_test(&s); secp256k1_scalar_get_num(&order, &s); secp256k1_scalar_set_int(&s, 0); secp256k1_scalar_get_num(&n, &s); @@ -535,7 +535,7 @@ void test_num_mod(void) { CHECK(secp256k1_num_is_zero(&n)); /* check that increasing the number past 2^256 does not break this */ - random_scalar_order_test(&s); + random_scalar_order_test(&s); secp256k1_scalar_get_num(&n, &s); /* multiply by 2^8, which'll test this case with high probability */ for (i = 0; i < 8; ++i) { @@ -568,7 +568,7 @@ void test_num_jacobi(void) { /* we first need a scalar which is not a multiple of 5 */ do { secp256k1_num fiven; - random_scalar_order_test(&sqr); + random_scalar_order_test(&sqr); secp256k1_scalar_get_num(&fiven, &five); secp256k1_scalar_get_num(&n, &sqr); secp256k1_num_mod(&n, &fiven); @@ -587,7 +587,7 @@ void test_num_jacobi(void) { /** test with secp group order as order */ secp256k1_scalar_order_get_num(&order); - random_scalar_order_test(&sqr); + random_scalar_order_test(&sqr); secp256k1_scalar_sqr(&sqr, &sqr); /* test residue */ secp256k1_scalar_get_num(&n, &sqr); @@ -1733,18 +1733,18 @@ void run_field_inv_all_var(void) { secp256k1_fe x[16], xi[16], xii[16]; int i; /* Check it's safe to call for 0 elements */ - secp256k1_fe_inv_all_var(0, xi, x); + secp256k1_fe_inv_all_var(xi, x, 0); for (i = 0; i < count; i++) { size_t j; size_t len = secp256k1_rand_int(15) + 1; for (j = 0; j < len; j++) { random_fe_non_zero(&x[j]); } - secp256k1_fe_inv_all_var(len, xi, x); + secp256k1_fe_inv_all_var(xi, x, len); for (j = 0; j < len; j++) { CHECK(check_fe_inverse(&x[j], &xi[j])); } - secp256k1_fe_inv_all_var(len, xii, xi); + secp256k1_fe_inv_all_var(xii, xi, len); for (j = 0; j < len; j++) { CHECK(check_fe_equal(&x[j], &xii[j])); } @@ -1930,7 +1930,7 @@ void test_ge(void) { zs[i] = gej[i].z; } } - secp256k1_fe_inv_all_var(4 * runs + 1, zinv, zs); + secp256k1_fe_inv_all_var(zinv, zs, 4 * runs + 1); free(zs); } @@ -2050,8 +2050,8 @@ void test_ge(void) { secp256k1_fe_mul(&zr[i + 1], &zinv[i], &gej[i + 1].z); } } - secp256k1_ge_set_table_gej_var(4 * runs + 1, ge_set_table, gej, zr); - secp256k1_ge_set_all_gej_var(4 * runs + 1, ge_set_all, gej, &ctx->error_callback); + secp256k1_ge_set_table_gej_var(ge_set_table, gej, zr, 4 * runs + 1); + secp256k1_ge_set_all_gej_var(ge_set_all, gej, 4 * runs + 1, &ctx->error_callback); for (i = 0; i < 4 * runs + 1; i++) { secp256k1_fe s; random_fe_non_zero(&s); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c new file mode 100644 index 0000000000..bda6ee475c --- /dev/null +++ b/src/secp256k1/src/tests_exhaustive.c @@ -0,0 +1,329 @@ +/*********************************************************************** + * Copyright (c) 2016 Andrew Poelstra * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#if defined HAVE_CONFIG_H +#include "libsecp256k1-config.h" +#endif + +#include <stdio.h> +#include <stdlib.h> + +#include <time.h> + +#undef USE_ECMULT_STATIC_PRECOMPUTATION + +#ifndef EXHAUSTIVE_TEST_ORDER +/* see group_impl.h for allowable values */ +#define EXHAUSTIVE_TEST_ORDER 13 +#define EXHAUSTIVE_TEST_LAMBDA 9 /* cube root of 1 mod 13 */ +#endif + +#include "include/secp256k1.h" +#include "group.h" +#include "secp256k1.c" +#include "testrand_impl.h" + +/** stolen from tests.c */ +void ge_equals_ge(const secp256k1_ge *a, const secp256k1_ge *b) { + CHECK(a->infinity == b->infinity); + if (a->infinity) { + return; + } + CHECK(secp256k1_fe_equal_var(&a->x, &b->x)); + CHECK(secp256k1_fe_equal_var(&a->y, &b->y)); +} + +void ge_equals_gej(const secp256k1_ge *a, const secp256k1_gej *b) { + secp256k1_fe z2s; + secp256k1_fe u1, u2, s1, s2; + CHECK(a->infinity == b->infinity); + if (a->infinity) { + return; + } + /* Check a.x * b.z^2 == b.x && a.y * b.z^3 == b.y, to avoid inverses. */ + secp256k1_fe_sqr(&z2s, &b->z); + secp256k1_fe_mul(&u1, &a->x, &z2s); + u2 = b->x; secp256k1_fe_normalize_weak(&u2); + secp256k1_fe_mul(&s1, &a->y, &z2s); secp256k1_fe_mul(&s1, &s1, &b->z); + s2 = b->y; secp256k1_fe_normalize_weak(&s2); + CHECK(secp256k1_fe_equal_var(&u1, &u2)); + CHECK(secp256k1_fe_equal_var(&s1, &s2)); +} + +void random_fe(secp256k1_fe *x) { + unsigned char bin[32]; + do { + secp256k1_rand256(bin); + if (secp256k1_fe_set_b32(x, bin)) { + return; + } + } while(1); +} +/** END stolen from tests.c */ + +int secp256k1_nonce_function_smallint(unsigned char *nonce32, const unsigned char *msg32, + const unsigned char *key32, const unsigned char *algo16, + void *data, unsigned int attempt) { + secp256k1_scalar s; + int *idata = data; + (void)msg32; + (void)key32; + (void)algo16; + /* Some nonces cannot be used because they'd cause s and/or r to be zero. + * The signing function has retry logic here that just re-calls the nonce + * function with an increased `attempt`. So if attempt > 0 this means we + * need to change the nonce to avoid an infinite loop. */ + if (attempt > 0) { + (*idata)++; + } + secp256k1_scalar_set_int(&s, *idata); + secp256k1_scalar_get_b32(nonce32, &s); + return 1; +} + +#ifdef USE_ENDOMORPHISM +void test_exhaustive_endomorphism(const secp256k1_ge *group, int order) { + int i; + for (i = 0; i < order; i++) { + secp256k1_ge res; + secp256k1_ge_mul_lambda(&res, &group[i]); + ge_equals_ge(&group[i * EXHAUSTIVE_TEST_LAMBDA % EXHAUSTIVE_TEST_ORDER], &res); + } +} +#endif + +void test_exhaustive_addition(const secp256k1_ge *group, const secp256k1_gej *groupj, int order) { + int i, j; + + /* Sanity-check (and check infinity functions) */ + CHECK(secp256k1_ge_is_infinity(&group[0])); + CHECK(secp256k1_gej_is_infinity(&groupj[0])); + for (i = 1; i < order; i++) { + CHECK(!secp256k1_ge_is_infinity(&group[i])); + CHECK(!secp256k1_gej_is_infinity(&groupj[i])); + } + + /* Check all addition formulae */ + for (j = 0; j < order; j++) { + secp256k1_fe fe_inv; + secp256k1_fe_inv(&fe_inv, &groupj[j].z); + for (i = 0; i < order; i++) { + secp256k1_ge zless_gej; + secp256k1_gej tmp; + /* add_var */ + secp256k1_gej_add_var(&tmp, &groupj[i], &groupj[j], NULL); + ge_equals_gej(&group[(i + j) % order], &tmp); + /* add_ge */ + if (j > 0) { + secp256k1_gej_add_ge(&tmp, &groupj[i], &group[j]); + ge_equals_gej(&group[(i + j) % order], &tmp); + } + /* add_ge_var */ + secp256k1_gej_add_ge_var(&tmp, &groupj[i], &group[j], NULL); + ge_equals_gej(&group[(i + j) % order], &tmp); + /* add_zinv_var */ + zless_gej.infinity = groupj[j].infinity; + zless_gej.x = groupj[j].x; + zless_gej.y = groupj[j].y; + secp256k1_gej_add_zinv_var(&tmp, &groupj[i], &zless_gej, &fe_inv); + ge_equals_gej(&group[(i + j) % order], &tmp); + } + } + + /* Check doubling */ + for (i = 0; i < order; i++) { + secp256k1_gej tmp; + if (i > 0) { + secp256k1_gej_double_nonzero(&tmp, &groupj[i], NULL); + ge_equals_gej(&group[(2 * i) % order], &tmp); + } + secp256k1_gej_double_var(&tmp, &groupj[i], NULL); + ge_equals_gej(&group[(2 * i) % order], &tmp); + } + + /* Check negation */ + for (i = 1; i < order; i++) { + secp256k1_ge tmp; + secp256k1_gej tmpj; + secp256k1_ge_neg(&tmp, &group[i]); + ge_equals_ge(&group[order - i], &tmp); + secp256k1_gej_neg(&tmpj, &groupj[i]); + ge_equals_gej(&group[order - i], &tmpj); + } +} + +void test_exhaustive_ecmult(const secp256k1_context *ctx, const secp256k1_ge *group, const secp256k1_gej *groupj, int order) { + int i, j, r_log; + for (r_log = 1; r_log < order; r_log++) { + for (j = 0; j < order; j++) { + for (i = 0; i < order; i++) { + secp256k1_gej tmp; + secp256k1_scalar na, ng; + secp256k1_scalar_set_int(&na, i); + secp256k1_scalar_set_int(&ng, j); + + secp256k1_ecmult(&ctx->ecmult_ctx, &tmp, &groupj[r_log], &na, &ng); + ge_equals_gej(&group[(i * r_log + j) % order], &tmp); + + if (i > 0) { + secp256k1_ecmult_const(&tmp, &group[i], &ng); + ge_equals_gej(&group[(i * j) % order], &tmp); + } + } + } + } +} + +void r_from_k(secp256k1_scalar *r, const secp256k1_ge *group, int k) { + secp256k1_fe x; + unsigned char x_bin[32]; + k %= EXHAUSTIVE_TEST_ORDER; + x = group[k].x; + secp256k1_fe_normalize(&x); + secp256k1_fe_get_b32(x_bin, &x); + secp256k1_scalar_set_b32(r, x_bin, NULL); +} + +void test_exhaustive_verify(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + int s, r, msg, key; + for (s = 1; s < order; s++) { + for (r = 1; r < order; r++) { + for (msg = 1; msg < order; msg++) { + for (key = 1; key < order; key++) { + secp256k1_ge nonconst_ge; + secp256k1_ecdsa_signature sig; + secp256k1_pubkey pk; + secp256k1_scalar sk_s, msg_s, r_s, s_s; + secp256k1_scalar s_times_k_s, msg_plus_r_times_sk_s; + int k, should_verify; + unsigned char msg32[32]; + + secp256k1_scalar_set_int(&s_s, s); + secp256k1_scalar_set_int(&r_s, r); + secp256k1_scalar_set_int(&msg_s, msg); + secp256k1_scalar_set_int(&sk_s, key); + + /* Verify by hand */ + /* Run through every k value that gives us this r and check that *one* works. + * Note there could be none, there could be multiple, ECDSA is weird. */ + should_verify = 0; + for (k = 0; k < order; k++) { + secp256k1_scalar check_x_s; + r_from_k(&check_x_s, group, k); + if (r_s == check_x_s) { + secp256k1_scalar_set_int(&s_times_k_s, k); + secp256k1_scalar_mul(&s_times_k_s, &s_times_k_s, &s_s); + secp256k1_scalar_mul(&msg_plus_r_times_sk_s, &r_s, &sk_s); + secp256k1_scalar_add(&msg_plus_r_times_sk_s, &msg_plus_r_times_sk_s, &msg_s); + should_verify |= secp256k1_scalar_eq(&s_times_k_s, &msg_plus_r_times_sk_s); + } + } + /* nb we have a "high s" rule */ + should_verify &= !secp256k1_scalar_is_high(&s_s); + + /* Verify by calling verify */ + secp256k1_ecdsa_signature_save(&sig, &r_s, &s_s); + memcpy(&nonconst_ge, &group[sk_s], sizeof(nonconst_ge)); + secp256k1_pubkey_save(&pk, &nonconst_ge); + secp256k1_scalar_get_b32(msg32, &msg_s); + CHECK(should_verify == + secp256k1_ecdsa_verify(ctx, &sig, msg32, &pk)); + } + } + } + } +} + +void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + int i, j, k; + + /* Loop */ + for (i = 1; i < order; i++) { /* message */ + for (j = 1; j < order; j++) { /* key */ + for (k = 1; k < order; k++) { /* nonce */ + secp256k1_ecdsa_signature sig; + secp256k1_scalar sk, msg, r, s, expected_r; + unsigned char sk32[32], msg32[32]; + secp256k1_scalar_set_int(&msg, i); + secp256k1_scalar_set_int(&sk, j); + secp256k1_scalar_get_b32(sk32, &sk); + secp256k1_scalar_get_b32(msg32, &msg); + + secp256k1_ecdsa_sign(ctx, &sig, msg32, sk32, secp256k1_nonce_function_smallint, &k); + + secp256k1_ecdsa_signature_load(ctx, &r, &s, &sig); + /* Note that we compute expected_r *after* signing -- this is important + * because our nonce-computing function function might change k during + * signing. */ + r_from_k(&expected_r, group, k); + CHECK(r == expected_r); + CHECK((k * s) % order == (i + r * j) % order || + (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + } + } + } + + /* We would like to verify zero-knowledge here by counting how often every + * possible (s, r) tuple appears, but because the group order is larger + * than the field order, when coercing the x-values to scalar values, some + * appear more often than others, so we are actually not zero-knowledge. + * (This effect also appears in the real code, but the difference is on the + * order of 1/2^128th the field order, so the deviation is not useful to a + * computationally bounded attacker.) + */ +} + +int main(void) { + int i; + secp256k1_gej groupj[EXHAUSTIVE_TEST_ORDER]; + secp256k1_ge group[EXHAUSTIVE_TEST_ORDER]; + + /* Build context */ + secp256k1_context *ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + + /* TODO set z = 1, then do num_tests runs with random z values */ + + /* Generate the entire group */ + secp256k1_gej_set_infinity(&groupj[0]); + secp256k1_ge_set_gej(&group[0], &groupj[0]); + for (i = 1; i < EXHAUSTIVE_TEST_ORDER; i++) { + /* Set a different random z-value for each Jacobian point */ + secp256k1_fe z; + random_fe(&z); + + secp256k1_gej_add_ge(&groupj[i], &groupj[i - 1], &secp256k1_ge_const_g); + secp256k1_ge_set_gej(&group[i], &groupj[i]); + secp256k1_gej_rescale(&groupj[i], &z); + + /* Verify against ecmult_gen */ + { + secp256k1_scalar scalar_i; + secp256k1_gej generatedj; + secp256k1_ge generated; + + secp256k1_scalar_set_int(&scalar_i, i); + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &generatedj, &scalar_i); + secp256k1_ge_set_gej(&generated, &generatedj); + + CHECK(group[i].infinity == 0); + CHECK(generated.infinity == 0); + CHECK(secp256k1_fe_equal_var(&generated.x, &group[i].x)); + CHECK(secp256k1_fe_equal_var(&generated.y, &group[i].y)); + } + } + + /* Run the tests */ +#ifdef USE_ENDOMORPHISM + test_exhaustive_endomorphism(group, EXHAUSTIVE_TEST_ORDER); +#endif + test_exhaustive_addition(group, groupj, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_ecmult(ctx, group, groupj, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_sign(ctx, group, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_verify(ctx, group, EXHAUSTIVE_TEST_ORDER); + + return 0; +} + diff --git a/src/serialize.h b/src/serialize.h index e28ca548c0..2be6f005e3 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -59,34 +59,6 @@ inline T* NCONST_PTR(const T* val) return const_cast<T*>(val); } -/** - * Important: Do not use the following functions in new code, but use v.data() - * and v.data() + v.size() respectively directly. They were once introduced to - * have a compatible, safe way to get the begin and end pointer of a vector. - * However with C++11 the language has built-in functionality for this and it's - * more readable to just use that. - */ -template <typename V> -inline typename V::value_type* begin_ptr(V& v) -{ - return v.data(); -} -template <typename V> -inline const typename V::value_type* begin_ptr(const V& v) -{ - return v.data(); -} -template <typename V> -inline typename V::value_type* end_ptr(V& v) -{ - return v.data() + v.size(); -} -template <typename V> -inline const typename V::value_type* end_ptr(const V& v) -{ - return v.data() + v.size(); -} - /* * Lowest-level serialization and conversion. * @note Sizes of these types are verified in the tests @@ -390,14 +362,14 @@ public: template <class T, class TAl> explicit CFlatData(std::vector<T,TAl> &v) { - pbegin = (char*)begin_ptr(v); - pend = (char*)end_ptr(v); + pbegin = (char*)v.data(); + pend = (char*)(v.data() + v.size()); } template <unsigned int N, typename T, typename S, typename D> explicit CFlatData(prevector<N, T, S, D> &v) { - pbegin = (char*)begin_ptr(v); - pend = (char*)end_ptr(v); + pbegin = (char*)v.data(); + pend = (char*)(v.data() + v.size()); } char* begin() { return pbegin; } const char* begin() const { return pbegin; } diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 1a818a5756..131d667e0b 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -29,9 +29,9 @@ extern unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans); struct COrphanTx { CTransaction tx; NodeId fromPeer; + int64_t nTimeExpire; }; extern std::map<uint256, COrphanTx> mapOrphanTransactions; -extern std::map<uint256, std::set<uint256> > mapOrphanTransactionsByPrev; CService ip(uint32_t i) { @@ -202,7 +202,6 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) BOOST_CHECK(mapOrphanTransactions.size() <= 10); LimitOrphanTxSize(0); BOOST_CHECK(mapOrphanTransactions.empty()); - BOOST_CHECK(mapOrphanTransactionsByPrev.empty()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index ac3ab4c83f..04a6506655 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58) std::vector<unsigned char> sourcedata = ParseHex(test[0].get_str()); std::string base58string = test[1].get_str(); BOOST_CHECK_MESSAGE( - EncodeBase58(begin_ptr(sourcedata), end_ptr(sourcedata)) == base58string, + EncodeBase58(sourcedata.data(), sourcedata.data() + sourcedata.size()) == base58string, strTest); } } diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index b013cda6d7..7478758f73 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -80,9 +80,9 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); - std::vector<CTransactionRef> removed; - pool.removeRecursive(*block.vtx[2], &removed); - BOOST_CHECK_EQUAL(removed.size(), 1); + size_t poolSize = pool.size(); + pool.removeRecursive(*block.vtx[2]); + BOOST_CHECK_EQUAL(pool.size(), poolSize - 1); CBlock block2; std::vector<CTransactionRef> vtx_missing; diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index ffa1fb6ad0..b15ff9e44b 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -114,16 +114,14 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) BOOST_AUTO_TEST_CASE(bloom_match) { // Random real transaction (b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b) - CTransaction tx; CDataStream stream(ParseHex("01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e88607d722c190000000008b4830450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a0141046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c339ffffffff021bff3d11000000001976a91404943fdd508053c75000106d3bc6e2754dbcff1988ac2f15de00000000001976a914a266436d2965547608b9e15d9032a7b9d64fa43188ac00000000"), SER_DISK, CLIENT_VERSION); - stream >> tx; + CTransaction tx(deserialize, stream); // and one which spends it (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; vector<unsigned char> vch(ch, ch + sizeof(ch) -1); CDataStream spendStream(vch, SER_DISK, CLIENT_VERSION); - CTransaction spendingTx; - spendStream >> spendingTx; + CTransaction spendingTx(deserialize, spendStream); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); filter.insert(uint256S("0xb4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b")); diff --git a/src/test/data/tx_valid.json b/src/test/data/tx_valid.json index 27acb2f16a..a3f47fcee2 100644 --- a/src/test/data/tx_valid.json +++ b/src/test/data/tx_valid.json @@ -478,7 +478,7 @@ ["1b2a9a426ba603ba357ce7773cb5805cb9c7c2b386d100d1fc9263513188e680", 0, "0x00 0x20 0xd9bbfbe56af7c4b7f960a70d7ea107156913d9e5a26b0a71429df5e097ca6537", 16777215]], "01000000000102e9b542c5176808107ff1df906f46bb1f2583b16112b95ee5380665ba7fcfc0010000000000ffffffff80e68831516392fcd100d186b3c2c7b95c80b53c77e77c35ba03a66b429a2a1b0000000000ffffffff0280969800000000001976a914de4b231626ef508c9a74a8517e6783c0546d6b2888ac80969800000000001976a9146648a8cd4531e1ec47f35916de8e259237294d1e88ac02483045022100f6a10b8604e6dc910194b79ccfc93e1bc0ec7c03453caaa8987f7d6c3413566002206216229ede9b4d6ec2d325be245c5b508ff0339bf1794078e20bfe0babc7ffe683270063ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac024730440220032521802a76ad7bf74d0e2c218b72cf0cbc867066e2e53db905ba37f130397e02207709e2188ed7f08f4c952d9d13986da504502b8c3be59617e043552f506c46ff83275163ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac00000000", "P2SH,WITNESS"], -["BIP143 example: Same as the previous example with input-output paris swapped"], +["BIP143 example: Same as the previous example with input-output pairs swapped"], [[["1b2a9a426ba603ba357ce7773cb5805cb9c7c2b386d100d1fc9263513188e680", 0, "0x00 0x20 0xd9bbfbe56af7c4b7f960a70d7ea107156913d9e5a26b0a71429df5e097ca6537", 16777215], ["01c0cf7fba650638e55eb91261b183251fbb466f90dff17f10086817c542b5e9", 0, "0x00 0x20 0xba468eea561b26301e4cf69fa34bde4ad60c81e70f059f045ca9a79931004a4d", 16777215]], "0100000000010280e68831516392fcd100d186b3c2c7b95c80b53c77e77c35ba03a66b429a2a1b0000000000ffffffffe9b542c5176808107ff1df906f46bb1f2583b16112b95ee5380665ba7fcfc0010000000000ffffffff0280969800000000001976a9146648a8cd4531e1ec47f35916de8e259237294d1e88ac80969800000000001976a914de4b231626ef508c9a74a8517e6783c0546d6b2888ac024730440220032521802a76ad7bf74d0e2c218b72cf0cbc867066e2e53db905ba37f130397e02207709e2188ed7f08f4c952d9d13986da504502b8c3be59617e043552f506c46ff83275163ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac02483045022100f6a10b8604e6dc910194b79ccfc93e1bc0ec7c03453caaa8987f7d6c3413566002206216229ede9b4d6ec2d325be245c5b508ff0339bf1794078e20bfe0babc7ffe683270063ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac00000000", "P2SH,WITNESS"], diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 1faf8b6aeb..bdd6eaf538 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -55,17 +55,17 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) CTxMemPool testPool(CFeeRate(0)); - std::vector<CTransactionRef> removed; // Nothing in pool, remove should do nothing: - testPool.removeRecursive(txParent, &removed); - BOOST_CHECK_EQUAL(removed.size(), 0); + unsigned int poolSize = testPool.size(); + testPool.removeRecursive(txParent); + BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Just the parent: testPool.addUnchecked(txParent.GetHash(), entry.FromTx(txParent)); - testPool.removeRecursive(txParent, &removed); - BOOST_CHECK_EQUAL(removed.size(), 1); - removed.clear(); + poolSize = testPool.size(); + testPool.removeRecursive(txParent); + BOOST_CHECK_EQUAL(testPool.size(), poolSize - 1); // Parent, children, grandchildren: testPool.addUnchecked(txParent.GetHash(), entry.FromTx(txParent)); @@ -75,19 +75,21 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) testPool.addUnchecked(txGrandChild[i].GetHash(), entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: - testPool.removeRecursive(txChild[0], &removed); - BOOST_CHECK_EQUAL(removed.size(), 2); - removed.clear(); + poolSize = testPool.size(); + testPool.removeRecursive(txChild[0]); + BOOST_CHECK_EQUAL(testPool.size(), poolSize - 2); // ... make sure grandchild and child are gone: - testPool.removeRecursive(txGrandChild[0], &removed); - BOOST_CHECK_EQUAL(removed.size(), 0); - testPool.removeRecursive(txChild[0], &removed); - BOOST_CHECK_EQUAL(removed.size(), 0); + poolSize = testPool.size(); + testPool.removeRecursive(txGrandChild[0]); + BOOST_CHECK_EQUAL(testPool.size(), poolSize); + poolSize = testPool.size(); + testPool.removeRecursive(txChild[0]); + BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Remove parent, all children/grandchildren should go: - testPool.removeRecursive(txParent, &removed); - BOOST_CHECK_EQUAL(removed.size(), 5); + poolSize = testPool.size(); + testPool.removeRecursive(txParent); + BOOST_CHECK_EQUAL(testPool.size(), poolSize - 5); BOOST_CHECK_EQUAL(testPool.size(), 0); - removed.clear(); // Add children and grandchildren, but NOT the parent (simulate the parent being in a block) for (int i = 0; i < 3; i++) @@ -97,10 +99,10 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) } // Now remove the parent, as might happen if a block-re-org occurs but the parent cannot be // put into the mempool (maybe because it is non-standard): - testPool.removeRecursive(txParent, &removed); - BOOST_CHECK_EQUAL(removed.size(), 6); + poolSize = testPool.size(); + testPool.removeRecursive(txParent); + BOOST_CHECK_EQUAL(testPool.size(), poolSize - 6); BOOST_CHECK_EQUAL(testPool.size(), 0); - removed.clear(); } template<typename name> @@ -412,7 +414,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) /* after tx6 is mined, tx7 should move up in the sort */ std::vector<CTransactionRef> vtx; vtx.push_back(MakeTransactionRef(tx6)); - pool.removeForBlock(vtx, 1, NULL, false); + pool.removeForBlock(vtx, 1); sortedOrder.erase(sortedOrder.begin()+1); sortedOrder.pop_back(); diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 3169afb13a..d7f3a3a657 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -176,10 +176,10 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript int libconsensus_flags = flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL; if (libconsensus_flags == flags) { if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); } else { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(begin_ptr(scriptPubKey), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect,message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect,message); } } #endif @@ -276,7 +276,7 @@ private: //! The Witness embedded script CScript witscript; CScriptWitness scriptWitness; - CTransaction creditTx; + CTransactionRef creditTx; CMutableTransaction spendTx; bool havePush; std::vector<unsigned char> push; @@ -319,8 +319,8 @@ public: redeemscript = scriptPubKey; scriptPubKey = CScript() << OP_HASH160 << ToByteVector(CScriptID(redeemscript)) << OP_EQUAL; } - creditTx = BuildCreditingTransaction(scriptPubKey, nValue); - spendTx = BuildSpendingTransaction(CScript(), CScriptWitness(), creditTx); + creditTx = MakeTransactionRef(BuildCreditingTransaction(scriptPubKey, nValue)); + spendTx = BuildSpendingTransaction(CScript(), CScriptWitness(), *creditTx); } TestBuilder& ScriptError(ScriptError_t err) @@ -421,7 +421,7 @@ public: { TestBuilder copy = *this; // Make a copy so we can rollback the push. DoPush(); - DoTest(creditTx.vout[0].scriptPubKey, spendTx.vin[0].scriptSig, scriptWitness, flags, comment, scriptError, nValue); + DoTest(creditTx->vout[0].scriptPubKey, spendTx.vin[0].scriptSig, scriptWitness, flags, comment, scriptError, nValue); *this = copy; return *this; } @@ -447,7 +447,7 @@ public: array.push_back(wit); } array.push_back(FormatScript(spendTx.vin[0].scriptSig)); - array.push_back(FormatScript(creditTx.vout[0].scriptPubKey)); + array.push_back(FormatScript(creditTx->vout[0].scriptPubKey)); array.push_back(FormatScriptFlags(flags)); array.push_back(FormatScriptError((ScriptError_t)scriptError)); array.push_back(comment); @@ -461,7 +461,7 @@ public: const CScript& GetScriptPubKey() { - return creditTx.vout[0].scriptPubKey; + return creditTx->vout[0].scriptPubKey; } }; diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index bbadf57957..1dc86eb116 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -21,10 +21,10 @@ protected: bool boolval; std::string stringval; const char* charstrval; - CTransaction txval; + CTransactionRef txval; public: CSerializeMethodsTestSingle() = default; - CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, CTransaction txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), charstrval(charstrvalin), txval(txvalin){} + CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, CTransaction txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), charstrval(charstrvalin), txval(MakeTransactionRef(txvalin)){} ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> @@ -42,7 +42,7 @@ public: boolval == rhs.boolval && \ stringval == rhs.stringval && \ strcmp(charstrval, rhs.charstrval) == 0 && \ - txval == rhs.txval; + *txval == *rhs.txval; } }; diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 38d58e7ab0..a524f5b946 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -184,7 +184,7 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) std::string raw_tx, raw_script, sigHashHex; int nIn, nHashType; uint256 sh; - CTransaction tx; + CTransactionRef tx; CScript scriptCode = CScript(); try { @@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) stream >> tx; CValidationState state; - BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); + BOOST_CHECK_MESSAGE(CheckTransaction(*tx, state), strTest); BOOST_CHECK(state.IsValid()); std::vector<unsigned char> raw = ParseHex(raw_script); @@ -209,7 +209,7 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) continue; } - sh = SignatureHash(scriptCode, tx, nIn, nHashType, 0, SIGVERSION_BASE); + sh = SignatureHash(scriptCode, *tx, nIn, nHashType, 0, SIGVERSION_BASE); BOOST_CHECK_MESSAGE(sh.GetHex() == sigHashHex, strTest); } } diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 7b4763669a..ae5406ef7e 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -150,8 +150,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) string transaction = test[1].get_str(); CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION); - CTransaction tx; - stream >> tx; + CTransaction tx(deserialize, stream); CValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); @@ -236,8 +235,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) string transaction = test[1].get_str(); CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION ); - CTransaction tx; - stream >> tx; + CTransaction tx(deserialize, stream); CValidationState state; fValid = CheckTransaction(tx, state) && state.IsValid(); @@ -346,7 +344,7 @@ BOOST_AUTO_TEST_CASE(test_Get) BOOST_CHECK_EQUAL(coins.GetValueIn(t1), (50+21+22)*CENT); } -void CreateCreditAndSpend(const CKeyStore& keystore, const CScript& outscript, CTransaction& output, CMutableTransaction& input, bool success = true) +void CreateCreditAndSpend(const CKeyStore& keystore, const CScript& outscript, CTransactionRef& output, CMutableTransaction& input, bool success = true) { CMutableTransaction outputm; outputm.nVersion = 1; @@ -360,22 +358,22 @@ void CreateCreditAndSpend(const CKeyStore& keystore, const CScript& outscript, C CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); ssout << outputm; ssout >> output; - assert(output.vin.size() == 1); - assert(output.vin[0] == outputm.vin[0]); - assert(output.vout.size() == 1); - assert(output.vout[0] == outputm.vout[0]); - assert(output.wit.vtxinwit.size() == 0); + assert(output->vin.size() == 1); + assert(output->vin[0] == outputm.vin[0]); + assert(output->vout.size() == 1); + assert(output->vout[0] == outputm.vout[0]); + assert(output->wit.vtxinwit.size() == 0); CMutableTransaction inputm; inputm.nVersion = 1; inputm.vin.resize(1); - inputm.vin[0].prevout.hash = output.GetHash(); + inputm.vin[0].prevout.hash = output->GetHash(); inputm.vin[0].prevout.n = 0; inputm.wit.vtxinwit.resize(1); inputm.vout.resize(1); inputm.vout[0].nValue = 1; inputm.vout[0].scriptPubKey = CScript(); - bool ret = SignSignature(keystore, output, inputm, 0, SIGHASH_ALL); + bool ret = SignSignature(keystore, *output, inputm, 0, SIGHASH_ALL); assert(ret == success); CDataStream ssin(SER_NETWORK, PROTOCOL_VERSION); ssin << inputm; @@ -393,11 +391,11 @@ void CreateCreditAndSpend(const CKeyStore& keystore, const CScript& outscript, C } } -void CheckWithFlag(const CTransaction& output, const CMutableTransaction& input, int flags, bool success) +void CheckWithFlag(const CTransactionRef& output, const CMutableTransaction& input, int flags, bool success) { ScriptError error; CTransaction inputi(input); - bool ret = VerifyScript(inputi.vin[0].scriptSig, output.vout[0].scriptPubKey, inputi.wit.vtxinwit.size() > 0 ? &inputi.wit.vtxinwit[0].scriptWitness : NULL, flags, TransactionSignatureChecker(&inputi, 0, output.vout[0].nValue), &error); + bool ret = VerifyScript(inputi.vin[0].scriptSig, output->vout[0].scriptPubKey, inputi.wit.vtxinwit.size() > 0 ? &inputi.wit.vtxinwit[0].scriptWitness : NULL, flags, TransactionSignatureChecker(&inputi, 0, output->vout[0].nValue), &error); assert(ret == success); } @@ -466,10 +464,10 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { assert(hashSigned); } - CTransaction tx; CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); - WithOrVersion(&ssout, 0) << mtx; - WithOrVersion(&ssout, 0) >> tx; + auto vstream = WithOrVersion(&ssout, 0); + vstream << mtx; + CTransaction tx(deserialize, vstream); // check all inputs concurrently, with the cache PrecomputedTransactionData txdata(tx); @@ -547,7 +545,7 @@ BOOST_AUTO_TEST_CASE(test_witness) keystore2.AddCScript(GetScriptForWitness(scriptMulti)); keystore2.AddKeyPubKey(key3, pubkey3); - CTransaction output1, output2; + CTransactionRef output1, output2; CMutableTransaction input1, input2; SignatureData sigdata; @@ -639,8 +637,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CheckWithFlag(output1, input1, 0, false); CreateCreditAndSpend(keystore2, scriptMulti, output2, input2, false); CheckWithFlag(output2, input2, 0, false); - BOOST_CHECK(output1 == output2); - UpdateTransaction(input1, 0, CombineSignatures(output1.vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1.vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); + BOOST_CHECK(*output1 == *output2); + UpdateTransaction(input1, 0, CombineSignatures(output1->vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1->vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); // P2SH 2-of-2 multisig @@ -650,8 +648,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore2, GetScriptForDestination(CScriptID(scriptMulti)), output2, input2, false); CheckWithFlag(output2, input2, 0, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH, false); - BOOST_CHECK(output1 == output2); - UpdateTransaction(input1, 0, CombineSignatures(output1.vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1.vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); + BOOST_CHECK(*output1 == *output2); + UpdateTransaction(input1, 0, CombineSignatures(output1->vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1->vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); @@ -662,8 +660,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore2, GetScriptForWitness(scriptMulti), output2, input2, false); CheckWithFlag(output2, input2, 0, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); - BOOST_CHECK(output1 == output2); - UpdateTransaction(input1, 0, CombineSignatures(output1.vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1.vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); + BOOST_CHECK(*output1 == *output2); + UpdateTransaction(input1, 0, CombineSignatures(output1->vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1->vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); @@ -674,8 +672,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CreateCreditAndSpend(keystore2, GetScriptForDestination(CScriptID(GetScriptForWitness(scriptMulti))), output2, input2, false); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); - BOOST_CHECK(output1 == output2); - UpdateTransaction(input1, 0, CombineSignatures(output1.vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1.vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); + BOOST_CHECK(*output1 == *output2); + UpdateTransaction(input1, 0, CombineSignatures(output1->vout[0].scriptPubKey, MutableTransactionSignatureChecker(&input1, 0, output1->vout[0].nValue), DataFromTransaction(input1, 0), DataFromTransaction(input2, 0))); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true); CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index ffb9993f90..67a69363f0 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -501,10 +501,10 @@ static std::vector<uint8_t> ComputeResponse(const std::string &key, const std::v { CHMAC_SHA256 computeHash((const uint8_t*)key.data(), key.size()); std::vector<uint8_t> computedHash(CHMAC_SHA256::OUTPUT_SIZE, 0); - computeHash.Write(begin_ptr(cookie), cookie.size()); - computeHash.Write(begin_ptr(clientNonce), clientNonce.size()); - computeHash.Write(begin_ptr(serverNonce), serverNonce.size()); - computeHash.Finalize(begin_ptr(computedHash)); + computeHash.Write(cookie.data(), cookie.size()); + computeHash.Write(clientNonce.data(), clientNonce.size()); + computeHash.Write(serverNonce.data(), serverNonce.size()); + computeHash.Finalize(computedHash.data()); return computedHash; } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c035a84db5..4334ebde6d 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -503,7 +503,7 @@ void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants } } -void CTxMemPool::removeRecursive(const CTransaction &origTx, std::vector<CTransactionRef>* removed) +void CTxMemPool::removeRecursive(const CTransaction &origTx) { // Remove transaction from memory pool { @@ -530,11 +530,6 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, std::vector<CTransa BOOST_FOREACH(txiter it, txToRemove) { CalculateDescendants(it, setAllRemoves); } - if (removed) { - BOOST_FOREACH(txiter it, setAllRemoves) { - removed->emplace_back(it->GetSharedTx()); - } - } RemoveStaged(setAllRemoves, false); } } @@ -576,7 +571,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem RemoveStaged(setAllRemoves, false); } -void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<CTransactionRef>* removed) +void CTxMemPool::removeConflicts(const CTransaction &tx) { // Remove transactions which depend on inputs of tx, recursively LOCK(cs); @@ -586,7 +581,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<CTransactio const CTransaction &txConflict = *it->second; if (txConflict != tx) { - removeRecursive(txConflict, removed); + removeRecursive(txConflict); ClearPrioritisation(txConflict.GetHash()); } } @@ -597,7 +592,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<CTransactio * Called when a block is connected. Removes from mempool and updates the miner fee estimator. */ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight, - std::vector<CTransactionRef>* conflicts, bool fCurrentEstimate) + bool fCurrentEstimate) { LOCK(cs); std::vector<CTxMemPoolEntry> entries; @@ -617,7 +612,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne stage.insert(it); RemoveStaged(stage, true); } - removeConflicts(*tx, conflicts); + removeConflicts(*tx); ClearPrioritisation(tx->GetHash()); } // After the txs in the new block have been removed from the mempool, update policy estimates diff --git a/src/txmempool.h b/src/txmempool.h index 23fe5a7abe..8a935391de 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -527,11 +527,11 @@ public: bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); - void removeRecursive(const CTransaction &tx, std::vector<CTransactionRef>* removed = NULL); + void removeRecursive(const CTransaction &tx); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); - void removeConflicts(const CTransaction &tx, std::vector<CTransactionRef>* removed = NULL); + void removeConflicts(const CTransaction &tx); void removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight, - std::vector<CTransactionRef>* conflicts = NULL, bool fCurrentEstimate = true); + bool fCurrentEstimate = true); void clear(); void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); diff --git a/src/util.cpp b/src/util.cpp index 014013d214..60701e7949 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -704,13 +704,13 @@ void ShrinkDebugFile() // Restart the file with some of the end std::vector <char> vch(200000,0); fseek(file, -((long)vch.size()), SEEK_END); - int nBytes = fread(begin_ptr(vch), 1, vch.size(), file); + int nBytes = fread(vch.data(), 1, vch.size(), file); fclose(file); file = fopen(pathLog.string().c_str(), "w"); if (file) { - fwrite(begin_ptr(vch), 1, nBytes, file); + fwrite(vch.data(), 1, nBytes, file); fclose(file); } } diff --git a/src/validation.cpp b/src/validation.cpp index 46a8803c0c..02c047f1d4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -540,14 +540,6 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C if (tx.IsCoinBase()) return state.DoS(100, false, REJECT_INVALID, "coinbase"); - // Don't relay version 2 transactions until CSV is active, and we can be - // sure that such transactions will be mined (unless we're on - // -testnet/-regtest). - const CChainParams& chainparams = Params(); - if (fRequireStandard && tx.nVersion >= 2 && VersionBitsTipState(chainparams.GetConsensus(), Consensus::DEPLOYMENT_CSV) != THRESHOLD_ACTIVE) { - return state.DoS(0, false, REJECT_NONSTANDARD, "premature-version2-tx"); - } - // Reject transactions with witness before segregated witness activates (override with -prematurewitness) bool witnessEnabled = IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus()); if (!GetBoolArg("-prematurewitness",false) && !tx.wit.IsNull() && !witnessEnabled) { @@ -984,7 +976,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa } /** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */ -bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) +bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { CBlockIndex *pindexSlow = NULL; @@ -993,7 +985,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P CTransactionRef ptx = mempool.get(hash); if (ptx) { - txOut = *ptx; + txOut = ptx; return true; } @@ -1012,7 +1004,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } hashBlock = header.GetHash(); - if (txOut.GetHash() != hash) + if (txOut->GetHash() != hash) return error("%s: txid mismatch", __func__); return true; } @@ -1035,7 +1027,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P if (ReadBlockFromDisk(block, pindexSlow, consensusParams)) { for (const auto& tx : block.vtx) { if (tx->GetHash() == hash) { - txOut = *tx; + txOut = tx; hashBlock = pindexSlow->GetBlockHash(); return true; } @@ -2108,7 +2100,8 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara CCoinsViewCache view(pcoinsTip); if (!DisconnectBlock(block, state, pindexDelete, view)) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); - assert(view.Flush()); + bool flushed = view.Flush(); + assert(flushed); } LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. @@ -2153,11 +2146,10 @@ static int64_t nTimeChainState = 0; static int64_t nTimePostConnect = 0; /** - * Used to track conflicted transactions removed from mempool and transactions - * applied to the UTXO state as a part of a single ActivateBestChainStep call. + * Used to track blocks whose transactions were applied to the UTXO state as a + * part of a single ActivateBestChainStep call. */ struct ConnectTrace { - std::vector<CTransactionRef> txConflicted; std::vector<std::pair<CBlockIndex*, std::shared_ptr<const CBlock> > > blocksConnected; }; @@ -2198,7 +2190,8 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; LogPrint("bench", " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); - assert(view.Flush()); + bool flushed = view.Flush(); + assert(flushed); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint("bench", " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); @@ -2208,7 +2201,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; - mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight, &connectTrace.txConflicted, !IsInitialBlockDownload()); + mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight, !IsInitialBlockDownload()); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); @@ -2433,11 +2426,6 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, // throw all transactions though the signal-interface // while _not_ holding the cs_main lock - for (const auto& tx : connectTrace.txConflicted) - { - GetMainSignals().SyncTransaction(*tx, pindexNewTip, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } - // ... and about transactions that got confirmed: for (const auto& pair : connectTrace.blocksConnected) { assert(pair.second); const CBlock& block = *(pair.second); @@ -2875,8 +2863,9 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc out.scriptPubKey[5] = 0xed; memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32); commitment = std::vector<unsigned char>(out.scriptPubKey.begin(), out.scriptPubKey.end()); - const_cast<std::vector<CTxOut>*>(&block.vtx[0]->vout)->push_back(out); - block.vtx[0]->UpdateHash(); + CMutableTransaction tx(*block.vtx[0]); + tx.vout.push_back(out); + block.vtx[0] = MakeTransactionRef(std::move(tx)); } } UpdateUncommittedBlockStructures(block, pindexPrev, consensusParams); @@ -4106,10 +4095,9 @@ bool LoadMempool(void) file >> num; double prioritydummy = 0; while (num--) { - CTransaction tx; int64_t nTime; int64_t nFeeDelta; - file >> tx; + CTransaction tx(deserialize, file); file >> nTime; file >> nFeeDelta; diff --git a/src/validation.h b/src/validation.h index c284c2fc18..e50a82b9dd 100644 --- a/src/validation.h +++ b/src/validation.h @@ -275,7 +275,7 @@ bool IsInitialBlockDownload(); */ std::string GetWarnings(const std::string& strFor); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ -bool GetTransaction(const uint256 &hash, CTransaction &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); +bool GetTransaction(const uint256 &hash, CTransactionRef &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); /** Find the best known block, and make it the tip of the block chain */ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 671b391c78..008a4ece19 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -267,11 +267,11 @@ UniValue importprunedfunds(const JSONRPCRequest& request) "2. \"txoutproof\" (string, required) The hex output from gettxoutproof that contains the transaction\n" ); - CTransaction tx; + CMutableTransaction tx; if (!DecodeHexTx(tx, request.params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); uint256 hashTx = tx.GetHash(); - CWalletTx wtx(pwalletMain,tx); + CWalletTx wtx(pwalletMain, MakeTransactionRef(std::move(tx))); CDataStream ssMB(ParseHexV(request.params[1], "proof"), SER_NETWORK, PROTOCOL_VERSION); CMerkleBlock merkleBlock; @@ -304,7 +304,7 @@ UniValue importprunedfunds(const JSONRPCRequest& request) LOCK2(cs_main, pwalletMain->cs_wallet); - if (pwalletMain->IsMine(tx)) { + if (pwalletMain->IsMine(wtx)) { pwalletMain->AddToWallet(wtx, false); return NullUniValue; } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index d0c13c41d1..5a4fcc743c 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -362,7 +362,7 @@ static void SendMoney(const CTxDestination &address, CAmount nValue, bool fSubtr CRecipient recipient = {scriptPubKey, nValue, fSubtractFeeFromAmount}; vecSend.push_back(recipient); if (!pwalletMain->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError)) { - if (!fSubtractFeeFromAmount && nValue + nFeeRequired > pwalletMain->GetBalance()) + if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); throw JSONRPCError(RPC_WALLET_ERROR, strError); } @@ -583,10 +583,10 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) { const CWalletTx& wtx = (*it).second; - if (wtx.IsCoinBase() || !CheckFinalTx(wtx)) + if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) if (txout.scriptPubKey == scriptPubKey) if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; @@ -637,10 +637,10 @@ UniValue getreceivedbyaccount(const JSONRPCRequest& request) for (map<uint256, CWalletTx>::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) { const CWalletTx& wtx = (*it).second; - if (wtx.IsCoinBase() || !CheckFinalTx(wtx)) + if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { CTxDestination address; if (ExtractDestination(txout.scriptPubKey, address) && IsMine(*pwalletMain, address) && setAddress.count(address)) @@ -1149,14 +1149,14 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) { const CWalletTx& wtx = (*it).second; - if (wtx.IsCoinBase() || !CheckFinalTx(wtx)) + if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; int nDepth = wtx.GetDepthInMainChain(); if (nDepth < nMinDepth) continue; - BOOST_FOREACH(const CTxOut& txout, wtx.vout) + BOOST_FOREACH(const CTxOut& txout, wtx.tx->vout) { CTxDestination address; if (!ExtractDestination(txout.scriptPubKey, address)) @@ -1780,7 +1780,7 @@ UniValue gettransaction(const JSONRPCRequest& request) CAmount nCredit = wtx.GetCredit(filter); CAmount nDebit = wtx.GetDebit(filter); CAmount nNet = nCredit - nDebit; - CAmount nFee = (wtx.IsFromMe(filter) ? wtx.GetValueOut() - nDebit : 0); + CAmount nFee = (wtx.IsFromMe(filter) ? wtx.tx->GetValueOut() - nDebit : 0); entry.push_back(Pair("amount", ValueFromAmount(nNet - nFee))); if (wtx.IsFromMe(filter)) @@ -1792,7 +1792,7 @@ UniValue gettransaction(const JSONRPCRequest& request) ListTransactions(wtx, "*", 0, false, details, filter); entry.push_back(Pair("details", details)); - string strHex = EncodeHexTx(static_cast<CTransaction>(wtx)); + string strHex = EncodeHexTx(static_cast<CTransaction>(wtx), RPCSerializationFlags()); entry.push_back(Pair("hex", strHex)); return entry; @@ -2420,7 +2420,7 @@ UniValue listunspent(const JSONRPCRequest& request) continue; CTxDestination address; - const CScript& scriptPubKey = out.tx->vout[out.i].scriptPubKey; + const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); if (setAddress.size() && (!fValidAddress || !setAddress.count(address))) @@ -2445,7 +2445,7 @@ UniValue listunspent(const JSONRPCRequest& request) } entry.push_back(Pair("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end()))); - entry.push_back(Pair("amount", ValueFromAmount(out.tx->vout[out.i].nValue))); + entry.push_back(Pair("amount", ValueFromAmount(out.tx->tx->vout[out.i].nValue))); entry.push_back(Pair("confirmations", out.nDepth)); entry.push_back(Pair("spendable", out.fSpendable)); entry.push_back(Pair("solvable", out.fSolvable)); @@ -2557,17 +2557,16 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) } // parse hex string from parameter - CTransaction origTx; - if (!DecodeHexTx(origTx, request.params[0].get_str(), true)) + CMutableTransaction tx; + if (!DecodeHexTx(tx, request.params[0].get_str(), true)) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); - if (origTx.vout.size() == 0) + if (tx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); - if (changePosition != -1 && (changePosition < 0 || (unsigned int)changePosition > origTx.vout.size())) + if (changePosition != -1 && (changePosition < 0 || (unsigned int)changePosition > tx.vout.size())) throw JSONRPCError(RPC_INVALID_PARAMETER, "changePosition out of bounds"); - CMutableTransaction tx(origTx); CAmount nFeeOut; string strFailReason; diff --git a/src/wallet/test/accounting_tests.cpp b/src/wallet/test/accounting_tests.cpp index a833be13d0..eb14d176bd 100644 --- a/src/wallet/test/accounting_tests.cpp +++ b/src/wallet/test/accounting_tests.cpp @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) { CMutableTransaction tx(wtx); --tx.nLockTime; // Just to change the hash :) - *static_cast<CTransaction*>(&wtx) = CTransaction(tx); + wtx.SetTx(MakeTransactionRef(std::move(tx))); } pwalletMain->AddToWallet(wtx); vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]); @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) { CMutableTransaction tx(wtx); --tx.nLockTime; // Just to change the hash :) - *static_cast<CTransaction*>(&wtx) = CTransaction(tx); + wtx.SetTx(MakeTransactionRef(std::move(tx))); } pwalletMain->AddToWallet(wtx); vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]); diff --git a/src/wallet/test/crypto_tests.cpp b/src/wallet/test/crypto_tests.cpp index ce35c53c48..c64c76244e 100644 --- a/src/wallet/test/crypto_tests.cpp +++ b/src/wallet/test/crypto_tests.cpp @@ -42,15 +42,19 @@ bool OldEncrypt(const CKeyingMaterial& vchPlaintext, std::vector<unsigned char> int nCLen = nLen + AES_BLOCK_SIZE, nFLen = 0; vchCiphertext = std::vector<unsigned char> (nCLen); - EVP_CIPHER_CTX ctx; + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + + if (!ctx) return false; bool fOk = true; - EVP_CIPHER_CTX_init(&ctx); - if (fOk) fOk = EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; - if (fOk) fOk = EVP_EncryptUpdate(&ctx, &vchCiphertext[0], &nCLen, &vchPlaintext[0], nLen) != 0; - if (fOk) fOk = EVP_EncryptFinal_ex(&ctx, (&vchCiphertext[0]) + nCLen, &nFLen) != 0; - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_init(ctx); + if (fOk) fOk = EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; + if (fOk) fOk = EVP_EncryptUpdate(ctx, &vchCiphertext[0], &nCLen, &vchPlaintext[0], nLen) != 0; + if (fOk) fOk = EVP_EncryptFinal_ex(ctx, (&vchCiphertext[0]) + nCLen, &nFLen) != 0; + EVP_CIPHER_CTX_cleanup(ctx); + + EVP_CIPHER_CTX_free(ctx); if (!fOk) return false; @@ -66,15 +70,19 @@ bool OldDecrypt(const std::vector<unsigned char>& vchCiphertext, CKeyingMaterial vchPlaintext = CKeyingMaterial(nPLen); - EVP_CIPHER_CTX ctx; + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + + if (!ctx) return false; bool fOk = true; - EVP_CIPHER_CTX_init(&ctx); - if (fOk) fOk = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; - if (fOk) fOk = EVP_DecryptUpdate(&ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen) != 0; - if (fOk) fOk = EVP_DecryptFinal_ex(&ctx, (&vchPlaintext[0]) + nPLen, &nFLen) != 0; - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_init(ctx); + if (fOk) fOk = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; + if (fOk) fOk = EVP_DecryptUpdate(ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen) != 0; + if (fOk) fOk = EVP_DecryptFinal_ex(ctx, (&vchPlaintext[0]) + nPLen, &nFLen) != 0; + EVP_CIPHER_CTX_cleanup(ctx); + + EVP_CIPHER_CTX_free(ctx); if (!fOk) return false; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index ecbbcb145d..8085706a99 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -42,7 +42,7 @@ static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = fa // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() tx.vin.resize(1); } - CWalletTx* wtx = new CWalletTx(&wallet, tx); + CWalletTx* wtx = new CWalletTx(&wallet, MakeTransactionRef(std::move(tx))); if (fIsFromMe) { wtx->fDebitCached = true; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 102c191d6a..4c6916c04a 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -75,7 +75,7 @@ struct CompareValueOnly std::string COutput::ToString() const { - return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->vout[i].nValue)); + return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const @@ -400,7 +400,7 @@ set<uint256> CWallet::GetConflicts(const uint256& txid) const std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range; - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapTxSpends.count(txin.prevout) <= 1) continue; // No conflict if zero or one spends @@ -552,7 +552,7 @@ void CWallet::AddToSpends(const uint256& wtxid) if (thisTx.IsCoinBase()) // Coinbases don't spend anything! return; - BOOST_FOREACH(const CTxIn& txin, thisTx.vin) + BOOST_FOREACH(const CTxIn& txin, thisTx.tx->vin) AddToSpends(txin.prevout, wtxid); } @@ -795,7 +795,7 @@ bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bFo for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end() && account.vchPubKey.IsValid(); ++it) - BOOST_FOREACH(const CTxOut& txout, (*it).second.vout) + BOOST_FOREACH(const CTxOut& txout, (*it).second.tx->vout) if (txout.scriptPubKey == scriptPubKey) { bForceNew = true; break; @@ -954,7 +954,7 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) wtx.BindWallet(this); wtxOrdered.insert(make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); AddToSpends(hash); - BOOST_FOREACH(const CTxIn& txin, wtx.vin) { + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapWallet.count(txin.prevout.hash)) { CWalletTx& prevtx = mapWallet[txin.prevout.hash]; if (prevtx.nIndex == -1 && !prevtx.hashUnset()) { @@ -993,7 +993,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex if (fExisted && !fUpdate) return false; if (fExisted || IsMine(tx) || IsFromMe(tx)) { - CWalletTx wtx(this,tx); + CWalletTx wtx(this, MakeTransactionRef(tx)); // Get merkle branch if transaction was found in a block if (posInBlock != -1) @@ -1052,7 +1052,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) } // If a transaction changes 'conflicted' state, that changes the balance // available of the outputs it spends. So force those to be recomputed - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapWallet.count(txin.prevout.hash)) mapWallet[txin.prevout.hash].MarkDirty(); @@ -1113,7 +1113,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } // If a transaction changes 'conflicted' state, that changes the balance // available of the outputs it spends. So force those to be recomputed - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (mapWallet.count(txin.prevout.hash)) mapWallet[txin.prevout.hash].MarkDirty(); @@ -1148,8 +1148,8 @@ isminetype CWallet::IsMine(const CTxIn &txin) const if (mi != mapWallet.end()) { const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.vout.size()) - return IsMine(prev.vout[txin.prevout.n]); + if (txin.prevout.n < prev.tx->vout.size()) + return IsMine(prev.tx->vout[txin.prevout.n]); } } return ISMINE_NO; @@ -1163,9 +1163,9 @@ CAmount CWallet::GetDebit(const CTxIn &txin, const isminefilter& filter) const if (mi != mapWallet.end()) { const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.vout.size()) - if (IsMine(prev.vout[txin.prevout.n]) & filter) - return prev.vout[txin.prevout.n].nValue; + if (txin.prevout.n < prev.tx->vout.size()) + if (IsMine(prev.tx->vout[txin.prevout.n]) & filter) + return prev.tx->vout[txin.prevout.n].nValue; } } return 0; @@ -1380,14 +1380,14 @@ void CWalletTx::GetAmounts(list<COutputEntry>& listReceived, CAmount nDebit = GetDebit(filter); if (nDebit > 0) // debit>0 means we signed/sent this transaction { - CAmount nValueOut = GetValueOut(); + CAmount nValueOut = tx->GetValueOut(); nFee = nDebit - nValueOut; } // Sent/received. - for (unsigned int i = 0; i < vout.size(); ++i) + for (unsigned int i = 0; i < tx->vout.size(); ++i) { - const CTxOut& txout = vout[i]; + const CTxOut& txout = tx->vout[i]; isminetype fIsMine = pwallet->IsMine(txout); // Only need to handle txouts if AT LEAST one of these is true: // 1) they debit from us (sent) @@ -1542,9 +1542,11 @@ void CWallet::ReacceptWalletTransactions() bool CWalletTx::RelayWalletTransaction(CConnman* connman) { assert(pwallet->GetBroadcastTransactions()); - if (!IsCoinBase()) + if (!IsCoinBase() && !isAbandoned() && GetDepthInMainChain() == 0) { - if (GetDepthInMainChain() == 0 && !isAbandoned() && InMempool()) { + CValidationState state; + /* GetDepthInMainChain already catches known conflicts. */ + if (InMempool() || AcceptToMemoryPool(maxTxFee, state)) { LogPrintf("Relaying wtx %s\n", GetHash().ToString()); if (connman) { CInv inv(MSG_TX, GetHash()); @@ -1573,7 +1575,7 @@ set<uint256> CWalletTx::GetConflicts() const CAmount CWalletTx::GetDebit(const isminefilter& filter) const { - if (vin.empty()) + if (tx->vin.empty()) return 0; CAmount debit = 0; @@ -1663,11 +1665,11 @@ CAmount CWalletTx::GetAvailableCredit(bool fUseCache) const CAmount nCredit = 0; uint256 hashTx = GetHash(); - for (unsigned int i = 0; i < vout.size(); i++) + for (unsigned int i = 0; i < tx->vout.size(); i++) { if (!pwallet->IsSpent(hashTx, i)) { - const CTxOut &txout = vout[i]; + const CTxOut &txout = tx->vout[i]; nCredit += pwallet->GetCredit(txout, ISMINE_SPENDABLE); if (!MoneyRange(nCredit)) throw std::runtime_error("CWalletTx::GetAvailableCredit() : value out of range"); @@ -1706,11 +1708,11 @@ CAmount CWalletTx::GetAvailableWatchOnlyCredit(const bool& fUseCache) const return nAvailableWatchCreditCached; CAmount nCredit = 0; - for (unsigned int i = 0; i < vout.size(); i++) + for (unsigned int i = 0; i < tx->vout.size(); i++) { if (!pwallet->IsSpent(GetHash(), i)) { - const CTxOut &txout = vout[i]; + const CTxOut &txout = tx->vout[i]; nCredit += pwallet->GetCredit(txout, ISMINE_WATCH_ONLY); if (!MoneyRange(nCredit)) throw std::runtime_error("CWalletTx::GetAvailableCredit() : value out of range"); @@ -1758,23 +1760,23 @@ bool CWalletTx::IsTrusted() const return false; // Trusted if all inputs are from us and are in the mempool: - BOOST_FOREACH(const CTxIn& txin, vin) + BOOST_FOREACH(const CTxIn& txin, tx->vin) { // Transactions not sent by us: not trusted const CWalletTx* parent = pwallet->GetWalletTx(txin.prevout.hash); if (parent == NULL) return false; - const CTxOut& parentOut = parent->vout[txin.prevout.n]; + const CTxOut& parentOut = parent->tx->vout[txin.prevout.n]; if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) return false; } return true; } -bool CWalletTx::IsEquivalentTo(const CWalletTx& tx) const +bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { - CMutableTransaction tx1 = *this; - CMutableTransaction tx2 = tx; + CMutableTransaction tx1 = *this->tx; + CMutableTransaction tx2 = *_tx.tx; for (unsigned int i = 0; i < tx1.vin.size(); i++) tx1.vin[i].scriptSig = CScript(); for (unsigned int i = 0; i < tx2.vin.size(); i++) tx2.vin[i].scriptSig = CScript(); return CTransaction(tx1) == CTransaction(tx2); @@ -1957,10 +1959,10 @@ void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const if (nDepth == 0 && !pcoin->InMempool()) continue; - for (unsigned int i = 0; i < pcoin->vout.size(); i++) { - isminetype mine = IsMine(pcoin->vout[i]); + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { + isminetype mine = IsMine(pcoin->tx->vout[i]); if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && - !IsLockedCoin((*it).first, i) && (pcoin->vout[i].nValue > 0 || fIncludeZeroValue) && + !IsLockedCoin((*it).first, i) && (pcoin->tx->vout[i].nValue > 0 || fIncludeZeroValue) && (!coinControl || !coinControl->HasSelected() || coinControl->fAllowOtherInputs || coinControl->IsSelected(COutPoint((*it).first, i)))) vCoins.push_back(COutput(pcoin, i, nDepth, ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || @@ -2043,7 +2045,7 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int continue; int i = output.i; - CAmount n = pcoin->vout[i].nValue; + CAmount n = pcoin->tx->vout[i].nValue; pair<CAmount,pair<const CWalletTx*,unsigned int> > coin = make_pair(n,make_pair(pcoin, i)); @@ -2130,7 +2132,7 @@ bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& { if (!out.fSpendable) continue; - nValueRet += out.tx->vout[out.i].nValue; + nValueRet += out.tx->tx->vout[out.i].nValue; setCoinsRet.insert(make_pair(out.tx, out.i)); } return (nValueRet >= nTargetValue); @@ -2150,9 +2152,9 @@ bool CWallet::SelectCoins(const vector<COutput>& vAvailableCoins, const CAmount& { const CWalletTx* pcoin = &it->second; // Clearly invalid input, fail - if (pcoin->vout.size() <= outpoint.n) + if (pcoin->tx->vout.size() <= outpoint.n) return false; - nValueFromPresetInputs += pcoin->vout[outpoint.n].nValue; + nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; setPresetCoins.insert(make_pair(pcoin, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs @@ -2208,10 +2210,10 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool ov return false; if (nChangePosInOut != -1) - tx.vout.insert(tx.vout.begin() + nChangePosInOut, wtx.vout[nChangePosInOut]); + tx.vout.insert(tx.vout.begin() + nChangePosInOut, wtx.tx->vout[nChangePosInOut]); // Add new txins (keeping original txin scriptSig/order) - BOOST_FOREACH(const CTxIn& txin, wtx.vin) + BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) { if (!coinControl.IsSelected(txin.prevout)) { @@ -2351,7 +2353,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt } BOOST_FOREACH(PAIRTYPE(const CWalletTx*, unsigned int) pcoin, setCoins) { - CAmount nCredit = pcoin.first->vout[pcoin.second].nValue; + CAmount nCredit = pcoin.first->tx->vout[pcoin.second].nValue; //The coin age after the next block (depth+1) is used instead of the current, //reflecting an assumption the user would accept a bit more delay for //a chance at a free transaction. @@ -2389,7 +2391,11 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt CPubKey vchPubKey; bool ret; ret = reservekey.GetReservedKey(vchPubKey); - assert(ret); // should never fail, as we just unlocked + if (!ret) + { + strFailReason = _("Keypool ran out, please call keypoolrefill first"); + return false; + } scriptChange = GetScriptForDestination(vchPubKey.GetID()); } @@ -2466,10 +2472,10 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt BOOST_FOREACH(const PAIRTYPE(const CWalletTx*,unsigned int)& coin, setCoins) { bool signSuccess; - const CScript& scriptPubKey = coin.first->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; SignatureData sigdata; if (sign) - signSuccess = ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.first->vout[coin.second].nValue, SIGHASH_ALL), scriptPubKey, sigdata); + signSuccess = ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.first->tx->vout[coin.second].nValue, SIGHASH_ALL), scriptPubKey, sigdata); else signSuccess = ProduceSignature(DummySignatureCreator(this), scriptPubKey, sigdata); @@ -2494,16 +2500,16 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt } // Embed the constructed transaction data in wtxNew. - *static_cast<CTransaction*>(&wtxNew) = CTransaction(txNew); + wtxNew.SetTx(MakeTransactionRef(std::move(txNew))); // Limit size - if (GetTransactionWeight(txNew) >= MAX_STANDARD_TX_WEIGHT) + if (GetTransactionWeight(wtxNew) >= MAX_STANDARD_TX_WEIGHT) { strFailReason = _("Transaction too large"); return false; } - dPriority = wtxNew.ComputePriority(dPriority, nBytes); + dPriority = wtxNew.tx->ComputePriority(dPriority, nBytes); // Allow to override the default confirmation target over the CoinControl instance int currentConfirmationTarget = nTxConfirmTarget; @@ -2555,7 +2561,7 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon { { LOCK2(cs_main, cs_wallet); - LogPrintf("CommitTransaction:\n%s", wtxNew.ToString()); + LogPrintf("CommitTransaction:\n%s", wtxNew.tx->ToString()); { // Take key pair from key pool so it won't be used again reservekey.KeepKey(); @@ -2565,7 +2571,7 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon AddToWallet(wtxNew); // Notify that old coins are spent - BOOST_FOREACH(const CTxIn& txin, wtxNew.vin) + BOOST_FOREACH(const CTxIn& txin, wtxNew.tx->vin) { CWalletTx &coin = mapWallet[txin.prevout.hash]; coin.BindWallet(this); @@ -2580,11 +2586,11 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon { // Broadcast if (!wtxNew.AcceptToMemoryPool(maxTxFee, state)) { - // This must not fail. The transaction has already been signed and recorded. - LogPrintf("CommitTransaction(): Error: Transaction not valid, %s\n", state.GetRejectReason()); - return false; + LogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", state.GetRejectReason()); + // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. + } else { + wtxNew.RelayWalletTransaction(connman); } - wtxNew.RelayWalletTransaction(connman); } } return true; @@ -2939,15 +2945,15 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() if (nDepth < (pcoin->IsFromMe(ISMINE_ALL) ? 0 : 1)) continue; - for (unsigned int i = 0; i < pcoin->vout.size(); i++) + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { CTxDestination addr; - if (!IsMine(pcoin->vout[i])) + if (!IsMine(pcoin->tx->vout[i])) continue; - if(!ExtractDestination(pcoin->vout[i].scriptPubKey, addr)) + if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(walletEntry.first, i) ? 0 : pcoin->vout[i].nValue; + CAmount n = IsSpent(walletEntry.first, i) ? 0 : pcoin->tx->vout[i].nValue; if (!balances.count(addr)) balances[addr] = 0; @@ -2969,16 +2975,16 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() { CWalletTx *pcoin = &walletEntry.second; - if (pcoin->vin.size() > 0) + if (pcoin->tx->vin.size() > 0) { bool any_mine = false; // group all input addresses with each other - BOOST_FOREACH(CTxIn txin, pcoin->vin) + BOOST_FOREACH(CTxIn txin, pcoin->tx->vin) { CTxDestination address; if(!IsMine(txin)) /* If this input isn't mine, ignore it */ continue; - if(!ExtractDestination(mapWallet[txin.prevout.hash].vout[txin.prevout.n].scriptPubKey, address)) + if(!ExtractDestination(mapWallet[txin.prevout.hash].tx->vout[txin.prevout.n].scriptPubKey, address)) continue; grouping.insert(address); any_mine = true; @@ -2987,7 +2993,7 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() // group change with input addresses if (any_mine) { - BOOST_FOREACH(CTxOut txout, pcoin->vout) + BOOST_FOREACH(CTxOut txout, pcoin->tx->vout) if (IsChange(txout)) { CTxDestination txoutAddr; @@ -3004,11 +3010,11 @@ set< set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (unsigned int i = 0; i < pcoin->vout.size(); i++) - if (IsMine(pcoin->vout[i])) + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) + if (IsMine(pcoin->tx->vout[i])) { CTxDestination address; - if(!ExtractDestination(pcoin->vout[i].scriptPubKey, address)) + if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, address)) continue; grouping.insert(address); groupings.insert(grouping); @@ -3275,7 +3281,7 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t> &mapKeyBirth) const { if (blit != mapBlockIndex.end() && chainActive.Contains(blit->second)) { // ... which are already in a block int nHeight = blit->second->nHeight; - BOOST_FOREACH(const CTxOut &txout, wtx.vout) { + BOOST_FOREACH(const CTxOut &txout, wtx.tx->vout) { // iterate over all their outputs CAffectedKeysVisitor(*this, vAffected).Process(txout.scriptPubKey); BOOST_FOREACH(const CKeyID &keyid, vAffected) { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 409d817046..f7103b6a80 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -161,13 +161,14 @@ struct COutputEntry }; /** A transaction with a merkle branch linking it to the block chain. */ -class CMerkleTx : public CTransaction +class CMerkleTx { private: /** Constant used in hashBlock to indicate tx has been abandoned */ static const uint256 ABANDON_HASH; public: + CTransactionRef tx; uint256 hashBlock; /* An nIndex == -1 means that hashBlock (in nonzero) refers to the earliest @@ -179,26 +180,37 @@ public: CMerkleTx() { + SetTx(MakeTransactionRef()); Init(); } - CMerkleTx(const CTransaction& txIn) : CTransaction(txIn) + CMerkleTx(CTransactionRef arg) { + SetTx(std::move(arg)); Init(); } + /** Helper conversion operator to allow passing CMerkleTx where CTransaction is expected. + * TODO: adapt callers and remove this operator. */ + operator const CTransaction&() const { return *tx; } + void Init() { hashBlock = uint256(); nIndex = -1; } + void SetTx(CTransactionRef arg) + { + tx = std::move(arg); + } + ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { std::vector<uint256> vMerkleBranch; // For compatibility with older versions. - READWRITE(*(CTransaction*)this); + READWRITE(tx); READWRITE(hashBlock); READWRITE(vMerkleBranch); READWRITE(nIndex); @@ -221,6 +233,9 @@ public: bool hashUnset() const { return (hashBlock.IsNull() || hashBlock == ABANDON_HASH); } bool isAbandoned() const { return (hashBlock == ABANDON_HASH); } void setAbandoned() { hashBlock = ABANDON_HASH; } + + const uint256& GetHash() const { return tx->GetHash(); } + bool IsCoinBase() const { return tx->IsCoinBase(); } }; /** @@ -267,17 +282,7 @@ public: Init(NULL); } - CWalletTx(const CWallet* pwalletIn) - { - Init(pwalletIn); - } - - CWalletTx(const CWallet* pwalletIn, const CMerkleTx& txIn) : CMerkleTx(txIn) - { - Init(pwalletIn); - } - - CWalletTx(const CWallet* pwalletIn, const CTransaction& txIn) : CMerkleTx(txIn) + CWalletTx(const CWallet* pwalletIn, CTransactionRef arg) : CMerkleTx(std::move(arg)) { Init(pwalletIn); } diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 99d42ab526..a11256dfd5 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -7,6 +7,7 @@ #include "zmqpublishnotifier.h" #include "validation.h" #include "util.h" +#include "rpc/server.h" static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; @@ -166,7 +167,7 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) LogPrint("zmq", "zmq: Publish rawblock %s\n", pindex->GetBlockHash().GetHex()); const Consensus::Params& consensusParams = Params().GetConsensus(); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); { LOCK(cs_main); CBlock block; @@ -186,7 +187,7 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr { uint256 hash = transaction.GetHash(); LogPrint("zmq", "zmq: Publish rawtx %s\n", hash.GetHex()); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ss << transaction; return SendMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } |