Merge #20166: [0.20] Backports

7566af419f3e7060152ae062f5a578f543a16a5a doc: Update data directory path comments (Hennadii Stepanov)
09261de6edd7f9fd876607fef350995052da63ba util: Add StripRedundantLastElementsOfPath function (Hennadii Stepanov)
8ef0dace867dbdc1b1dead98f0d69f64eb886d67 macOS deploy: use the new plistlib API (Jonas Schnelli)
314e79581f05881284ed9dfb661754830ed54833 build: fix mutex detection when building bdb on macOS (fanquake)
1f67a30e8374951997af924293c60eff56ae39ed random: fixes read buffer resizing in RandAddSeedPerfmon (Ethan Heilman)
6113b547f4cce8be732bf45687a74eccf4a3abfc net: Send post-verack handshake messages at most once (MarcoFalke)
bdf15d0d5d12eb33594b90ebb48366ea7bb5c3d9 rpc: Adjust witness-tx deserialize error message (MarcoFalke)
731502a18385276887a57ad8574fdbaef976920d rpc: Properly deserialize txs with witness before signing (MarcoFalke)
ee0082b886264fcddd73c5bb7c41ba8316527b9c Avoid the use of abs64 in timedata (Pieter Wuille)
05bd0c220aa0f4cb6202dd19d3164ba2e7cb015e docs: Correct description for getblockstats's txs field (Nadav Ivgi)

Pull request description:

  Backports the following PRs to the 0.20 branch:
  * https://github.com/bitcoin/bitcoin/pull/19777 - docs: Correct description for getblockstats's txs field
  * https://github.com/bitcoin/bitcoin/pull/19836 - rpc: Properly deserialize txs with witness before signing
  * https://github.com/bitcoin/bitcoin/pull/20080 - Strip any trailing `/` in -datadir and -blocksdir paths
  * https://github.com/bitcoin/bitcoin/pull/20082 - [bugfix] random: fixes read buffer to use min rather than max
  * https://github.com/bitcoin/bitcoin/pull/20141 - Avoid the use of abs64 in timedata
  * https://github.com/bitcoin/bitcoin/pull/20146 - net: Send post-verack handshake messages at most once
  * https://github.com/bitcoin/bitcoin/pull/20195 - build: fix mutex detection when building bdb on macOS
  * https://github.com/bitcoin/bitcoin/pull/20298 - macOS deploy: use the new plistlib API

  Will add additional commits as they become available.

ACKs for top commit:
  MarcoFalke:
    review ACK 7566af419f3e7060152ae062f5a578f543a16a5a 🗡

Tree-SHA512: add6bb978313c12c3e07bc232636ae9d1ab0edd0b816705c5c70eeb1cc04097165fd5e29d60c706886943ceb1f749a422020766b4aa2d23be51e9f839157a4bb

11 files changed, 70 insertions, 31 deletions

diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus
index d8088aa123..524104398b 100755
--- a/contrib/macdeploy/macdeployqtplus
+++ b/contrib/macdeploy/macdeployqtplus
@@ -586,7 +586,8 @@ if len(config.fancy) == 1:
         sys.exit(1)
     
     try:
-        fancy = plistlib.readPlist(p)
+        with open(p, 'rb') as fp:
+            fancy = plistlib.load(fp, fmt=plistlib.FMT_XML)
     except:
         if verbose >= 1:
             sys.stderr.write("Error: Could not parse fancy disk image plist at \"{}\"\n".format(p))
diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk
index b679438c6f..f55ee06ccd 100644
--- a/depends/packages/bdb.mk
+++ b/depends/packages/bdb.mk
@@ -9,6 +9,7 @@ define $(package)_set_vars
 $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking
 $(package)_config_opts_mingw32=--enable-mingw
 $(package)_config_opts_linux=--with-pic
+$(package)_cflags+=-Wno-error=implicit-function-declaration
 $(package)_cxxflags=-std=c++11
 $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE
 endef
diff --git a/src/net_processing.cpp b/src/net_processing.cpp
index 79a0ce7559..c4e02f7336 100644
--- a/src/net_processing.cpp
+++ b/src/net_processing.cpp
@@ -2183,6 +2183,8 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec
 
     if (msg_type == NetMsgType::VERACK)
     {
+        if (pfrom->fSuccessfullyConnected) return true;
+
         pfrom->SetRecvVersion(std::min(pfrom->nVersion.load(), PROTOCOL_VERSION));
 
         if (!pfrom->fInbound) {
diff --git a/src/randomenv.cpp b/src/randomenv.cpp
index 8b3d478529..d04f80d06a 100644
--- a/src/randomenv.cpp
+++ b/src/randomenv.cpp
@@ -67,7 +67,8 @@ void RandAddSeedPerfmon(CSHA512& hasher)
 #ifdef WIN32
     // Seed with the entire set of perfmon data
 
-    // This can take up to 2 seconds, so only do it every 10 minutes
+    // This can take up to 2 seconds, so only do it every 10 minutes.
+    // Initialize last_perfmon to 0 seconds, we don't skip the first call.
     static std::atomic<std::chrono::seconds> last_perfmon{std::chrono::seconds{0}};
     auto last_time = last_perfmon.load();
     auto current_time = GetTime<std::chrono::seconds>();
@@ -83,7 +84,7 @@ void RandAddSeedPerfmon(CSHA512& hasher)
         ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize);
         if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize)
             break;
-        vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially
+        vData.resize(std::min((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially
     }
     RegCloseKey(HKEY_PERFORMANCE_DATA);
     if (ret == ERROR_SUCCESS) {
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 097b1f2241..267425e108 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -1716,7 +1716,7 @@ static UniValue getblockstats(const JSONRPCRequest& request)
                 {RPCResult::Type::NUM, "total_size", "Total size of all non-coinbase transactions"},
                 {RPCResult::Type::NUM, "total_weight", "Total weight of all non-coinbase transactions divided by segwit scale factor (4)"},
                 {RPCResult::Type::NUM, "totalfee", "The fee total"},
-                {RPCResult::Type::NUM, "txs", "The number of transactions (excluding coinbase)"},
+                {RPCResult::Type::NUM, "txs", "The number of transactions (including coinbase)"},
                 {RPCResult::Type::NUM, "utxo_increase", "The increase/decrease in the number of unspent outputs"},
                 {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"},
             }},
diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp
index b9ec3c53d6..d581349bc5 100644
--- a/src/rpc/rawtransaction.cpp
+++ b/src/rpc/rawtransaction.cpp
@@ -642,8 +642,8 @@ static UniValue combinerawtransaction(const JSONRPCRequest& request)
     std::vector<CMutableTransaction> txVariants(txs.size());
 
     for (unsigned int idx = 0; idx < txs.size(); idx++) {
-        if (!DecodeHexTx(txVariants[idx], txs[idx].get_str(), true)) {
-            throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed for tx %d", idx));
+        if (!DecodeHexTx(txVariants[idx], txs[idx].get_str())) {
+            throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed for tx %d. Make sure the tx has at least one input.", idx));
         }
     }
 
@@ -764,8 +764,8 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request)
     RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR, UniValue::VARR, UniValue::VSTR}, true);
 
     CMutableTransaction mtx;
-    if (!DecodeHexTx(mtx, request.params[0].get_str(), true)) {
-        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed");
+    if (!DecodeHexTx(mtx, request.params[0].get_str())) {
+        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input.");
     }
 
     FillableSigningProvider keystore;
@@ -828,10 +828,10 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request)
         UniValueType(), // NUM or BOOL, checked later
     });
 
-    // parse hex string from parameter
     CMutableTransaction mtx;
-    if (!DecodeHexTx(mtx, request.params[0].get_str()))
-        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed");
+    if (!DecodeHexTx(mtx, request.params[0].get_str())) {
+        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input.");
+    }
     CTransactionRef tx(MakeTransactionRef(std::move(mtx)));
 
     CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE;
@@ -905,7 +905,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request)
 
     CMutableTransaction mtx;
     if (!DecodeHexTx(mtx, request.params[0].get_array()[0].get_str())) {
-        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed");
+        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input.");
     }
     CTransactionRef tx(MakeTransactionRef(std::move(mtx)));
     const uint256& tx_hash = tx->GetHash();
diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp
index 73b37f909f..bba145696c 100644
--- a/src/test/util_tests.cpp
+++ b/src/test/util_tests.cpp
@@ -41,6 +41,28 @@ namespace BCLog {
 
 BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup)
 
+BOOST_AUTO_TEST_CASE(util_datadir)
+{
+    ClearDatadirCache();
+    const fs::path dd_norm = GetDataDir();
+
+    gArgs.ForceSetArg("-datadir", dd_norm.string() + "/");
+    ClearDatadirCache();
+    BOOST_CHECK_EQUAL(dd_norm, GetDataDir());
+
+    gArgs.ForceSetArg("-datadir", dd_norm.string() + "/.");
+    ClearDatadirCache();
+    BOOST_CHECK_EQUAL(dd_norm, GetDataDir());
+
+    gArgs.ForceSetArg("-datadir", dd_norm.string() + "/./");
+    ClearDatadirCache();
+    BOOST_CHECK_EQUAL(dd_norm, GetDataDir());
+
+    gArgs.ForceSetArg("-datadir", dd_norm.string() + "/.//");
+    ClearDatadirCache();
+    BOOST_CHECK_EQUAL(dd_norm, GetDataDir());
+}
+
 BOOST_AUTO_TEST_CASE(util_criticalsection)
 {
     RecursiveMutex cs;
diff --git a/src/timedata.cpp b/src/timedata.cpp
index 942b3cb919..2a584e3f02 100644
--- a/src/timedata.cpp
+++ b/src/timedata.cpp
@@ -37,11 +37,6 @@ int64_t GetAdjustedTime()
     return GetTime() + GetTimeOffset();
 }
 
-static int64_t abs64(int64_t n)
-{
-    return (n >= 0 ? n : -n);
-}
-
 #define BITCOIN_TIMEDATA_MAX_SAMPLES 200
 
 void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample)
@@ -81,8 +76,8 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample)
         int64_t nMedian = vTimeOffsets.median();
         std::vector<int64_t> vSorted = vTimeOffsets.sorted();
         // Only let other nodes change our time by so much
-        if (abs64(nMedian) <= std::max<int64_t>(0, gArgs.GetArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT)))
-        {
+        int64_t max_adjustment = std::max<int64_t>(0, gArgs.GetArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT));
+        if (nMedian >= -max_adjustment && nMedian <= max_adjustment) {
             nTimeOffset = nMedian;
         }
         else
@@ -94,9 +89,10 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample)
             {
                 // If nobody has a time different than ours but within 5 minutes of ours, give a warning
                 bool fMatch = false;
-                for (const int64_t nOffset : vSorted)
-                    if (nOffset != 0 && abs64(nOffset) < 5 * 60)
+                for (const int64_t nOffset : vSorted) {
+                    if (nOffset != 0 && nOffset > -5 * 60 && nOffset < 5 * 60)
                         fMatch = true;
+                }
 
                 if (!fMatch)
                 {
diff --git a/src/util/system.cpp b/src/util/system.cpp
index b6a7f3926d..702397ec3b 100644
--- a/src/util/system.cpp
+++ b/src/util/system.cpp
@@ -29,6 +29,7 @@
 #endif // __linux__
 
 #include <algorithm>
+#include <cassert>
 #include <fcntl.h>
 #include <sched.h>
 #include <sys/resource.h>
@@ -555,10 +556,9 @@ void PrintExceptionContinue(const std::exception* pex, const char* pszThread)
 
 fs::path GetDefaultDataDir()
 {
-    // Windows < Vista: C:\Documents and Settings\Username\Application Data\Bitcoin
-    // Windows >= Vista: C:\Users\Username\AppData\Roaming\Bitcoin
-    // Mac: ~/Library/Application Support/Bitcoin
-    // Unix: ~/.bitcoin
+    // Windows: C:\Users\Username\AppData\Roaming\Bitcoin
+    // macOS: ~/Library/Application Support/Bitcoin
+    // Unix-like: ~/.bitcoin
 #ifdef WIN32
     // Windows
     return GetSpecialFolderPath(CSIDL_APPDATA) / "Bitcoin";
@@ -570,15 +570,28 @@ fs::path GetDefaultDataDir()
     else
         pathRet = fs::path(pszHome);
 #ifdef MAC_OSX
-    // Mac
+    // macOS
     return pathRet / "Library/Application Support/Bitcoin";
 #else
-    // Unix
+    // Unix-like
     return pathRet / ".bitcoin";
 #endif
 #endif
 }
 
+namespace {
+fs::path StripRedundantLastElementsOfPath(const fs::path& path)
+{
+    auto result = path;
+    while (result.filename().string() == ".") {
+        result = result.parent_path();
+    }
+
+    assert(fs::equivalent(result, path));
+    return result;
+}
+} // namespace
+
 static fs::path g_blocks_path_cache_net_specific;
 static fs::path pathCached;
 static fs::path pathCachedNetSpecific;
@@ -606,6 +619,7 @@ const fs::path &GetBlocksDir()
     path /= BaseParams().DataDir();
     path /= "blocks";
     fs::create_directories(path);
+    path = StripRedundantLastElementsOfPath(path);
     return path;
 }
 
@@ -636,6 +650,7 @@ const fs::path &GetDataDir(bool fNetSpecific)
         fs::create_directories(path / "wallets");
     }
 
+    path = StripRedundantLastElementsOfPath(path);
     return path;
 }
 
diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp
index ea54027c48..e8385eef1d 100644
--- a/src/wallet/rpcdump.cpp
+++ b/src/wallet/rpcdump.cpp
@@ -343,8 +343,9 @@ UniValue importprunedfunds(const JSONRPCRequest& request)
             }.Check(request);
 
     CMutableTransaction tx;
-    if (!DecodeHexTx(tx, request.params[0].get_str()))
-        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed");
+    if (!DecodeHexTx(tx, request.params[0].get_str())) {
+        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input.");
+    }
     uint256 hashTx = tx.GetHash();
     CWalletTx wtx(pwallet, MakeTransactionRef(std::move(tx)));
 
diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp
index a5036413c8..337e03624c 100644
--- a/src/wallet/rpcwallet.cpp
+++ b/src/wallet/rpcwallet.cpp
@@ -3309,8 +3309,8 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request)
     RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR, UniValue::VSTR}, true);
 
     CMutableTransaction mtx;
-    if (!DecodeHexTx(mtx, request.params[0].get_str(), true)) {
-        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed");
+    if (!DecodeHexTx(mtx, request.params[0].get_str())) {
+        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input.");
     }
 
     // Sign the transaction