Merge #12196: Add scantxoutset RPC method

be98b2d9a8fcb5f0e29ea88f026d604442fe984c [QA] Add scantxoutset test (Jonas Schnelli)
eec7cf7b33cafac6a969bce38905cbacda56b1a5 scantxoutset: mention that scanning by address will miss P2PK txouts (Jonas Schnelli)
94d73d32abe927e74271a7b3eac7ba56658a535e scantxoutset: support legacy P2PK script type (Jonas Schnelli)
892de1dfea283a5d6ac18b8c74b57f61a920c762 scantxoutset: add support for scripts (Jonas Schnelli)
78304941f771b8bd918deddd37d01bc8f21873e1 Blockchain/RPC: Add scantxoutset method to scan UTXO set (Jonas Schnelli)
90485755115424ed9ea70206f54da2b13777fa6c Add FindScriptPubKey() to search the UTXO set (Jonas Schnelli)

Pull request description:

  Alternative to #9152.

  This takes `<n>` pubkeys and optionally  `<n>` xpubs (together with a definable lookup windows where the default is 0-1000) and looks up common scripts in the UTXO set of all given or derived keys.

  The output will be an array similar to `listunspent`. That array is compatible with `createrawtransaction` as well as with `signrawtransaction`.

  This makes it possible to prepare sweeps and have them signed in a secure (cold) space.

Tree-SHA512: a2b22a117cf6e27febeb97e5d6fe30184926d50c0c7cbc77bb4121f490fed65560c52f8eac67a9720d7bf8f420efa42459768685c7e7cc03722859f51a5e1e3b

4 files changed, 339 insertions, 0 deletions

diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index d9d803ac7d..012e3e3ac1 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -6,6 +6,8 @@
 #include <rpc/blockchain.h>
 
 #include <amount.h>
+#include <base58.h>
+#include <chain.h>
 #include <chainparams.h>
 #include <checkpoints.h>
 #include <coins.h>
@@ -13,6 +15,7 @@
 #include <validation.h>
 #include <core_io.h>
 #include <index/txindex.h>
+#include <key_io.h>
 #include <policy/feerate.h>
 #include <policy/policy.h>
 #include <primitives/transaction.h>
@@ -27,6 +30,7 @@
 #include <validationinterface.h>
 #include <warnings.h>
 
+#include <assert.h>
 #include <stdint.h>
 
 #include <univalue.h>
@@ -1920,6 +1924,290 @@ static UniValue savemempool(const JSONRPCRequest& request)
     return NullUniValue;
 }
 
+//! Search for a given set of pubkey scripts
+bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set<CScript>& needles, std::map<COutPoint, Coin>& out_results) {
+    scan_progress = 0;
+    count = 0;
+    while (cursor->Valid()) {
+        COutPoint key;
+        Coin coin;
+        if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false;
+        if (++count % 8192 == 0) {
+            boost::this_thread::interruption_point();
+            if (should_abort) {
+                // allow to abort the scan via the abort reference
+                return false;
+            }
+        }
+        if (count % 256 == 0) {
+            // update progress reference every 256 item
+            uint32_t high = 0x100 * *key.hash.begin() + *(key.hash.begin() + 1);
+            scan_progress = (int)(high * 100.0 / 65536.0 + 0.5);
+        }
+        if (needles.count(coin.out.scriptPubKey)) {
+            out_results.emplace(key, coin);
+        }
+        cursor->Next();
+    }
+    scan_progress = 100;
+    return true;
+}
+
+/** RAII object to prevent concurrency issue when scanning the txout set */
+static std::mutex g_utxosetscan;
+static std::atomic<int> g_scan_progress;
+static std::atomic<bool> g_scan_in_progress;
+static std::atomic<bool> g_should_abort_scan;
+class CoinsViewScanReserver
+{
+private:
+    bool m_could_reserve;
+public:
+    explicit CoinsViewScanReserver() : m_could_reserve(false) {}
+
+    bool reserve() {
+        assert (!m_could_reserve);
+        std::lock_guard<std::mutex> lock(g_utxosetscan);
+        if (g_scan_in_progress) {
+            return false;
+        }
+        g_scan_in_progress = true;
+        m_could_reserve = true;
+        return true;
+    }
+
+    ~CoinsViewScanReserver() {
+        if (m_could_reserve) {
+            std::lock_guard<std::mutex> lock(g_utxosetscan);
+            g_scan_in_progress = false;
+        }
+    }
+};
+
+static const char *g_default_scantxoutset_script_types[] = { "P2PKH", "P2SH_P2WPKH", "P2WPKH" };
+
+enum class OutputScriptType {
+    UNKNOWN,
+    P2PK,
+    P2PKH,
+    P2SH_P2WPKH,
+    P2WPKH
+};
+
+static inline OutputScriptType GetOutputScriptTypeFromString(const std::string& outputtype)
+{
+    if (outputtype == "P2PK") return OutputScriptType::P2PK;
+    else if (outputtype == "P2PKH") return OutputScriptType::P2PKH;
+    else if (outputtype == "P2SH_P2WPKH") return OutputScriptType::P2SH_P2WPKH;
+    else if (outputtype == "P2WPKH") return OutputScriptType::P2WPKH;
+    else return OutputScriptType::UNKNOWN;
+}
+
+CTxDestination GetDestinationForKey(const CPubKey& key, OutputScriptType type)
+{
+    switch (type) {
+    case OutputScriptType::P2PKH: return key.GetID();
+    case OutputScriptType::P2SH_P2WPKH:
+    case OutputScriptType::P2WPKH: {
+        if (!key.IsCompressed()) return key.GetID();
+        CTxDestination witdest = WitnessV0KeyHash(key.GetID());
+        if (type == OutputScriptType::P2SH_P2WPKH) {
+            CScript witprog = GetScriptForDestination(witdest);
+            return CScriptID(witprog);
+        } else {
+            return witdest;
+        }
+    }
+    default: assert(false);
+    }
+}
+
+UniValue scantxoutset(const JSONRPCRequest& request)
+{
+    if (request.fHelp || request.params.size() < 1 || request.params.size() > 2)
+        throw std::runtime_error(
+            "scantxoutset <action> ( <scanobjects> )\n"
+            "\nScans the unspent transaction output set for possible entries that matches common scripts of given public keys.\n"
+            "Using addresses as scanobjects will _not_ detect unspent P2PK txouts\n"
+            "\nArguments:\n"
+            "1. \"action\"                       (string, required) The action to execute\n"
+            "                                      \"start\" for starting a scan\n"
+            "                                      \"abort\" for aborting the current scan (returns true when abort was successful)\n"
+            "                                      \"status\" for progress report (in %) of the current scan\n"
+            "2. \"scanobjects\"                  (array, optional) Array of scan objects (only one object type per scan object allowed)\n"
+            "      [\n"
+            "        { \"address\" : \"<address>\" },       (string, optional) Bitcoin address\n"
+            "        { \"script\"  : \"<scriptPubKey>\" },  (string, optional) HEX encoded script (scriptPubKey)\n"
+            "        { \"pubkey\"  :                      (object, optional) Public key\n"
+            "          {\n"
+            "            \"pubkey\" : \"<pubkey\">,         (string, required) HEX encoded public key\n"
+            "            \"script_types\" : [ ... ],      (array, optional) Array of script-types to derive from the pubkey (possible values: \"P2PK\", \"P2PKH\", \"P2SH-P2WPKH\", \"P2WPKH\")\n"
+            "          }\n"
+            "        },\n"
+            "      ]\n"
+            "\nResult:\n"
+            "{\n"
+            "  \"unspents\": [\n"
+            "    {\n"
+            "    \"txid\" : \"transactionid\",     (string) The transaction id\n"
+            "    \"vout\": n,                    (numeric) the vout value\n"
+            "    \"scriptPubKey\" : \"script\",    (string) the script key\n"
+            "    \"amount\" : x.xxx,             (numeric) The total amount in " + CURRENCY_UNIT + " of the unspent output\n"
+            "    \"height\" : n,                 (numeric) Height of the unspent transaction output\n"
+            "   }\n"
+            "   ,...], \n"
+            " \"total_amount\" : x.xxx,          (numeric) The total amount of all found unspent outputs in " + CURRENCY_UNIT + "\n"
+            "]\n"
+        );
+
+    RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR});
+
+    UniValue result(UniValue::VOBJ);
+    if (request.params[0].get_str() == "status") {
+        CoinsViewScanReserver reserver;
+        if (reserver.reserve()) {
+            // no scan in progress
+            return NullUniValue;
+        }
+        result.pushKV("progress", g_scan_progress);
+        return result;
+    } else if (request.params[0].get_str() == "abort") {
+        CoinsViewScanReserver reserver;
+        if (reserver.reserve()) {
+            // reserve was possible which means no scan was running
+            return false;
+        }
+        // set the abort flag
+        g_should_abort_scan = true;
+        return true;
+    } else if (request.params[0].get_str() == "start") {
+        CoinsViewScanReserver reserver;
+        if (!reserver.reserve()) {
+            throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\"");
+        }
+        std::set<CScript> needles;
+        CAmount total_in = 0;
+
+        // loop through the scan objects
+        for (const UniValue& scanobject : request.params[1].get_array().getValues()) {
+            if (!scanobject.isObject()) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid scan object");
+            }
+            UniValue address_uni = find_value(scanobject, "address");
+            UniValue pubkey_uni  = find_value(scanobject, "pubkey");
+            UniValue script_uni  = find_value(scanobject, "script");
+
+            // make sure only one object type is present
+            if (1 != !address_uni.isNull() + !pubkey_uni.isNull() + !script_uni.isNull()) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one object type is allowed per scan object");
+            } else if (!address_uni.isNull() && !address_uni.isStr()) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "Scanobject \"address\" must contain a single string as value");
+            } else if (!pubkey_uni.isNull() && !pubkey_uni.isObject()) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "Scanobject \"pubkey\" must contain an object as value");
+            } else if (!script_uni.isNull() && !script_uni.isStr()) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "Scanobject \"script\" must contain a single string as value");
+            } else if (address_uni.isStr()) {
+                // type: address
+                // decode destination and derive the scriptPubKey
+                // add the script to the scan containers
+                CTxDestination dest = DecodeDestination(address_uni.get_str());
+                if (!IsValidDestination(dest)) {
+                    throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address");
+                }
+                CScript script = GetScriptForDestination(dest);
+                assert(!script.empty());
+                needles.insert(script);
+            } else if (pubkey_uni.isObject()) {
+                // type: pubkey
+                // derive script(s) according to the script_type parameter
+                UniValue script_types_uni = find_value(pubkey_uni, "script_types");
+                UniValue pubkeydata_uni = find_value(pubkey_uni, "pubkey");
+
+                // check the script types and use the default if not provided
+                if (!script_types_uni.isNull() && !script_types_uni.isArray()) {
+                    throw JSONRPCError(RPC_INVALID_PARAMETER, "script_types must be an array");
+                } else if (script_types_uni.isNull()) {
+                    // use the default script types
+                    script_types_uni = UniValue(UniValue::VARR);
+                    for (const char *t : g_default_scantxoutset_script_types) {
+                        script_types_uni.push_back(t);
+                    }
+                }
+
+                // check the acctual pubkey
+                if (!pubkeydata_uni.isStr() || !IsHex(pubkeydata_uni.get_str())) {
+                    throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Public key must be hex encoded");
+                }
+                CPubKey pubkey(ParseHexV(pubkeydata_uni, "pubkey"));
+                if (!pubkey.IsFullyValid()) {
+                    throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid public key");
+                }
+
+                // loop through the script types and derive the script
+                for (const UniValue& script_type_uni : script_types_uni.get_array().getValues()) {
+                    OutputScriptType script_type = GetOutputScriptTypeFromString(script_type_uni.get_str());
+                    if (script_type == OutputScriptType::UNKNOWN) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid script type");
+                    CScript script;
+                    if (script_type == OutputScriptType::P2PK) {
+                        // support legacy P2PK scripts
+                        script << ToByteVector(pubkey) << OP_CHECKSIG;
+                    } else {
+                        script = GetScriptForDestination(GetDestinationForKey(pubkey, script_type));
+                    }
+                    assert(!script.empty());
+                    needles.insert(script);
+                }
+            } else if (script_uni.isStr()) {
+                // type: script
+                // check and add the script to the scan containers (needles array)
+                CScript script(ParseHexV(script_uni, "script"));
+                // TODO: check script: max length, has OP, is unspenable etc.
+                needles.insert(script);
+            }
+        }
+
+        // Scan the unspent transaction output set for inputs
+        UniValue unspents(UniValue::VARR);
+        std::vector<CTxOut> input_txos;
+        std::map<COutPoint, Coin> coins;
+        g_should_abort_scan = false;
+        g_scan_progress = 0;
+        int64_t count = 0;
+        std::unique_ptr<CCoinsViewCursor> pcursor;
+        {
+            LOCK(cs_main);
+            FlushStateToDisk();
+            pcursor = std::unique_ptr<CCoinsViewCursor>(pcoinsdbview->Cursor());
+            assert(pcursor);
+        }
+        bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins);
+        result.pushKV("success", res);
+        result.pushKV("searched_items", count);
+
+        for (const auto& it : coins) {
+            const COutPoint& outpoint = it.first;
+            const Coin& coin = it.second;
+            const CTxOut& txo = coin.out;
+            input_txos.push_back(txo);
+            total_in += txo.nValue;
+
+            UniValue unspent(UniValue::VOBJ);
+            unspent.pushKV("txid", outpoint.hash.GetHex());
+            unspent.pushKV("vout", (int32_t)outpoint.n);
+            unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey.begin(), txo.scriptPubKey.end()));
+            unspent.pushKV("amount", ValueFromAmount(txo.nValue));
+            unspent.pushKV("height", (int32_t)coin.nHeight);
+
+            unspents.push_back(unspent);
+        }
+        result.pushKV("unspents", unspents);
+        result.pushKV("total_amount", ValueFromAmount(total_in));
+    } else {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid command");
+    }
+    return result;
+}
+
 static const CRPCCommand commands[] =
 { //  category              name                      actor (function)         argNames
   //  --------------------- ------------------------  -----------------------  ----------
@@ -1945,6 +2233,7 @@ static const CRPCCommand commands[] =
     { "blockchain",         "verifychain",            &verifychain,            {"checklevel","nblocks"} },
 
     { "blockchain",         "preciousblock",          &preciousblock,          {"blockhash"} },
+    { "blockchain",         "scantxoutset",           &scantxoutset,           {"action", "scanobjects"} },
 
     /* Not shown in help */
     { "hidden",             "invalidateblock",        &invalidateblock,        {"blockhash"} },
diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp
index 0f35fd3770..ce608631ff 100644
--- a/src/rpc/client.cpp
+++ b/src/rpc/client.cpp
@@ -78,6 +78,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "sendmany", 4, "subtractfeefrom" },
     { "sendmany", 5 , "replaceable" },
     { "sendmany", 6 , "conf_target" },
+    { "scantxoutset", 1, "scanobjects" },
     { "addmultisigaddress", 0, "nrequired" },
     { "addmultisigaddress", 1, "keys" },
     { "createmultisig", 0, "nrequired" },
diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py
new file mode 100755
index 0000000000..ce5d4da9e7
--- /dev/null
+++ b/test/functional/rpc_scantxoutset.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the scantxoutset rpc call."""
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+
+import shutil
+import os
+
+class ScantxoutsetTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+        self.setup_clean_chain = True
+    def run_test(self):
+        self.log.info("Mining blocks...")
+        self.nodes[0].generate(110)
+
+        addr_P2SH_SEGWIT = self.nodes[0].getnewaddress("", "p2sh-segwit")
+        pubk1 = self.nodes[0].getaddressinfo(addr_P2SH_SEGWIT)['pubkey']
+        addr_LEGACY = self.nodes[0].getnewaddress("", "legacy")
+        pubk2 = self.nodes[0].getaddressinfo(addr_LEGACY)['pubkey']
+        addr_BECH32 = self.nodes[0].getnewaddress("", "bech32")
+        pubk3 = self.nodes[0].getaddressinfo(addr_BECH32)['pubkey']
+        self.nodes[0].sendtoaddress(addr_P2SH_SEGWIT, 1)
+        self.nodes[0].sendtoaddress(addr_LEGACY, 2)
+        self.nodes[0].sendtoaddress(addr_BECH32, 3)
+        self.nodes[0].generate(1)
+
+        self.log.info("Stop node, remove wallet, mine again some blocks...")
+        self.stop_node(0)
+        shutil.rmtree(os.path.join(self.nodes[0].datadir, "regtest", 'wallets'))
+        self.start_node(0)
+        self.nodes[0].generate(110)
+
+        self.restart_node(0, ['-nowallet'])
+        self.log.info("Test if we have found the non HD unspent outputs.")
+        assert_equal(self.nodes[0].scantxoutset("start", [ {"pubkey": {"pubkey": pubk1}}, {"pubkey": {"pubkey": pubk2}}, {"pubkey": {"pubkey": pubk3}}])['total_amount'], 6)
+        assert_equal(self.nodes[0].scantxoutset("start", [ {"address": addr_P2SH_SEGWIT}, {"address": addr_LEGACY}, {"address": addr_BECH32}])['total_amount'], 6)
+        assert_equal(self.nodes[0].scantxoutset("start", [ {"address": addr_P2SH_SEGWIT}, {"address": addr_LEGACY}, {"pubkey": {"pubkey": pubk3}} ])['total_amount'], 6)
+
+        self.log.info("Test invalid parameters.")
+        assert_raises_rpc_error(-8, 'Scanobject "pubkey" must contain an object as value', self.nodes[0].scantxoutset, "start", [ {"pubkey": pubk1}]) #missing pubkey object
+        assert_raises_rpc_error(-8, 'Scanobject "address" must contain a single string as value', self.nodes[0].scantxoutset, "start", [ {"address": {"address": addr_P2SH_SEGWIT}}]) #invalid object for address object
+
+if __name__ == '__main__':
+    ScantxoutsetTest().main()
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 49833e5dd4..5d039d7bc4 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -142,6 +142,7 @@ BASE_SCRIPTS = [
     'feature_uacomment.py',
     'p2p_unrequested_blocks.py',
     'feature_includeconf.py',
+    'rpc_scantxoutset.py',
     'feature_logging.py',
     'p2p_node_network_limited.py',
     'feature_blocksdir.py',