From 90485755115424ed9ea70206f54da2b13777fa6c Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Mon, 15 Jan 2018 11:23:44 -1000 Subject: Add FindScriptPubKey() to search the UTXO set --- src/rpc/blockchain.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f70d506e13..715f011da6 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1916,6 +1916,35 @@ static UniValue savemempool(const JSONRPCRequest& request) return NullUniValue; } +//! Search for a given set of pubkey scripts +bool FindScriptPubKey(std::atomic& scan_progress, const std::atomic& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set& needles, std::map& out_results) { + scan_progress = 0; + count = 0; + while (cursor->Valid()) { + COutPoint key; + Coin coin; + if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; + if (++count % 8192 == 0) { + boost::this_thread::interruption_point(); + if (should_abort) { + // allow to abort the scan via the abort reference + return false; + } + } + if (count % 256 == 0) { + // update progress reference every 256 item + uint32_t high = 0x100 * *key.hash.begin() + *(key.hash.begin() + 1); + scan_progress = (int)(high * 100.0 / 65536.0 + 0.5); + } + if (needles.count(coin.out.scriptPubKey)) { + out_results.emplace(key, coin); + } + cursor->Next(); + } + scan_progress = 100; + return true; +} + static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- -- cgit v1.2.3 From 78304941f771b8bd918deddd37d01bc8f21873e1 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Mon, 15 Jan 2018 20:33:59 -1000 Subject: Blockchain/RPC: Add scantxoutset method to scan UTXO set --- src/rpc/blockchain.cpp | 245 +++++++++++++++++++++++++++++++++++++++++++++++++ src/rpc/client.cpp | 1 + 2 files changed, 246 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 715f011da6..183ef0b3a4 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include #include @@ -13,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +30,7 @@ #include #include +#include #include #include @@ -1945,6 +1949,246 @@ bool FindScriptPubKey(std::atomic& scan_progress, const std::atomic& return true; } +/** RAII object to prevent concurrency issue when scanning the txout set */ +static std::mutex g_utxosetscan; +static std::atomic g_scan_progress; +static std::atomic g_scan_in_progress; +static std::atomic g_should_abort_scan; +class CoinsViewScanReserver +{ +private: + bool m_could_reserve; +public: + explicit CoinsViewScanReserver() : m_could_reserve(false) {} + + bool reserve() { + assert (!m_could_reserve); + std::lock_guard lock(g_utxosetscan); + if (g_scan_in_progress) { + return false; + } + g_scan_in_progress = true; + m_could_reserve = true; + return true; + } + + ~CoinsViewScanReserver() { + if (m_could_reserve) { + std::lock_guard lock(g_utxosetscan); + g_scan_in_progress = false; + } + } +}; + +static const char *g_default_scantxoutset_script_types[] = { "P2PKH", "P2SH_P2WPKH", "P2WPKH" }; + +enum class OutputScriptType { + UNKNOWN, + P2PK, + P2PKH, + P2SH_P2WPKH, + P2WPKH +}; + +static inline OutputScriptType GetOutputScriptTypeFromString(const std::string& outputtype) +{ + if (outputtype == "P2PK") return OutputScriptType::P2PK; + else if (outputtype == "P2PKH") return OutputScriptType::P2PKH; + else if (outputtype == "P2SH_P2WPKH") return OutputScriptType::P2SH_P2WPKH; + else if (outputtype == "P2WPKH") return OutputScriptType::P2WPKH; + else return OutputScriptType::UNKNOWN; +} + +CTxDestination GetDestinationForKey(const CPubKey& key, OutputScriptType type) +{ + switch (type) { + case OutputScriptType::P2PKH: return key.GetID(); + case OutputScriptType::P2SH_P2WPKH: + case OutputScriptType::P2WPKH: { + if (!key.IsCompressed()) return key.GetID(); + CTxDestination witdest = WitnessV0KeyHash(key.GetID()); + if (type == OutputScriptType::P2SH_P2WPKH) { + CScript witprog = GetScriptForDestination(witdest); + return CScriptID(witprog); + } else { + return witdest; + } + } + default: assert(false); + } +} + +UniValue scantxoutset(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) + throw std::runtime_error( + "scantxoutset ( )\n" + "\nScans the unspent transaction output set for possible entries that matches common scripts of given public keys.\n" + "\nArguments:\n" + "1. \"action\" (string, required) The action to execute\n" + " \"start\" for starting a scan\n" + " \"abort\" for aborting the current scan (returns true when abort was successful)\n" + " \"status\" for progress report (in %) of the current scan\n" + "2. \"scanobjects\" (array, optional) Array of scan objects (only one object type per scan object allowed)\n" + " [\n" + " { \"address\" : \"
\" }, (string, optional) Bitcoin address\n" + " { \"pubkey\" : (object, optional) Public key\n" + " {\n" + " \"pubkey\" : \", (string, required) HEX encoded public key\n" + " \"script_types\" : [ ... ], (array, optional) Array of script-types to derive from the pubkey (possible values: \"P2PKH\", \"P2SH-P2WPKH\", \"P2WPKH\")\n" + " }\n" + " },\n" + " ]\n" + "\nResult:\n" + "{\n" + " \"unspents\": [\n" + " {\n" + " \"txid\" : \"transactionid\", (string) The transaction id\n" + " \"vout\": n, (numeric) the vout value\n" + " \"scriptPubKey\" : \"script\", (string) the script key\n" + " \"amount\" : x.xxx, (numeric) The total amount in " + CURRENCY_UNIT + " of the unspent output\n" + " \"height\" : n, (numeric) Height of the unspent transaction output\n" + " }\n" + " ,...], \n" + " \"total_amount\" : x.xxx, (numeric) The total amount of all found unspent outputs in " + CURRENCY_UNIT + "\n" + "]\n" + ); + + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR}); + + UniValue result(UniValue::VOBJ); + if (request.params[0].get_str() == "status") { + CoinsViewScanReserver reserver; + if (reserver.reserve()) { + // no scan in progress + return NullUniValue; + } + result.pushKV("progress", g_scan_progress); + return result; + } else if (request.params[0].get_str() == "abort") { + CoinsViewScanReserver reserver; + if (reserver.reserve()) { + // reserve was possible which means no scan was running + return false; + } + // set the abort flag + g_should_abort_scan = true; + return true; + } else if (request.params[0].get_str() == "start") { + CoinsViewScanReserver reserver; + if (!reserver.reserve()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\""); + } + std::set needles; + CAmount total_in = 0; + + // loop through the scan objects + for (const UniValue& scanobject : request.params[1].get_array().getValues()) { + if (!scanobject.isObject()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid scan object"); + } + UniValue address_uni = find_value(scanobject, "address"); + UniValue pubkey_uni = find_value(scanobject, "pubkey"); + + // make sure only one object type is present + if (1 != !address_uni.isNull() + !pubkey_uni.isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one object type is allowed per scan object"); + } else if (!address_uni.isNull() && !address_uni.isStr()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Scanobject \"address\" must contain a single string as value"); + } else if (!pubkey_uni.isNull() && !pubkey_uni.isObject()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Scanobject \"pubkey\" must contain an object as value"); + } else if (address_uni.isStr()) { + // type: address + // decode destination and derive the scriptPubKey + // add the script to the scan containers + CTxDestination dest = DecodeDestination(address_uni.get_str()); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + CScript script = GetScriptForDestination(dest); + assert(!script.empty()); + needles.insert(script); + } else if (pubkey_uni.isObject()) { + // type: pubkey + // derive script(s) according to the script_type parameter + UniValue script_types_uni = find_value(pubkey_uni, "script_types"); + UniValue pubkeydata_uni = find_value(pubkey_uni, "pubkey"); + + // check the script types and use the default if not provided + if (!script_types_uni.isNull() && !script_types_uni.isArray()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "script_types must be an array"); + } + else if (script_types_uni.isNull()) { + // use the default script types + script_types_uni = UniValue(UniValue::VARR); + for (const char *t : g_default_scantxoutset_script_types) { + script_types_uni.push_back(t); + } + } + + // check the acctual pubkey + if (!pubkeydata_uni.isStr() || !IsHex(pubkeydata_uni.get_str())) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Public key must be hex encoded"); + } + CPubKey pubkey(ParseHexV(pubkeydata_uni, "pubkey")); + if (!pubkey.IsFullyValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid public key"); + } + + // loop through the script types and derive the script + for (const UniValue& script_type_uni : script_types_uni.get_array().getValues()) { + OutputScriptType script_type = GetOutputScriptTypeFromString(script_type_uni.get_str()); + if (script_type == OutputScriptType::UNKNOWN) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid script type"); + + CScript script = GetScriptForDestination(GetDestinationForKey(pubkey, script_type)); + assert(!script.empty()); + needles.insert(script); + } + } + } + + // Scan the unspent transaction output set for inputs + UniValue unspents(UniValue::VARR); + std::vector input_txos; + std::map coins; + g_should_abort_scan = false; + g_scan_progress = 0; + int64_t count = 0; + std::unique_ptr pcursor; + { + LOCK(cs_main); + FlushStateToDisk(); + pcursor = std::unique_ptr(pcoinsdbview->Cursor()); + assert(pcursor); + } + bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins); + result.pushKV("success", res); + result.pushKV("searched_items", count); + + for (const auto& it : coins) { + const COutPoint& outpoint = it.first; + const Coin& coin = it.second; + const CTxOut& txo = coin.out; + input_txos.push_back(txo); + total_in += txo.nValue; + + UniValue unspent(UniValue::VOBJ); + unspent.pushKV("txid", outpoint.hash.GetHex()); + unspent.pushKV("vout", (int32_t)outpoint.n); + unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey.begin(), txo.scriptPubKey.end())); + unspent.pushKV("amount", ValueFromAmount(txo.nValue)); + unspent.pushKV("height", (int32_t)coin.nHeight); + + unspents.push_back(unspent); + } + result.pushKV("unspents", unspents); + result.pushKV("total_amount", ValueFromAmount(total_in)); + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid command"); + } + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- @@ -1970,6 +2214,7 @@ static const CRPCCommand commands[] = { "blockchain", "verifychain", &verifychain, {"checklevel","nblocks"} }, { "blockchain", "preciousblock", &preciousblock, {"blockhash"} }, + { "blockchain", "scantxoutset", &scantxoutset, {"action", "scanobjects"} }, /* Not shown in help */ { "hidden", "invalidateblock", &invalidateblock, {"blockhash"} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index bb68f72ccc..055ab6ed3e 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -79,6 +79,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendmany", 4, "subtractfeefrom" }, { "sendmany", 5 , "replaceable" }, { "sendmany", 6 , "conf_target" }, + { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, { "createmultisig", 0, "nrequired" }, -- cgit v1.2.3 From 892de1dfea283a5d6ac18b8c74b57f61a920c762 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Thu, 31 May 2018 15:12:16 +0200 Subject: scantxoutset: add support for scripts --- src/rpc/blockchain.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 183ef0b3a4..8fec32f05e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2032,6 +2032,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) "2. \"scanobjects\" (array, optional) Array of scan objects (only one object type per scan object allowed)\n" " [\n" " { \"address\" : \"
\" }, (string, optional) Bitcoin address\n" + " { \"script\" : \"\" }, (string, optional) HEX encoded script (scriptPubKey)\n" " { \"pubkey\" : (object, optional) Public key\n" " {\n" " \"pubkey\" : \", (string, required) HEX encoded public key\n" @@ -2089,14 +2090,17 @@ UniValue scantxoutset(const JSONRPCRequest& request) } UniValue address_uni = find_value(scanobject, "address"); UniValue pubkey_uni = find_value(scanobject, "pubkey"); + UniValue script_uni = find_value(scanobject, "script"); // make sure only one object type is present - if (1 != !address_uni.isNull() + !pubkey_uni.isNull()) { + if (1 != !address_uni.isNull() + !pubkey_uni.isNull() + !script_uni.isNull()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one object type is allowed per scan object"); } else if (!address_uni.isNull() && !address_uni.isStr()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Scanobject \"address\" must contain a single string as value"); } else if (!pubkey_uni.isNull() && !pubkey_uni.isObject()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Scanobject \"pubkey\" must contain an object as value"); + } else if (!script_uni.isNull() && !script_uni.isStr()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Scanobject \"script\" must contain a single string as value"); } else if (address_uni.isStr()) { // type: address // decode destination and derive the scriptPubKey @@ -2117,8 +2121,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) // check the script types and use the default if not provided if (!script_types_uni.isNull() && !script_types_uni.isArray()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "script_types must be an array"); - } - else if (script_types_uni.isNull()) { + } else if (script_types_uni.isNull()) { // use the default script types script_types_uni = UniValue(UniValue::VARR); for (const char *t : g_default_scantxoutset_script_types) { @@ -2144,6 +2147,12 @@ UniValue scantxoutset(const JSONRPCRequest& request) assert(!script.empty()); needles.insert(script); } + } else if (script_uni.isStr()) { + // type: script + // check and add the script to the scan containers (needles array) + CScript script(ParseHexV(script_uni, "script")); + // TODO: check script: max length, has OP, is unspenable etc. + needles.insert(script); } } -- cgit v1.2.3 From 94d73d32abe927e74271a7b3eac7ba56658a535e Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Wed, 13 Jun 2018 16:00:30 +0200 Subject: scantxoutset: support legacy P2PK script type --- src/rpc/blockchain.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 8fec32f05e..a339bd10f0 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2036,7 +2036,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) " { \"pubkey\" : (object, optional) Public key\n" " {\n" " \"pubkey\" : \", (string, required) HEX encoded public key\n" - " \"script_types\" : [ ... ], (array, optional) Array of script-types to derive from the pubkey (possible values: \"P2PKH\", \"P2SH-P2WPKH\", \"P2WPKH\")\n" + " \"script_types\" : [ ... ], (array, optional) Array of script-types to derive from the pubkey (possible values: \"P2PK\", \"P2PKH\", \"P2SH-P2WPKH\", \"P2WPKH\")\n" " }\n" " },\n" " ]\n" @@ -2142,8 +2142,13 @@ UniValue scantxoutset(const JSONRPCRequest& request) for (const UniValue& script_type_uni : script_types_uni.get_array().getValues()) { OutputScriptType script_type = GetOutputScriptTypeFromString(script_type_uni.get_str()); if (script_type == OutputScriptType::UNKNOWN) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid script type"); - - CScript script = GetScriptForDestination(GetDestinationForKey(pubkey, script_type)); + CScript script; + if (script_type == OutputScriptType::P2PK) { + // support legacy P2PK scripts + script << ToByteVector(pubkey) << OP_CHECKSIG; + } else { + script = GetScriptForDestination(GetDestinationForKey(pubkey, script_type)); + } assert(!script.empty()); needles.insert(script); } -- cgit v1.2.3 From eec7cf7b33cafac6a969bce38905cbacda56b1a5 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Wed, 13 Jun 2018 22:38:40 +0200 Subject: scantxoutset: mention that scanning by address will miss P2PK txouts --- src/rpc/blockchain.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a339bd10f0..852abe55de 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2024,6 +2024,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) throw std::runtime_error( "scantxoutset ( )\n" "\nScans the unspent transaction output set for possible entries that matches common scripts of given public keys.\n" + "Using addresses as scanobjects will _not_ detect unspent P2PK txouts\n" "\nArguments:\n" "1. \"action\" (string, required) The action to execute\n" " \"start\" for starting a scan\n" -- cgit v1.2.3 From be98b2d9a8fcb5f0e29ea88f026d604442fe984c Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Mon, 15 Jan 2018 20:34:07 -1000 Subject: [QA] Add scantxoutset test --- test/functional/rpc_scantxoutset.py | 48 +++++++++++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 2 files changed, 49 insertions(+) create mode 100755 test/functional/rpc_scantxoutset.py diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py new file mode 100755 index 0000000000..ce5d4da9e7 --- /dev/null +++ b/test/functional/rpc_scantxoutset.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the scantxoutset rpc call.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +import shutil +import os + +class ScantxoutsetTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + def run_test(self): + self.log.info("Mining blocks...") + self.nodes[0].generate(110) + + addr_P2SH_SEGWIT = self.nodes[0].getnewaddress("", "p2sh-segwit") + pubk1 = self.nodes[0].getaddressinfo(addr_P2SH_SEGWIT)['pubkey'] + addr_LEGACY = self.nodes[0].getnewaddress("", "legacy") + pubk2 = self.nodes[0].getaddressinfo(addr_LEGACY)['pubkey'] + addr_BECH32 = self.nodes[0].getnewaddress("", "bech32") + pubk3 = self.nodes[0].getaddressinfo(addr_BECH32)['pubkey'] + self.nodes[0].sendtoaddress(addr_P2SH_SEGWIT, 1) + self.nodes[0].sendtoaddress(addr_LEGACY, 2) + self.nodes[0].sendtoaddress(addr_BECH32, 3) + self.nodes[0].generate(1) + + self.log.info("Stop node, remove wallet, mine again some blocks...") + self.stop_node(0) + shutil.rmtree(os.path.join(self.nodes[0].datadir, "regtest", 'wallets')) + self.start_node(0) + self.nodes[0].generate(110) + + self.restart_node(0, ['-nowallet']) + self.log.info("Test if we have found the non HD unspent outputs.") + assert_equal(self.nodes[0].scantxoutset("start", [ {"pubkey": {"pubkey": pubk1}}, {"pubkey": {"pubkey": pubk2}}, {"pubkey": {"pubkey": pubk3}}])['total_amount'], 6) + assert_equal(self.nodes[0].scantxoutset("start", [ {"address": addr_P2SH_SEGWIT}, {"address": addr_LEGACY}, {"address": addr_BECH32}])['total_amount'], 6) + assert_equal(self.nodes[0].scantxoutset("start", [ {"address": addr_P2SH_SEGWIT}, {"address": addr_LEGACY}, {"pubkey": {"pubkey": pubk3}} ])['total_amount'], 6) + + self.log.info("Test invalid parameters.") + assert_raises_rpc_error(-8, 'Scanobject "pubkey" must contain an object as value', self.nodes[0].scantxoutset, "start", [ {"pubkey": pubk1}]) #missing pubkey object + assert_raises_rpc_error(-8, 'Scanobject "address" must contain a single string as value', self.nodes[0].scantxoutset, "start", [ {"address": {"address": addr_P2SH_SEGWIT}}]) #invalid object for address object + +if __name__ == '__main__': + ScantxoutsetTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 65e4c0817e..dcf04c39e1 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -140,6 +140,7 @@ BASE_SCRIPTS = [ 'feature_uacomment.py', 'p2p_unrequested_blocks.py', 'feature_includeconf.py', + 'rpc_scantxoutset.py', 'feature_logging.py', 'p2p_node_network_limited.py', 'feature_blocksdir.py', -- cgit v1.2.3