From 97ee8665497c78aff2f30c2f652b7afd376b5323 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Mon, 1 Dec 2014 12:38:42 +0100 Subject: [REST] getutxos REST command (based on Bip64) has parts of @mhearn #4351 * allows querying the utxos over REST * same binary input and outputs as mentioned in Bip64 * input format = output format * various rpc/rest regtests --- qa/rpc-tests/rest.py | 186 +++++++++++++++++++++++++++++++++++++-- src/rest.cpp | 244 +++++++++++++++++++++++++++++++++++++++++++++++---- src/rpcserver.cpp | 2 +- src/rpcserver.h | 1 + 4 files changed, 407 insertions(+), 26 deletions(-) diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py index 9b7008531c..3c4f83b90d 100755 --- a/qa/rpc-tests/rest.py +++ b/qa/rpc-tests/rest.py @@ -9,7 +9,10 @@ from test_framework import BitcoinTestFramework from util import * +from struct import * +import binascii import json +import StringIO try: import http.client as httplib @@ -20,45 +23,210 @@ try: except ImportError: import urlparse -def http_get_call(host, port, path, response_object = 0): +def deser_uint256(f): + r = 0 + for i in range(8): + t = unpack(b" +#include using namespace std; using namespace json_spirit; +static const int MAX_GETUTXOS_OUTPOINTS = 100; //allow a max of 100 outpoints to be queried at once + enum RetFormat { RF_UNDEF, RF_BINARY, @@ -34,6 +38,22 @@ static const struct { {RF_JSON, "json"}, }; +struct CCoin { + uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE + uint32_t nHeight; + CTxOut out; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + { + READWRITE(nTxVer); + READWRITE(nHeight); + READWRITE(out); + } +}; + class RestErr { public: @@ -43,6 +63,7 @@ public: extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry); extern Object blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); +extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, Object& out, bool fIncludeHex); static RestErr RESTERR(enum HTTPStatusCode status, string message) { @@ -90,12 +111,13 @@ static bool ParseHashStr(const string& strReq, uint256& v) } static bool rest_headers(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map& mapHeaders, bool fRun) { vector params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); vector path; boost::split(path, params[0], boost::is_any_of("/")); @@ -153,13 +175,14 @@ static bool rest_headers(AcceptedConnection* conn, } static bool rest_block(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map& mapHeaders, bool fRun, bool showTxDetails) { vector params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; @@ -211,28 +234,31 @@ static bool rest_block(AcceptedConnection* conn, } static bool rest_block_extended(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map& mapHeaders, bool fRun) { - return rest_block(conn, strReq, mapHeaders, fRun, true); + return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, true); } static bool rest_block_notxdetails(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map& mapHeaders, bool fRun) { - return rest_block(conn, strReq, mapHeaders, fRun, false); + return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, false); } static bool rest_chaininfo(AcceptedConnection* conn, - const std::string& strReq, - const std::map& mapHeaders, - bool fRun) + const std::string& strURIPart, + const std::string& strRequest, + const std::map& mapHeaders, + bool fRun) { vector params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); switch (rf) { case RF_JSON: { @@ -253,12 +279,13 @@ static bool rest_chaininfo(AcceptedConnection* conn, } static bool rest_tx(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map& mapHeaders, bool fRun) { vector params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; @@ -303,10 +330,191 @@ static bool rest_tx(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } +static bool rest_getutxos(AcceptedConnection* conn, + const std::string& strURIPart, + const std::string& strRequest, + const std::map& mapHeaders, + bool fRun) +{ + vector params; + enum RetFormat rf = ParseDataFormat(params, strURIPart); + + // throw exception in case of a empty request + if (strRequest.length() == 0) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + + bool fCheckMemPool = false; + vector vOutPoints; + + // parse/deserialize input + // input-format = output-format, rest/getutxos/bin requires binary input, gives binary output, ... + + string strRequestMutable = strRequest; //convert const string to string for allowing hex to bin converting + + switch (rf) { + case RF_HEX: { + // convert hex to bin, continue then with bin part + std::vector strRequestV = ParseHex(strRequest); + strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); + } + + case RF_BINARY: { + try { + //deserialize + CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); + oss << strRequestMutable; + oss >> fCheckMemPool; + oss >> vOutPoints; + } catch (const std::ios_base::failure& e) { + // abort in case of unreadable binary data + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + } + break; + } + + case RF_JSON: { + try { + // parse json request + Value valRequest; + if (!read_string(strRequest, valRequest)) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + + Object jsonObject = valRequest.get_obj(); + const Value& checkMempoolValue = find_value(jsonObject, "checkmempool"); + + if (!checkMempoolValue.is_null()) { + fCheckMemPool = checkMempoolValue.get_bool(); + } + const Value& outpointsValue = find_value(jsonObject, "outpoints"); + if (!outpointsValue.is_null()) { + Array outPoints = outpointsValue.get_array(); + BOOST_FOREACH (const Value& outPoint, outPoints) { + Object outpointObject = outPoint.get_obj(); + uint256 txid = ParseHashO(outpointObject, "txid"); + Value nValue = find_value(outpointObject, "n"); + int nOutput = nValue.get_int(); + vOutPoints.push_back(COutPoint(txid, nOutput)); + } + } + } catch (...) { + // return HTTP 500 if there was a json parsing error + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + } + break; + } + default: { + throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } + + // limit max outpoints + if (vOutPoints.size() > MAX_GETUTXOS_OUTPOINTS) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); + + // check spentness and form a bitmap (as well as a JSON capable human-readble string representation) + vector bitmap; + vector outs; + std::string bitmapStringRepresentation; + boost::dynamic_bitset hits(vOutPoints.size()); + { + LOCK2(cs_main, mempool.cs); + + CCoinsView viewDummy; + CCoinsViewCache view(&viewDummy); + + CCoinsViewCache& viewChain = *pcoinsTip; + CCoinsViewMemPool viewMempool(&viewChain, mempool); + + if (fCheckMemPool) + view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool + + for (size_t i = 0; i < vOutPoints.size(); i++) { + CCoins coins; + uint256 hash = vOutPoints[i].hash; + if (view.GetCoins(hash, coins)) { + mempool.pruneSpent(hash, coins); + if (coins.IsAvailable(vOutPoints[i].n)) { + hits[i] = true; + // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if + // n is valid but points to an already spent output (IsNull). + CCoin coin; + coin.nTxVer = coins.nVersion; + coin.nHeight = coins.nHeight; + coin.out = coins.vout.at(vOutPoints[i].n); + assert(!coin.out.IsNull()); + outs.push_back(coin); + } + } + + bitmapStringRepresentation.append(hits[i] ? "1" : "0"); // form a binary string representation (human-readable for json output) + } + } + boost::to_block_range(hits, std::back_inserter(bitmap)); + + switch (rf) { + case RF_BINARY: { + // serialize data + // use exact same output as mentioned in Bip64 + CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); + ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; + string ssGetUTXOResponseString = ssGetUTXOResponse.str(); + + conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, ssGetUTXOResponseString.size(), "application/octet-stream") << ssGetUTXOResponseString << std::flush; + return true; + } + + case RF_HEX: { + CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); + ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; + string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; + + conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + return true; + } + + case RF_JSON: { + Object objGetUTXOResponse; + + // pack in some essentials + // use more or less the same output as mentioned in Bip64 + objGetUTXOResponse.push_back(Pair("chainHeight", chainActive.Height())); + objGetUTXOResponse.push_back(Pair("chaintipHash", chainActive.Tip()->GetBlockHash().GetHex())); + objGetUTXOResponse.push_back(Pair("bitmap", bitmapStringRepresentation)); + + Array utxos; + BOOST_FOREACH (const CCoin& coin, outs) { + Object utxo; + utxo.push_back(Pair("txvers", (int32_t)coin.nTxVer)); + utxo.push_back(Pair("height", (int32_t)coin.nHeight)); + utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); + + // include the script in a json output + Object o; + ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + utxo.push_back(Pair("scriptPubKey", o)); + utxos.push_back(utxo); + } + objGetUTXOResponse.push_back(Pair("utxos", utxos)); + + // return json string + string strJSON = write_string(Value(objGetUTXOResponse), false) + "\n"; + conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + return true; + } + default: { + throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } + + // not reached + return true; // continue to process further HTTP reqs on this cxn +} + static const struct { const char* prefix; bool (*handler)(AcceptedConnection* conn, - const std::string& strURI, + const std::string& strURIPart, + const std::string& strRequest, const std::map& mapHeaders, bool fRun); } uri_prefixes[] = { @@ -315,10 +523,12 @@ static const struct { {"/rest/block/", rest_block_extended}, {"/rest/chaininfo", rest_chaininfo}, {"/rest/headers/", rest_headers}, + {"/rest/getutxos", rest_getutxos}, }; bool HTTPReq_REST(AcceptedConnection* conn, const std::string& strURI, + const string& strRequest, const std::map& mapHeaders, bool fRun) { @@ -330,8 +540,8 @@ bool HTTPReq_REST(AcceptedConnection* conn, for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) { unsigned int plen = strlen(uri_prefixes[i].prefix); if (strURI.substr(0, plen) == uri_prefixes[i].prefix) { - string strReq = strURI.substr(plen); - return uri_prefixes[i].handler(conn, strReq, mapHeaders, fRun); + string strURIPart = strURI.substr(plen); + return uri_prefixes[i].handler(conn, strURIPart, strRequest, mapHeaders, fRun); } } } catch (const RestErr& re) { diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index e2df41fe21..254eea833e 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -992,7 +992,7 @@ void ServiceConnection(AcceptedConnection *conn) // Process via HTTP REST API } else if (strURI.substr(0, 6) == "/rest/" && GetBoolArg("-rest", false)) { - if (!HTTPReq_REST(conn, strURI, mapHeaders, fRun)) + if (!HTTPReq_REST(conn, strURI, strRequest, mapHeaders, fRun)) break; } else { diff --git a/src/rpcserver.h b/src/rpcserver.h index c3200d8c35..8dcad26106 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -237,6 +237,7 @@ extern json_spirit::Value reconsiderblock(const json_spirit::Array& params, bool // in rest.cpp extern bool HTTPReq_REST(AcceptedConnection *conn, const std::string& strURI, + const std::string& strRequest, const std::map& mapHeaders, bool fRun); -- cgit v1.2.3 From 6b4feb89a81a6f166f13ecf480d18ccea5ff462e Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Tue, 21 Apr 2015 20:32:37 +0200 Subject: [QA] rest.py RPC test: change setgenerate() to generate() --- doc/REST-interface.md | 7 +++++++ qa/rpc-tests/rest.py | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index f14aed7287..830445af73 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -34,6 +34,13 @@ Only supports JSON as output format. * verificationprogress : (numeric) estimate of verification progress [0..1] * chainwork : (string) total amount of work in active chain, in hexadecimal +`GET /rest/getutxos` + +The getutxo command allows querying of the UTXO set given a set of of outpoints. +See BIP64 for input and output serialisation: +https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki + + Risks ------------- Running a webbrowser on the same node with a REST enabled bitcoind can be a risk. Accessing prepared XSS websites could read out tx/block data of your node by placing links like `