diff options
-rwxr-xr-x | qa/rpc-tests/rest.py | 186 | ||||
-rw-r--r-- | src/rest.cpp | 244 | ||||
-rw-r--r-- | src/rpcserver.cpp | 2 | ||||
-rw-r--r-- | src/rpcserver.h | 1 |
4 files changed, 407 insertions, 26 deletions
diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py index 9b7008531c..3c4f83b90d 100755 --- a/qa/rpc-tests/rest.py +++ b/qa/rpc-tests/rest.py @@ -9,7 +9,10 @@ from test_framework import BitcoinTestFramework from util import * +from struct import * +import binascii import json +import StringIO try: import http.client as httplib @@ -20,45 +23,210 @@ try: except ImportError: import urlparse -def http_get_call(host, port, path, response_object = 0): +def deser_uint256(f): + r = 0 + for i in range(8): + t = unpack(b"<I", f.read(4))[0] + r += t << (i * 32) + return r + +#allows simple http get calls with a request body +def http_get_call(host, port, path, requestdata = '', response_object = 0): conn = httplib.HTTPConnection(host, port) - conn.request('GET', path) + conn.request('GET', path, requestdata) if response_object: return conn.getresponse() return conn.getresponse().read() - class RESTTest (BitcoinTestFramework): FORMAT_SEPARATOR = "." + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 3) + + def setup_network(self, split=False): + self.nodes = start_nodes(3, self.options.tmpdir) + connect_nodes_bi(self.nodes,0,1) + connect_nodes_bi(self.nodes,1,2) + connect_nodes_bi(self.nodes,0,2) + self.is_network_split=False + self.sync_all() + def run_test(self): url = urlparse.urlparse(self.nodes[0].url) + print "Mining blocks..." + + self.nodes[0].setgenerate(True, 1) + self.sync_all() + self.nodes[2].setgenerate(True, 100) + self.sync_all() + + assert_equal(self.nodes[0].getbalance(), 50) + + txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) + self.sync_all() + self.nodes[2].setgenerate(True, 1) + self.sync_all() + bb_hash = self.nodes[0].getbestblockhash() + + assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) #balance now should be 0.1 on node 1 + + # load the latest 0.1 tx over the REST API + json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json") + json_obj = json.loads(json_string) + vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then) + # get n of 0.1 outpoint + n = 0 + for vout in json_obj['vout']: + if vout['value'] == 0.1: + n = vout['n'] + + + ###################################### + # GETUTXOS: query a unspent outpoint # + ###################################### + json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + + #check chainTip response + assert_equal(json_obj['chaintipHash'], bb_hash) + + #make sure there is one utxo + assert_equal(len(json_obj['utxos']), 1) + assert_equal(json_obj['utxos'][0]['value'], 0.1) + + + ################################################ + # GETUTXOS: now query a already spent outpoint # + ################################################ + json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+vintx+'","n":0}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + + #check chainTip response + assert_equal(json_obj['chaintipHash'], bb_hash) + + #make sure there is no utox in the response because this oupoint has been spent + assert_equal(len(json_obj['utxos']), 0) + + #check bitmap + assert_equal(json_obj['bitmap'], "0") + + + ################################################## + # GETUTXOS: now check both with the same request # + ################################################## + json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'},{"txid":"'+vintx+'","n":0}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + assert_equal(len(json_obj['utxos']), 1) + assert_equal(json_obj['bitmap'], "10") + + #test binary response bb_hash = self.nodes[0].getbestblockhash() + binaryRequest = b'\x01\x02' + binaryRequest += binascii.unhexlify(txid) + binaryRequest += pack("i", n); + binaryRequest += binascii.unhexlify(vintx); + binaryRequest += pack("i", 0); + + bin_response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest) + + output = StringIO.StringIO() + output.write(bin_response) + output.seek(0) + chainHeight = unpack("i", output.read(4))[0] + hashFromBinResponse = hex(deser_uint256(output))[2:].zfill(65).rstrip("L") + + assert_equal(bb_hash, hashFromBinResponse) #check if getutxo's chaintip during calculation was fine + assert_equal(chainHeight, 102) #chain height must be 102 + + + ############################ + # GETUTXOS: mempool checks # + ############################ + + # do a tx and don't sync + txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) + json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json") + json_obj = json.loads(json_string) + vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then) + # get n of 0.1 outpoint + n = 0 + for vout in json_obj['vout']: + if vout['value'] == 0.1: + n = vout['n'] + + json_request = '{"checkmempool":false,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + assert_equal(len(json_obj['utxos']), 0) #there should be a outpoint because it has just added to the mempool + + json_request = '{"checkmempool":true,"outpoints":[{"txid":"'+txid+'","n":'+str(n)+'}]}' + json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request) + json_obj = json.loads(json_string) + assert_equal(len(json_obj['utxos']), 1) #there should be a outpoint because it has just added to the mempool + + #do some invalid requests + json_request = '{"checkmempool' + response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 500) #must be a 500 because we send a invalid json request + + json_request = '{"checkmempool' + response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True) + assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request + + #test limits + json_request = '{"checkmempool":true,"outpoints":[' + for x in range(0, 200): + json_request += '{"txid":"'+txid+'","n":'+str(n)+'},' + json_request = json_request.rstrip(",") + json_request+="]}"; + response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 500) #must be a 500 because we exceeding the limits + + json_request = '{"checkmempool":true,"outpoints":[' + for x in range(0, 90): + json_request += '{"txid":"'+txid+'","n":'+str(n)+'},' + json_request = json_request.rstrip(",") + json_request+="]}"; + response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 200) #must be a 500 because we exceeding the limits + + self.nodes[0].setgenerate(True, 1) #generate block to not affect upcomming tests + self.sync_all() + + ################ + # /rest/block/ # + ################ + # check binary format - response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True) + response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True) assert_equal(response.status, 200) assert_greater_than(int(response.getheader('content-length')), 80) response_str = response.read() # compare with block header - response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True) + response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True) assert_equal(response_header.status, 200) assert_equal(int(response_header.getheader('content-length')), 80) response_header_str = response_header.read() assert_equal(response_str[0:80], response_header_str) # check block hex format - response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) + response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True) assert_equal(response_hex.status, 200) assert_greater_than(int(response_hex.getheader('content-length')), 160) response_hex_str = response_hex.read() assert_equal(response_str.encode("hex")[0:160], response_hex_str[0:160]) # compare with hex block header - response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) + response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True) assert_equal(response_header_hex.status, 200) assert_greater_than(int(response_header_hex.getheader('content-length')), 160) response_header_hex_str = response_header_hex.read() @@ -77,9 +245,11 @@ class RESTTest (BitcoinTestFramework): assert_equal(json_obj['txid'], tx_hash) # check hex format response - hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", True) + hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", "", True) assert_equal(hex_string.status, 200) assert_greater_than(int(response.getheader('content-length')), 10) + + # check block tx details # let's make 3 tx and mine them on node 1 diff --git a/src/rest.cpp b/src/rest.cpp index adc2d56284..78139682a6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -9,14 +9,18 @@ #include "rpcserver.h" #include "streams.h" #include "sync.h" +#include "txmempool.h" #include "utilstrencodings.h" #include "version.h" #include <boost/algorithm/string.hpp> +#include <boost/dynamic_bitset.hpp> using namespace std; using namespace json_spirit; +static const int MAX_GETUTXOS_OUTPOINTS = 100; //allow a max of 100 outpoints to be queried at once + enum RetFormat { RF_UNDEF, RF_BINARY, @@ -34,6 +38,22 @@ static const struct { {RF_JSON, "json"}, }; +struct CCoin { + uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE + uint32_t nHeight; + CTxOut out; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + { + READWRITE(nTxVer); + READWRITE(nHeight); + READWRITE(out); + } +}; + class RestErr { public: @@ -43,6 +63,7 @@ public: extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry); extern Object blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); +extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, Object& out, bool fIncludeHex); static RestErr RESTERR(enum HTTPStatusCode status, string message) { @@ -90,12 +111,13 @@ static bool ParseHashStr(const string& strReq, uint256& v) } static bool rest_headers(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { vector<string> params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); vector<string> path; boost::split(path, params[0], boost::is_any_of("/")); @@ -153,13 +175,14 @@ static bool rest_headers(AcceptedConnection* conn, } static bool rest_block(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun, bool showTxDetails) { vector<string> params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; @@ -211,28 +234,31 @@ static bool rest_block(AcceptedConnection* conn, } static bool rest_block_extended(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { - return rest_block(conn, strReq, mapHeaders, fRun, true); + return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, true); } static bool rest_block_notxdetails(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { - return rest_block(conn, strReq, mapHeaders, fRun, false); + return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, false); } static bool rest_chaininfo(AcceptedConnection* conn, - const std::string& strReq, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) + const std::string& strURIPart, + const std::string& strRequest, + const std::map<std::string, std::string>& mapHeaders, + bool fRun) { vector<string> params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); switch (rf) { case RF_JSON: { @@ -253,12 +279,13 @@ static bool rest_chaininfo(AcceptedConnection* conn, } static bool rest_tx(AcceptedConnection* conn, - const std::string& strReq, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { vector<string> params; - const RetFormat rf = ParseDataFormat(params, strReq); + const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; @@ -303,10 +330,191 @@ static bool rest_tx(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } +static bool rest_getutxos(AcceptedConnection* conn, + const std::string& strURIPart, + const std::string& strRequest, + const std::map<std::string, std::string>& mapHeaders, + bool fRun) +{ + vector<string> params; + enum RetFormat rf = ParseDataFormat(params, strURIPart); + + // throw exception in case of a empty request + if (strRequest.length() == 0) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + + bool fCheckMemPool = false; + vector<COutPoint> vOutPoints; + + // parse/deserialize input + // input-format = output-format, rest/getutxos/bin requires binary input, gives binary output, ... + + string strRequestMutable = strRequest; //convert const string to string for allowing hex to bin converting + + switch (rf) { + case RF_HEX: { + // convert hex to bin, continue then with bin part + std::vector<unsigned char> strRequestV = ParseHex(strRequest); + strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); + } + + case RF_BINARY: { + try { + //deserialize + CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); + oss << strRequestMutable; + oss >> fCheckMemPool; + oss >> vOutPoints; + } catch (const std::ios_base::failure& e) { + // abort in case of unreadable binary data + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + } + break; + } + + case RF_JSON: { + try { + // parse json request + Value valRequest; + if (!read_string(strRequest, valRequest)) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + + Object jsonObject = valRequest.get_obj(); + const Value& checkMempoolValue = find_value(jsonObject, "checkmempool"); + + if (!checkMempoolValue.is_null()) { + fCheckMemPool = checkMempoolValue.get_bool(); + } + const Value& outpointsValue = find_value(jsonObject, "outpoints"); + if (!outpointsValue.is_null()) { + Array outPoints = outpointsValue.get_array(); + BOOST_FOREACH (const Value& outPoint, outPoints) { + Object outpointObject = outPoint.get_obj(); + uint256 txid = ParseHashO(outpointObject, "txid"); + Value nValue = find_value(outpointObject, "n"); + int nOutput = nValue.get_int(); + vOutPoints.push_back(COutPoint(txid, nOutput)); + } + } + } catch (...) { + // return HTTP 500 if there was a json parsing error + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + } + break; + } + default: { + throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } + + // limit max outpoints + if (vOutPoints.size() > MAX_GETUTXOS_OUTPOINTS) + throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); + + // check spentness and form a bitmap (as well as a JSON capable human-readble string representation) + vector<unsigned char> bitmap; + vector<CCoin> outs; + std::string bitmapStringRepresentation; + boost::dynamic_bitset<unsigned char> hits(vOutPoints.size()); + { + LOCK2(cs_main, mempool.cs); + + CCoinsView viewDummy; + CCoinsViewCache view(&viewDummy); + + CCoinsViewCache& viewChain = *pcoinsTip; + CCoinsViewMemPool viewMempool(&viewChain, mempool); + + if (fCheckMemPool) + view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool + + for (size_t i = 0; i < vOutPoints.size(); i++) { + CCoins coins; + uint256 hash = vOutPoints[i].hash; + if (view.GetCoins(hash, coins)) { + mempool.pruneSpent(hash, coins); + if (coins.IsAvailable(vOutPoints[i].n)) { + hits[i] = true; + // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if + // n is valid but points to an already spent output (IsNull). + CCoin coin; + coin.nTxVer = coins.nVersion; + coin.nHeight = coins.nHeight; + coin.out = coins.vout.at(vOutPoints[i].n); + assert(!coin.out.IsNull()); + outs.push_back(coin); + } + } + + bitmapStringRepresentation.append(hits[i] ? "1" : "0"); // form a binary string representation (human-readable for json output) + } + } + boost::to_block_range(hits, std::back_inserter(bitmap)); + + switch (rf) { + case RF_BINARY: { + // serialize data + // use exact same output as mentioned in Bip64 + CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); + ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; + string ssGetUTXOResponseString = ssGetUTXOResponse.str(); + + conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, ssGetUTXOResponseString.size(), "application/octet-stream") << ssGetUTXOResponseString << std::flush; + return true; + } + + case RF_HEX: { + CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); + ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; + string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; + + conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + return true; + } + + case RF_JSON: { + Object objGetUTXOResponse; + + // pack in some essentials + // use more or less the same output as mentioned in Bip64 + objGetUTXOResponse.push_back(Pair("chainHeight", chainActive.Height())); + objGetUTXOResponse.push_back(Pair("chaintipHash", chainActive.Tip()->GetBlockHash().GetHex())); + objGetUTXOResponse.push_back(Pair("bitmap", bitmapStringRepresentation)); + + Array utxos; + BOOST_FOREACH (const CCoin& coin, outs) { + Object utxo; + utxo.push_back(Pair("txvers", (int32_t)coin.nTxVer)); + utxo.push_back(Pair("height", (int32_t)coin.nHeight)); + utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); + + // include the script in a json output + Object o; + ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + utxo.push_back(Pair("scriptPubKey", o)); + utxos.push_back(utxo); + } + objGetUTXOResponse.push_back(Pair("utxos", utxos)); + + // return json string + string strJSON = write_string(Value(objGetUTXOResponse), false) + "\n"; + conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + return true; + } + default: { + throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } + + // not reached + return true; // continue to process further HTTP reqs on this cxn +} + static const struct { const char* prefix; bool (*handler)(AcceptedConnection* conn, - const std::string& strURI, + const std::string& strURIPart, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun); } uri_prefixes[] = { @@ -315,10 +523,12 @@ static const struct { {"/rest/block/", rest_block_extended}, {"/rest/chaininfo", rest_chaininfo}, {"/rest/headers/", rest_headers}, + {"/rest/getutxos", rest_getutxos}, }; bool HTTPReq_REST(AcceptedConnection* conn, const std::string& strURI, + const string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun) { @@ -330,8 +540,8 @@ bool HTTPReq_REST(AcceptedConnection* conn, for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) { unsigned int plen = strlen(uri_prefixes[i].prefix); if (strURI.substr(0, plen) == uri_prefixes[i].prefix) { - string strReq = strURI.substr(plen); - return uri_prefixes[i].handler(conn, strReq, mapHeaders, fRun); + string strURIPart = strURI.substr(plen); + return uri_prefixes[i].handler(conn, strURIPart, strRequest, mapHeaders, fRun); } } } catch (const RestErr& re) { diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index e2df41fe21..254eea833e 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -992,7 +992,7 @@ void ServiceConnection(AcceptedConnection *conn) // Process via HTTP REST API } else if (strURI.substr(0, 6) == "/rest/" && GetBoolArg("-rest", false)) { - if (!HTTPReq_REST(conn, strURI, mapHeaders, fRun)) + if (!HTTPReq_REST(conn, strURI, strRequest, mapHeaders, fRun)) break; } else { diff --git a/src/rpcserver.h b/src/rpcserver.h index c3200d8c35..8dcad26106 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -237,6 +237,7 @@ extern json_spirit::Value reconsiderblock(const json_spirit::Array& params, bool // in rest.cpp extern bool HTTPReq_REST(AcceptedConnection *conn, const std::string& strURI, + const std::string& strRequest, const std::map<std::string, std::string>& mapHeaders, bool fRun); |