From 40b556d3742a1f65d67e2d4c760d0b13fe8be5b7 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 23 Jan 2015 07:53:17 +0100 Subject: evhttpd implementation - *Replace usage of boost::asio with [libevent2](http://libevent.org/)*. boost::asio is not part of C++11, so unlike other boost there is no forwards-compatibility reason to stick with it. Together with #4738 (convert json_spirit to UniValue), this rids Bitcoin Core of the worst offenders with regard to compile-time slowness. - *Replace spit-and-duct-tape http server with evhttp*. Front-end http handling is handled by libevent, a work queue (with configurable depth and parallelism) is used to handle application requests. - *Wrap HTTP request in C++ class*; this makes the application code mostly HTTP-server-neutral - *Refactor RPC to move all http-specific code to a separate file*. Theoreticaly this can allow building without HTTP server but with another RPC backend, e.g. Qt's debug console (currently not implemented) or future RPC mechanisms people may want to use. - *HTTP dispatch mechanism*; services (e.g., RPC, REST) register which URL paths they want to handle. By using a proven, high-performance asynchronous networking library (also used by Tor) and HTTP server, problems such as #5674, #5655, #344 should be avoided. What works? bitcoind, bitcoin-cli, bitcoin-qt. Unit tests and RPC/REST tests pass. The aim for now is everything but SSL support. Configuration options: - `-rpcthreads`: repurposed as "number of work handler threads". Still defaults to 4. - `-rpcworkqueue`: maximum depth of work queue. When this is reached, new requests will return a 500 Internal Error. - `-rpctimeout`: inactivity time, in seconds, after which to disconnect a client. - `-debug=http`: low-level http activity logging --- src/rest.cpp | 243 +++++++++++++++++++++++++++-------------------------------- 1 file changed, 111 insertions(+), 132 deletions(-) (limited to 'src/rest.cpp') diff --git a/src/rest.cpp b/src/rest.cpp index 74d27e73bb..9405267067 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -7,6 +7,7 @@ #include "primitives/block.h" #include "primitives/transaction.h" #include "main.h" +#include "httpserver.h" #include "rpcserver.h" #include "streams.h" #include "sync.h" @@ -56,13 +57,6 @@ struct CCoin { } }; -class RestErr -{ -public: - enum HTTPStatusCode status; - string message; -}; - extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); extern UniValue mempoolInfoToJSON(); @@ -70,15 +64,14 @@ extern UniValue mempoolToJSON(bool fVerbose = false); extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); -static RestErr RESTERR(enum HTTPStatusCode status, string message) +static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, string message) { - RestErr re; - re.status = status; - re.message = message; - return re; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(status, message + "\r\n"); + return false; } -static enum RetFormat ParseDataFormat(vector& params, const string strReq) +static enum RetFormat ParseDataFormat(vector& params, const string& strReq) { boost::split(params, strReq, boost::is_any_of(".")); if (params.size() > 1) { @@ -115,28 +108,35 @@ static bool ParseHashStr(const string& strReq, uint256& v) return true; } -static bool rest_headers(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool CheckWarmup(HTTPRequest* req) +{ + std::string statusmessage; + if (RPCIsInWarmup(&statusmessage)) + return RESTERR(req, HTTP_SERVICE_UNAVAILABLE, "Service temporarily unavailable: " + statusmessage); + return true; +} + +static bool rest_headers(HTTPRequest* req, + const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); vector path; boost::split(path, params[0], boost::is_any_of("/")); if (path.size() != 2) - throw RESTERR(HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers//.."); + return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers//.."); long count = strtol(path[0].c_str(), NULL, 10); if (count < 1 || count > 2000) - throw RESTERR(HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); + return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); string hashStr = path[1]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); std::vector headers; headers.reserve(count); @@ -160,28 +160,29 @@ static bool rest_headers(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryHeader = ssHeader.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryHeader.size(), "application/octet-stream") << binaryHeader << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryHeader); return true; } case RF_HEX: { string strHex = HexStr(ssHeader.begin(), ssHeader.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } - case RF_JSON: { UniValue jsonHeaders(UniValue::VARR); BOOST_FOREACH(const CBlockIndex *pindex, headers) { jsonHeaders.push_back(blockheaderToJSON(pindex)); } string strJSON = jsonHeaders.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } - default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: .bin, .hex)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex)"); } } @@ -189,34 +190,33 @@ static bool rest_headers(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_block(AcceptedConnection* conn, +static bool rest_block(HTTPRequest* req, const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun, bool showTxDetails) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CBlock block; CBlockIndex* pblockindex = NULL; { LOCK(cs_main); if (mapBlockIndex.count(hash) == 0) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); pblockindex = mapBlockIndex[hash]; if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); if (!ReadBlockFromDisk(block, pblockindex)) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); @@ -225,25 +225,28 @@ static bool rest_block(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryBlock = ssBlock.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryBlock.size(), "application/octet-stream") << binaryBlock << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryBlock); return true; } case RF_HEX: { string strHex = HexStr(ssBlock.begin(), ssBlock.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } case RF_JSON: { UniValue objBlock = blockToJSON(block, pblockindex, showTxDetails); string strJSON = objBlock.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -251,30 +254,20 @@ static bool rest_block(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_block_extended(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_block_extended(HTTPRequest* req, const std::string& strURIPart) { - return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, true); + return rest_block(req, strURIPart, true); } -static bool rest_block_notxdetails(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_block_notxdetails(HTTPRequest* req, const std::string& strURIPart) { - return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, false); + return rest_block(req, strURIPart, false); } -static bool rest_chaininfo(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); @@ -283,11 +276,12 @@ static bool rest_chaininfo(AcceptedConnection* conn, UniValue rpcParams(UniValue::VARR); UniValue chainInfoObject = getblockchaininfo(rpcParams, false); string strJSON = chainInfoObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -295,12 +289,10 @@ static bool rest_chaininfo(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_mempool_info(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); @@ -309,11 +301,12 @@ static bool rest_mempool_info(AcceptedConnection* conn, UniValue mempoolInfoObject = mempoolInfoToJSON(); string strJSON = mempoolInfoObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -321,12 +314,10 @@ static bool rest_mempool_info(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_mempool_contents(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); @@ -335,11 +326,12 @@ static bool rest_mempool_contents(AcceptedConnection* conn, UniValue mempoolObject = mempoolToJSON(true); string strJSON = mempoolObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -347,24 +339,22 @@ static bool rest_mempool_contents(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_tx(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CTransaction tx; uint256 hashBlock = uint256(); if (!GetTransaction(hash, tx, hashBlock, true)) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << tx; @@ -372,13 +362,15 @@ static bool rest_tx(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryTx = ssTx.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryTx.size(), "application/octet-stream") << binaryTx << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryTx); return true; } case RF_HEX: { string strHex = HexStr(ssTx.begin(), ssTx.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } @@ -386,12 +378,13 @@ static bool rest_tx(AcceptedConnection* conn, UniValue objTx(UniValue::VOBJ); TxToJSON(tx, hashBlock, objTx); string strJSON = objTx.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -399,12 +392,10 @@ static bool rest_tx(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_getutxos(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; enum RetFormat rf = ParseDataFormat(params, strURIPart); @@ -416,8 +407,9 @@ static bool rest_getutxos(AcceptedConnection* conn, } // throw exception in case of a empty request - if (strRequest.length() == 0 && uriParts.size() == 0) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + std::string strRequestMutable = req->ReadBody(); + if (strRequestMutable.length() == 0 && uriParts.size() == 0) + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); bool fInputParsed = false; bool fCheckMemPool = false; @@ -441,7 +433,7 @@ static bool rest_getutxos(AcceptedConnection* conn, std::string strOutput = uriParts[i].substr(uriParts[i].find("-")+1); if (!ParseInt32(strOutput, &nOutput) || !IsHex(strTxid)) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Parse error"); txid.SetHex(strTxid); vOutPoints.push_back(COutPoint(txid, (uint32_t)nOutput)); @@ -450,15 +442,13 @@ static bool rest_getutxos(AcceptedConnection* conn, if (vOutPoints.size() > 0) fInputParsed = true; else - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); } - string strRequestMutable = strRequest; //convert const string to string for allowing hex to bin converting - switch (rf) { case RF_HEX: { // convert hex to bin, continue then with bin part - std::vector strRequestV = ParseHex(strRequest); + std::vector strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); } @@ -468,7 +458,7 @@ static bool rest_getutxos(AcceptedConnection* conn, if (strRequestMutable.size() > 0) { if (fInputParsed) //don't allow sending input over URI and HTTP RAW DATA - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Combination of URI scheme inputs and raw post data is not allowed"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Combination of URI scheme inputs and raw post data is not allowed"); CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); oss << strRequestMutable; @@ -477,24 +467,24 @@ static bool rest_getutxos(AcceptedConnection* conn, } } catch (const std::ios_base::failure& e) { // abort in case of unreadable binary data - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Parse error"); } break; } case RF_JSON: { if (!fInputParsed) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); break; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } // limit max outpoints if (vOutPoints.size() > MAX_GETUTXOS_OUTPOINTS) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); // check spentness and form a bitmap (as well as a JSON capable human-readble string representation) vector bitmap; @@ -544,7 +534,8 @@ static bool rest_getutxos(AcceptedConnection* conn, ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; string ssGetUTXOResponseString = ssGetUTXOResponse.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, ssGetUTXOResponseString.size(), "application/octet-stream") << ssGetUTXOResponseString << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, ssGetUTXOResponseString); return true; } @@ -553,7 +544,8 @@ static bool rest_getutxos(AcceptedConnection* conn, ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } @@ -583,11 +575,12 @@ static bool rest_getutxos(AcceptedConnection* conn, // return json string string strJSON = objGetUTXOResponse.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -597,11 +590,7 @@ static bool rest_getutxos(AcceptedConnection* conn, static const struct { const char* prefix; - bool (*handler)(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun); + bool (*handler)(HTTPRequest* req, const std::string& strReq); } uri_prefixes[] = { {"/rest/tx/", rest_tx}, {"/rest/block/notxdetails/", rest_block_notxdetails}, @@ -613,29 +602,19 @@ static const struct { {"/rest/getutxos", rest_getutxos}, }; -bool HTTPReq_REST(AcceptedConnection* conn, - const std::string& strURI, - const string& strRequest, - const std::map& mapHeaders, - bool fRun) +bool StartREST() { - try { - std::string statusmessage; - if (RPCIsInWarmup(&statusmessage)) - throw RESTERR(HTTP_SERVICE_UNAVAILABLE, "Service temporarily unavailable: " + statusmessage); - - for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) { - unsigned int plen = strlen(uri_prefixes[i].prefix); - if (strURI.substr(0, plen) == uri_prefixes[i].prefix) { - string strURIPart = strURI.substr(plen); - return uri_prefixes[i].handler(conn, strURIPart, strRequest, mapHeaders, fRun); - } - } - } catch (const RestErr& re) { - conn->stream() << HTTPReply(re.status, re.message + "\r\n", false, false, "text/plain") << std::flush; - return false; - } + for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) + RegisterHTTPHandler(uri_prefixes[i].prefix, false, uri_prefixes[i].handler); + return true; +} - conn->stream() << HTTPError(HTTP_NOT_FOUND, false) << std::flush; - return false; +void InterruptREST() +{ +} + +void StopREST() +{ + for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) + UnregisterHTTPHandler(uri_prefixes[i].prefix, false); } -- cgit v1.2.3