diff options
Diffstat (limited to 'src')
36 files changed, 701 insertions, 280 deletions
diff --git a/src/addrman.cpp b/src/addrman.cpp index 00f6fe99e0..cebb1c8e5e 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -263,7 +263,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP pinfo->nTime = std::max((int64_t)0, addr.nTime - nTimePenalty); // add services - pinfo->nServices |= addr.nServices; + pinfo->nServices = ServiceFlags(pinfo->nServices | addr.nServices); // do not update if no new information is present if (!addr.nTime || (pinfo->nTime && addr.nTime <= pinfo->nTime)) @@ -502,6 +502,24 @@ void CAddrMan::Connected_(const CService& addr, int64_t nTime) info.nTime = nTime; } +void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) +{ + CAddrInfo* pinfo = Find(addr); + + // if not found, bail out + if (!pinfo) + return; + + CAddrInfo& info = *pinfo; + + // check whether we are talking about the exact same CService (including same port) + if (info != addr) + return; + + // update info + info.nServices = nServices; +} + int CAddrMan::RandomInt(int nMax){ return GetRandInt(nMax); } diff --git a/src/addrman.h b/src/addrman.h index c5923e9417..1caf540758 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -256,6 +256,9 @@ protected: //! Mark an entry as currently-connected-to. void Connected_(const CService &addr, int64_t nTime); + //! Update an entry's service bits. + void SetServices_(const CService &addr, ServiceFlags nServices); + public: /** * serialized format: @@ -589,6 +592,14 @@ public: } } + void SetServices(const CService &addr, ServiceFlags nServices) + { + LOCK(cs); + Check(); + SetServices_(addr, nServices); + Check(); + } + }; #endif // BITCOIN_ADDRMAN_H diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 0005115671..8c27a578bb 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -16,14 +16,6 @@ #include "chainparamsseeds.h" -std::string CDNSSeedData::getHost(uint64_t requiredServiceBits) const { - //use default host for non-filter-capable seeds or if we use the default service bits (NODE_NETWORK) - if (!supportsServiceBitsFiltering || requiredServiceBits == NODE_NETWORK) - return host; - - return strprintf("x%x.%s", requiredServiceBits, host); -} - static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesisOutputScript, uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { CMutableTransaction txNew; diff --git a/src/chainparams.h b/src/chainparams.h index 7168daaf43..638893e9ad 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -13,11 +13,9 @@ #include <vector> -class CDNSSeedData { -public: +struct CDNSSeedData { std::string name, host; bool supportsServiceBitsFiltering; - std::string getHost(uint64_t requiredServiceBits) const; CDNSSeedData(const std::string &strName, const std::string &strHost, bool supportsServiceBitsFilteringIn = false) : name(strName), host(strHost), supportsServiceBitsFiltering(supportsServiceBitsFilteringIn) {} }; diff --git a/src/init.cpp b/src/init.cpp index ec4ce6b6da..b572bfc327 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -479,11 +479,20 @@ std::string HelpMessage(HelpMessageMode mode) std::string LicenseInfo() { + const std::string URL_SOURCE_CODE = "<https://github.com/bitcoin/bitcoin>"; + const std::string URL_WEBSITE = "<https://bitcoincore.org>"; // todo: remove urls from translations on next change return CopyrightHolders(strprintf(_("Copyright (C) %i-%i"), 2009, COPYRIGHT_YEAR) + " ") + "\n" + "\n" + - _("This is experimental software.") + "\n" + + strprintf(_("Please contribute if you find %s useful. " + "Visit %s for further information about the software."), + PACKAGE_NAME, URL_WEBSITE) + + "\n" + + strprintf(_("The source code is available from %s."), + URL_SOURCE_CODE) + "\n" + + "\n" + + _("This is experimental software.") + "\n" + _("Distributed under the MIT software license, see the accompanying file COPYING or <http://www.opensource.org/licenses/mit-license.php>.") + "\n" + "\n" + _("This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit <https://www.openssl.org/> and cryptographic software written by Eric Young and UPnP software written by Thomas Bernard.") + @@ -950,7 +959,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) SetMockTime(GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op if (GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) - nLocalServices |= NODE_BLOOM; + nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); nMaxTipAge = GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); @@ -1361,7 +1370,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // after any wallet rescanning has taken place. if (fPruneMode) { LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); - nLocalServices &= ~NODE_NETWORK; + nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { uiInterface.InitMessage(_("Pruning blockstore...")); PruneAndFlush(); diff --git a/src/main.cpp b/src/main.cpp index 6092e7a12e..bdb3457f8e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3449,8 +3449,9 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state } /** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */ -static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp) +static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) { + if (fNewBlock) *fNewBlock = false; AssertLockHeld(cs_main); CBlockIndex *pindexDummy = NULL; @@ -3479,6 +3480,7 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha if (!fHasMoreWork) return true; // Don't process less-work chains if (fTooFarAhead) return true; // Block height is too high } + if (fNewBlock) *fNewBlock = true; if ((!CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime())) || !ContextualCheckBlock(block, state, pindex->pprev)) { if (state.IsInvalid() && !state.CorruptionPossible()) { @@ -3526,7 +3528,7 @@ static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned } -bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp) +bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp) { { LOCK(cs_main); @@ -3535,9 +3537,11 @@ bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, c // Store to disk CBlockIndex *pindex = NULL; - bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fRequested, dbp); + bool fNewBlock = false; + bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fRequested, dbp, &fNewBlock); if (pindex && pfrom) { mapBlockSource[pindex->GetBlockHash()] = pfrom->GetId(); + if (fNewBlock) pfrom->nLastBlockTime = GetTime(); } CheckBlockIndex(chainparams.GetConsensus()); if (!ret) @@ -4107,7 +4111,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { LOCK(cs_main); CValidationState state; - if (AcceptBlock(block, state, chainparams, NULL, true, dbp)) + if (AcceptBlock(block, state, chainparams, NULL, true, dbp, NULL)) nLoaded++; if (state.IsError()) break; @@ -4140,7 +4144,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB head.ToString()); LOCK(cs_main); CValidationState dummy; - if (AcceptBlock(block, dummy, chainparams, NULL, true, &it->second)) + if (AcceptBlock(block, dummy, chainparams, NULL, true, &it->second, NULL)) { nLoaded++; queue.push_back(block.GetHash()); @@ -4611,7 +4615,22 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CAddress addrMe; CAddress addrFrom; uint64_t nNonce = 1; - vRecv >> pfrom->nVersion >> pfrom->nServices >> nTime >> addrMe; + uint64_t nServiceInt; + vRecv >> pfrom->nVersion >> nServiceInt >> nTime >> addrMe; + pfrom->nServices = ServiceFlags(nServiceInt); + if (!pfrom->fInbound) + { + addrman.SetServices(pfrom->addr, pfrom->nServices); + } + if (pfrom->nServicesExpected & ~pfrom->nServices) + { + LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, pfrom->nServices, pfrom->nServicesExpected); + pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + strprintf("Expected to offer services %08x", pfrom->nServicesExpected)); + pfrom->fDisconnect = true; + return false; + } + if (pfrom->nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version @@ -4772,6 +4791,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { boost::this_thread::interruption_point(); + if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + continue; + if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom->AddAddressKnown(addr); @@ -5040,6 +5062,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, RelayTransaction(tx); vWorkQueue.push_back(inv.hash); + pfrom->nLastTXTime = GetTime(); + LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", pfrom->id, tx.GetHash().ToString(), diff --git a/src/main.h b/src/main.h index 9b99ae7c87..e2bfdfdf6e 100644 --- a/src/main.h +++ b/src/main.h @@ -215,7 +215,7 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals); * @param[out] dbp The already known disk position of pblock, or NULL if not yet stored. * @return True if state.IsValid() */ -bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp); +bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp); /** Check whether enough disk space is available for an incoming block */ bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); /** Open a block file (blk?????.dat) */ diff --git a/src/net.cpp b/src/net.cpp index 173eba57c8..4eca3d75cc 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -71,12 +71,15 @@ namespace { const static std::string NET_MESSAGE_COMMAND_OTHER = "*other*"; +/** Services this node implementation cares about */ +static const ServiceFlags nRelevantServices = NODE_NETWORK; + // // Global state variables // bool fDiscover = true; bool fListen = true; -uint64_t nLocalServices = NODE_NETWORK; +ServiceFlags nLocalServices = NODE_NETWORK; bool fRelayTxes = true; CCriticalSection cs_mapLocalHost; std::map<CNetAddr, LocalServiceInfo> mapLocalHost; @@ -159,7 +162,7 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn { struct in6_addr ip; memcpy(&ip, i->addr, sizeof(ip)); - CAddress addr(CService(ip, i->port)); + CAddress addr(CService(ip, i->port), NODE_NETWORK); addr.nTime = GetTime() - GetRand(nOneWeek) - nOneWeek; vSeedsOut.push_back(addr); } @@ -172,13 +175,12 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn // one by discovery. CAddress GetLocalAddress(const CNetAddr *paddrPeer) { - CAddress ret(CService("0.0.0.0",GetListenPort()),0); + CAddress ret(CService("0.0.0.0",GetListenPort()), NODE_NONE); CService addr; if (GetLocal(addr, paddrPeer)) { - ret = CAddress(addr); + ret = CAddress(addr, nLocalServices); } - ret.nServices = nLocalServices; ret.nTime = GetAdjustedTime(); return ret; } @@ -398,6 +400,26 @@ CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure return NULL; } + if (pszDest && addrConnect.IsValid()) { + // It is possible that we already have a connection to the IP/port pszDest resolved to. + // In that case, drop the connection that was just created, and return the existing CNode instead. + // Also store the name we used to connect in that CNode, so that future FindNode() calls to that + // name catch this early. + CNode* pnode = FindNode((CService)addrConnect); + if (pnode) + { + pnode->AddRef(); + { + LOCK(cs_vNodes); + if (pnode->addrName.empty()) { + pnode->addrName = std::string(pszDest); + } + } + CloseSocket(hSocket); + return pnode; + } + } + addrman.Attempt(addrConnect, fCountFailure); // Add node @@ -409,6 +431,7 @@ CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure vNodes.push_back(pnode); } + pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->nTimeConnected = GetTime(); return pnode; @@ -461,14 +484,14 @@ void CNode::PushVersion() int nBestHeight = GetNodeSignals().GetHeight().get_value_or(0); int64_t nTime = (fInbound ? GetAdjustedTime() : GetTime()); - CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService("0.0.0.0",0))); + CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService("0.0.0.0", 0), addr.nServices)); CAddress addrMe = GetLocalAddress(&addr); GetRandBytes((unsigned char*)&nLocalHostNonce, sizeof(nLocalHostNonce)); if (fLogIPs) LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nBestHeight, addrMe.ToString(), addrYou.ToString(), id); else LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nBestHeight, addrMe.ToString(), id); - PushMessage(NetMsgType::VERSION, PROTOCOL_VERSION, nLocalServices, nTime, addrYou, addrMe, + PushMessage(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalServices, nTime, addrYou, addrMe, nLocalHostNonce, strSubVersion, nBestHeight, ::fRelayTxes); } @@ -838,6 +861,11 @@ struct NodeEvictionCandidate NodeId id; int64_t nTimeConnected; int64_t nMinPingUsecTime; + int64_t nLastBlockTime; + int64_t nLastTXTime; + bool fNetworkNode; + bool fRelayTxes; + bool fBloomFilter; CAddress addr; uint64_t nKeyedNetGroup; }; @@ -854,7 +882,24 @@ static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, cons static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { return a.nKeyedNetGroup < b.nKeyedNetGroup; -}; +} + +static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + // There is a fall-through here because it is common for a node to have many peers which have not yet relayed a block. + if (a.nLastBlockTime != b.nLastBlockTime) return a.nLastBlockTime < b.nLastBlockTime; + if (a.fNetworkNode != b.fNetworkNode) return b.fNetworkNode; + return a.nTimeConnected > b.nTimeConnected; +} + +static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + // There is a fall-through here because it is common for a node to have more than a few peers that have not yet relayed txn. + if (a.nLastTXTime != b.nLastTXTime) return a.nLastTXTime < b.nLastTXTime; + if (a.fRelayTxes != b.fRelayTxes) return b.fRelayTxes; + if (a.fBloomFilter != b.fBloomFilter) return a.fBloomFilter; + return a.nTimeConnected > b.nTimeConnected; +} /** Try to find a connection to evict when the node is full. * Extreme care must be taken to avoid opening the node to attacker @@ -864,7 +909,7 @@ static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvict * to forge. In order to partition a node the attacker must be * simultaneously better at all of them than honest peers. */ -static bool AttemptToEvictConnection(bool fPreferNewConnection) { +static bool AttemptToEvictConnection() { std::vector<NodeEvictionCandidate> vEvictionCandidates; { LOCK(cs_vNodes); @@ -876,7 +921,9 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { continue; if (node->fDisconnect) continue; - NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr, node->nKeyedNetGroup}; + NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, + node->nLastBlockTime, node->nLastTXTime, node->fNetworkNode, + node->fRelayTxes, node->pfilter != NULL, node->addr, node->nKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } } @@ -899,6 +946,20 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { if (vEvictionCandidates.empty()) return false; + // Protect 4 nodes that most recently sent us transactions. + // An attacker cannot manipulate this metric without performing useful work. + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeTXTime); + vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); + + if (vEvictionCandidates.empty()) return false; + + // Protect 4 nodes that most recently sent us blocks. + // An attacker cannot manipulate this metric without performing useful work. + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeBlockTime); + vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); + + if (vEvictionCandidates.empty()) return false; + // Protect the half of the remaining nodes which have been connected the longest. // This replicates the non-eviction implicit behavior, and precludes attacks that start later. std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeTimeConnected); @@ -927,13 +988,6 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { // Reduce to the network group with the most connections vEvictionCandidates = std::move(mapAddrCounts[naMostConnections]); - // Do not disconnect peers if there is only one unprotected connection from their network group. - // This step excessively favors netgroup diversity, and should be removed once more protective criteria are established. - if (vEvictionCandidates.size() <= 1) - // unless we prefer the new connection (for whitelisted peers) - if (!fPreferNewConnection) - return false; - // Disconnect from the network group with the most connections NodeId evicted = vEvictionCandidates.front().id; LOCK(cs_vNodes); @@ -999,7 +1053,7 @@ static void AcceptConnection(const ListenSocket& hListenSocket) { if (nInbound >= nMaxInbound) { - if (!AttemptToEvictConnection(whitelisted)) { + if (!AttemptToEvictConnection()) { // No connection to evict, disconnect the new connection LogPrint("net", "failed to find an eviction candidate - connection dropped (full)\n"); CloseSocket(hSocket); @@ -1412,6 +1466,18 @@ void MapPort(bool) +static std::string GetDNSHost(const CDNSSeedData& data, ServiceFlags* requiredServiceBits) +{ + //use default host for non-filter-capable seeds or if we use the default service bits (NODE_NETWORK) + if (!data.supportsServiceBitsFiltering || *requiredServiceBits == NODE_NETWORK) { + *requiredServiceBits = NODE_NETWORK; + return data.host; + } + + return strprintf("x%x.%s", *requiredServiceBits, data.host); +} + + void ThreadDNSAddressSeed() { // goal: only query DNS seeds if address need is acute @@ -1437,8 +1503,8 @@ void ThreadDNSAddressSeed() } else { std::vector<CNetAddr> vIPs; std::vector<CAddress> vAdd; - uint64_t requiredServiceBits = NODE_NETWORK; - if (LookupHost(seed.getHost(requiredServiceBits).c_str(), vIPs, 0, true)) + ServiceFlags requiredServiceBits = nRelevantServices; + if (LookupHost(GetDNSHost(seed, &requiredServiceBits).c_str(), vIPs, 0, true)) { BOOST_FOREACH(const CNetAddr& ip, vIPs) { @@ -1520,7 +1586,7 @@ void ThreadOpenConnections() ProcessOneShot(); BOOST_FOREACH(const std::string& strAddr, mapMultiArgs["-connect"]) { - CAddress addr; + CAddress addr(CService(), NODE_NONE); OpenNetworkConnection(addr, false, NULL, strAddr.c_str()); for (int i = 0; i < 10 && i < nLoop; i++) { @@ -1592,6 +1658,10 @@ void ThreadOpenConnections() if (IsLimited(addr)) continue; + // only connect to full nodes + if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + continue; + // only consider very recently tried nodes after 30 failed attempts if (nANow - addr.nLastTry < 600 && nTries < 30) continue; @@ -1609,66 +1679,79 @@ void ThreadOpenConnections() } } -void ThreadOpenAddedConnections() +std::vector<AddedNodeInfo> GetAddedNodeInfo() { + std::vector<AddedNodeInfo> ret; + + std::list<std::string> lAddresses(0); { LOCK(cs_vAddedNodes); - vAddedNodes = mapMultiArgs["-addnode"]; + ret.reserve(vAddedNodes.size()); + BOOST_FOREACH(const std::string& strAddNode, vAddedNodes) + lAddresses.push_back(strAddNode); } - if (HaveNameProxy()) { - while(true) { - std::list<std::string> lAddresses(0); - { - LOCK(cs_vAddedNodes); - BOOST_FOREACH(const std::string& strAddNode, vAddedNodes) - lAddresses.push_back(strAddNode); + + // Build a map of all already connected addresses (by IP:port and by name) to inbound/outbound and resolved CService + std::map<CService, bool> mapConnected; + std::map<std::string, std::pair<bool, CService>> mapConnectedByName; + { + LOCK(cs_vNodes); + for (const CNode* pnode : vNodes) { + if (pnode->addr.IsValid()) { + mapConnected[pnode->addr] = pnode->fInbound; } - BOOST_FOREACH(const std::string& strAddNode, lAddresses) { - CAddress addr; - CSemaphoreGrant grant(*semOutbound); - OpenNetworkConnection(addr, false, &grant, strAddNode.c_str()); - MilliSleep(500); + if (!pnode->addrName.empty()) { + mapConnectedByName[pnode->addrName] = std::make_pair(pnode->fInbound, static_cast<const CService&>(pnode->addr)); + } + } + } + + BOOST_FOREACH(const std::string& strAddNode, lAddresses) { + CService service(strAddNode, Params().GetDefaultPort()); + if (service.IsValid()) { + // strAddNode is an IP:port + auto it = mapConnected.find(service); + if (it != mapConnected.end()) { + ret.push_back(AddedNodeInfo{strAddNode, service, true, it->second}); + } else { + ret.push_back(AddedNodeInfo{strAddNode, CService(), false, false}); + } + } else { + // strAddNode is a name + auto it = mapConnectedByName.find(strAddNode); + if (it != mapConnectedByName.end()) { + ret.push_back(AddedNodeInfo{strAddNode, it->second.second, true, it->second.first}); + } else { + ret.push_back(AddedNodeInfo{strAddNode, CService(), false, false}); } - MilliSleep(120000); // Retry every 2 minutes } } + return ret; +} + +void ThreadOpenAddedConnections() +{ + { + LOCK(cs_vAddedNodes); + vAddedNodes = mapMultiArgs["-addnode"]; + } + for (unsigned int i = 0; true; i++) { - std::list<std::string> lAddresses(0); - { - LOCK(cs_vAddedNodes); - BOOST_FOREACH(const std::string& strAddNode, vAddedNodes) - lAddresses.push_back(strAddNode); + std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(); + for (const AddedNodeInfo& info : vInfo) { + if (!info.fConnected) { + CSemaphoreGrant grant(*semOutbound); + // If strAddedNode is an IP/port, decode it immediately, so + // OpenNetworkConnection can detect existing connections to that IP/port. + CService service(info.strAddedNode, Params().GetDefaultPort()); + OpenNetworkConnection(CAddress(service, NODE_NONE), false, &grant, info.strAddedNode.c_str(), false); + MilliSleep(500); + } } - std::list<std::vector<CService> > lservAddressesToAdd(0); - BOOST_FOREACH(const std::string& strAddNode, lAddresses) { - std::vector<CService> vservNode(0); - if(Lookup(strAddNode.c_str(), vservNode, Params().GetDefaultPort(), fNameLookup, 0)) - lservAddressesToAdd.push_back(vservNode); - } - // Attempt to connect to each IP for each addnode entry until at least one is successful per addnode entry - // (keeping in mind that addnode entries can have many IPs if fNameLookup) - { - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - for (std::list<std::vector<CService> >::iterator it = lservAddressesToAdd.begin(); it != lservAddressesToAdd.end(); it++) - BOOST_FOREACH(const CService& addrNode, *(it)) - if (pnode->addr == addrNode) - { - it = lservAddressesToAdd.erase(it); - it--; - break; - } - } - BOOST_FOREACH(std::vector<CService>& vserv, lservAddressesToAdd) - { - CSemaphoreGrant grant(*semOutbound); - OpenNetworkConnection(CAddress(vserv[i % vserv.size()]), false, &grant); - MilliSleep(500); - } MilliSleep(120000); // Retry every 2 minutes } } @@ -2324,7 +2407,8 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa addrKnown(5000, 0.001), filterInventoryKnown(50000, 0.000001) { - nServices = 0; + nServices = NODE_NONE; + nServicesExpected = NODE_NONE; hSocket = hSocketIn; nRecvVersion = INIT_PROTO_VERSION; nLastSend = 0; @@ -2358,6 +2442,8 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa fSentAddr = false; pfilter = new CBloomFilter(); timeLastMempoolReq = 0; + nLastBlockTime = 0; + nLastTXTime = 0; nPingNonceSent = 0; nPingUsecStart = 0; nPingUsecTime = 0; @@ -72,6 +72,8 @@ static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; +static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; + // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban @@ -152,7 +154,7 @@ CAddress GetLocalAddress(const CNetAddr *paddrPeer = NULL); extern bool fDiscover; extern bool fListen; -extern uint64_t nLocalServices; +extern ServiceFlags nLocalServices; extern bool fRelayTxes; extern uint64_t nLocalHostNonce; extern CAddrMan addrman; @@ -186,7 +188,7 @@ class CNodeStats { public: NodeId nodeid; - uint64_t nServices; + ServiceFlags nServices; bool fRelayTxes; int64_t nLastSend; int64_t nLastRecv; @@ -316,7 +318,8 @@ class CNode { public: // socket - uint64_t nServices; + ServiceFlags nServices; + ServiceFlags nServicesExpected; SOCKET hSocket; CDataStream ssSend; size_t nSendSize; // total size of all vSendMsg entries @@ -416,6 +419,11 @@ public: // Last time a "MEMPOOL" request was serviced. std::atomic<int64_t> timeLastMempoolReq; + + // Block and TXN accept times + std::atomic<int64_t> nLastBlockTime; + std::atomic<int64_t> nLastTXTime; + // Ping time measurement: // The pong reply we're expecting, or 0 if no pong expected. uint64_t nPingNonceSent; @@ -815,4 +823,14 @@ public: /** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds); +struct AddedNodeInfo +{ + std::string strAddedNode; + CService resolvedAddress; + bool fConnected; + bool fInbound; +}; + +std::vector<AddedNodeInfo> GetAddedNodeInfo(); + #endif // BITCOIN_NET_H diff --git a/src/netbase.cpp b/src/netbase.cpp index 572ae70871..e2a516986c 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -621,10 +621,10 @@ bool ConnectSocketByName(CService &addr, SOCKET& hSocketRet, const char *pszDest proxyType nameProxy; GetNameProxy(nameProxy); - CService addrResolved; - if (Lookup(strDest.c_str(), addrResolved, port, fNameLookup && !HaveNameProxy())) { - if (addrResolved.IsValid()) { - addr = addrResolved; + std::vector<CService> addrResolved; + if (Lookup(strDest.c_str(), addrResolved, port, fNameLookup && !HaveNameProxy(), 256)) { + if (addrResolved.size() > 0) { + addr = addrResolved[GetRand(addrResolved.size())]; return ConnectSocket(addr, hSocketRet, nTimeout); } } diff --git a/src/protocol.cpp b/src/protocol.cpp index 8c4bd05725..422ef6f636 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -133,7 +133,7 @@ CAddress::CAddress() : CService() Init(); } -CAddress::CAddress(CService ipIn, uint64_t nServicesIn) : CService(ipIn) +CAddress::CAddress(CService ipIn, ServiceFlags nServicesIn) : CService(ipIn) { Init(); nServices = nServicesIn; @@ -141,7 +141,7 @@ CAddress::CAddress(CService ipIn, uint64_t nServicesIn) : CService(ipIn) void CAddress::Init() { - nServices = NODE_NETWORK; + nServices = NODE_NONE; nTime = 100000000; } diff --git a/src/protocol.h b/src/protocol.h index 1b049e52af..ab0a581783 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -223,7 +223,9 @@ extern const char *FEEFILTER; const std::vector<std::string> &getAllNetMessageTypes(); /** nServices flags */ -enum { +enum ServiceFlags : uint64_t { + // Nothing + NODE_NONE = 0, // NODE_NETWORK means that the node is capable of serving the block chain. It is currently // set by all Bitcoin Core nodes, and is unset by SPV clients or other peers that just want // network services but don't provide them. @@ -251,7 +253,7 @@ class CAddress : public CService { public: CAddress(); - explicit CAddress(CService ipIn, uint64_t nServicesIn = NODE_NETWORK); + explicit CAddress(CService ipIn, ServiceFlags nServicesIn); void Init(); @@ -267,13 +269,15 @@ public: if ((nType & SER_DISK) || (nVersion >= CADDR_TIME_VERSION && !(nType & SER_GETHASH))) READWRITE(nTime); - READWRITE(nServices); + uint64_t nServicesInt = nServices; + READWRITE(nServicesInt); + nServices = (ServiceFlags)nServicesInt; READWRITE(*(CService*)this); } // TODO: make private (improves encapsulation) public: - uint64_t nServices; + ServiceFlags nServices; // disk and network only unsigned int nTime; diff --git a/src/qt/forms/receiverequestdialog.ui b/src/qt/forms/receiverequestdialog.ui index 1e484dd9a0..4163f4189c 100644 --- a/src/qt/forms/receiverequestdialog.ui +++ b/src/qt/forms/receiverequestdialog.ui @@ -22,7 +22,7 @@ <property name="minimumSize"> <size> <width>300</width> - <height>300</height> + <height>320</height> </size> </property> <property name="toolTip"> diff --git a/src/qt/guiconstants.h b/src/qt/guiconstants.h index 4b2c10dd48..bab9923d20 100644 --- a/src/qt/guiconstants.h +++ b/src/qt/guiconstants.h @@ -43,7 +43,7 @@ static const int TOOLTIP_WRAP_THRESHOLD = 80; static const int MAX_URI_LENGTH = 255; /* QRCodeDialog -- size of exported QR Code image */ -#define EXPORT_IMAGE_SIZE 256 +#define QR_IMAGE_SIZE 300 /* Number of frames in spinner animation */ #define SPINNER_FRAMES 36 diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index a1e9156eea..b13ea3df70 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -45,7 +45,7 @@ QImage QRImageWidget::exportImage() { if(!pixmap()) return QImage(); - return pixmap()->toImage().scaled(EXPORT_IMAGE_SIZE, EXPORT_IMAGE_SIZE); + return pixmap()->toImage(); } void QRImageWidget::mousePressEvent(QMouseEvent *event) @@ -166,20 +166,32 @@ void ReceiveRequestDialog::update() ui->lblQRCode->setText(tr("Error encoding URI into QR Code.")); return; } - QImage myImage = QImage(code->width + 8, code->width + 8, QImage::Format_RGB32); - myImage.fill(0xffffff); + QImage qrImage = QImage(code->width + 8, code->width + 8, QImage::Format_RGB32); + qrImage.fill(0xffffff); unsigned char *p = code->data; for (int y = 0; y < code->width; y++) { for (int x = 0; x < code->width; x++) { - myImage.setPixel(x + 4, y + 4, ((*p & 1) ? 0x0 : 0xffffff)); + qrImage.setPixel(x + 4, y + 4, ((*p & 1) ? 0x0 : 0xffffff)); p++; } } QRcode_free(code); - ui->lblQRCode->setPixmap(QPixmap::fromImage(myImage).scaled(300, 300)); + QImage qrAddrImage = QImage(QR_IMAGE_SIZE, QR_IMAGE_SIZE+20, QImage::Format_RGB32); + qrAddrImage.fill(0xffffff); + QPainter painter(&qrAddrImage); + painter.drawImage(0, 0, qrImage.scaled(QR_IMAGE_SIZE, QR_IMAGE_SIZE)); + QFont font = GUIUtil::fixedPitchFont(); + font.setPixelSize(12); + painter.setFont(font); + QRect paddedRect = qrAddrImage.rect(); + paddedRect.setHeight(QR_IMAGE_SIZE+12); + painter.drawText(paddedRect, Qt::AlignBottom|Qt::AlignCenter, info.address); + painter.end(); + + ui->lblQRCode->setPixmap(QPixmap::fromImage(qrAddrImage)); ui->btnSaveAs->setEnabled(true); } } diff --git a/src/qt/receiverequestdialog.h b/src/qt/receiverequestdialog.h index 4cab4caff1..676745a858 100644 --- a/src/qt/receiverequestdialog.h +++ b/src/qt/receiverequestdialog.h @@ -10,6 +10,7 @@ #include <QDialog> #include <QImage> #include <QLabel> +#include <QPainter> class OptionsModel; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index cae964e46d..b85c7b2e1a 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -271,25 +271,22 @@ UniValue getaddednodeinfo(const UniValue& params, bool fHelp) { if (fHelp || params.size() < 1 || params.size() > 2) throw runtime_error( - "getaddednodeinfo dns ( \"node\" )\n" + "getaddednodeinfo dummy ( \"node\" )\n" "\nReturns information about the given added node, or all added nodes\n" "(note that onetry addnodes are not listed here)\n" - "If dns is false, only a list of added nodes will be provided,\n" - "otherwise connected information will also be available.\n" "\nArguments:\n" - "1. dns (boolean, required) If false, only a list of added nodes will be provided, otherwise connected information will also be available.\n" + "1. dummy (boolean, required) Kept for historical purposes but ignored\n" "2. \"node\" (string, optional) If provided, return information about this specific node, otherwise all nodes are returned.\n" "\nResult:\n" "[\n" " {\n" - " \"addednode\" : \"192.168.0.201\", (string) The node ip address\n" + " \"addednode\" : \"192.168.0.201\", (string) The node ip address or name (as provided to addnode)\n" " \"connected\" : true|false, (boolean) If connected\n" - " \"addresses\" : [\n" + " \"addresses\" : [ (list of objects) Only when connected = true\n" " {\n" - " \"address\" : \"192.168.0.201:8333\", (string) The bitcoin server host and port\n" + " \"address\" : \"192.168.0.201:8333\", (string) The bitcoin server IP and port we're connected to\n" " \"connected\" : \"outbound\" (string) connection, inbound or outbound\n" " }\n" - " ,...\n" " ]\n" " }\n" " ,...\n" @@ -300,83 +297,35 @@ UniValue getaddednodeinfo(const UniValue& params, bool fHelp) + HelpExampleRpc("getaddednodeinfo", "true, \"192.168.0.201\"") ); - bool fDns = params[0].get_bool(); + std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(); - list<string> laddedNodes(0); - if (params.size() == 1) - { - LOCK(cs_vAddedNodes); - BOOST_FOREACH(const std::string& strAddNode, vAddedNodes) - laddedNodes.push_back(strAddNode); - } - else - { - string strNode = params[1].get_str(); - LOCK(cs_vAddedNodes); - BOOST_FOREACH(const std::string& strAddNode, vAddedNodes) { - if (strAddNode == strNode) - { - laddedNodes.push_back(strAddNode); + if (params.size() == 2) { + bool found = false; + for (const AddedNodeInfo& info : vInfo) { + if (info.strAddedNode == params[1].get_str()) { + vInfo.assign(1, info); + found = true; break; } } - if (laddedNodes.size() == 0) + if (!found) { throw JSONRPCError(RPC_CLIENT_NODE_NOT_ADDED, "Error: Node has not been added."); - } - - UniValue ret(UniValue::VARR); - if (!fDns) - { - BOOST_FOREACH (const std::string& strAddNode, laddedNodes) { - UniValue obj(UniValue::VOBJ); - obj.push_back(Pair("addednode", strAddNode)); - ret.push_back(obj); } - return ret; } - list<pair<string, vector<CService> > > laddedAddreses(0); - BOOST_FOREACH(const std::string& strAddNode, laddedNodes) { - vector<CService> vservNode(0); - if(Lookup(strAddNode.c_str(), vservNode, Params().GetDefaultPort(), fNameLookup, 0)) - laddedAddreses.push_back(make_pair(strAddNode, vservNode)); - else - { - UniValue obj(UniValue::VOBJ); - obj.push_back(Pair("addednode", strAddNode)); - obj.push_back(Pair("connected", false)); - UniValue addresses(UniValue::VARR); - obj.push_back(Pair("addresses", addresses)); - ret.push_back(obj); - } - } + UniValue ret(UniValue::VARR); - LOCK(cs_vNodes); - for (list<pair<string, vector<CService> > >::iterator it = laddedAddreses.begin(); it != laddedAddreses.end(); it++) - { + for (const AddedNodeInfo& info : vInfo) { UniValue obj(UniValue::VOBJ); - obj.push_back(Pair("addednode", it->first)); - + obj.push_back(Pair("addednode", info.strAddedNode)); + obj.push_back(Pair("connected", info.fConnected)); UniValue addresses(UniValue::VARR); - bool fConnected = false; - BOOST_FOREACH(const CService& addrNode, it->second) { - bool fFound = false; - UniValue node(UniValue::VOBJ); - node.push_back(Pair("address", addrNode.ToString())); - BOOST_FOREACH(CNode* pnode, vNodes) { - if (pnode->addr == addrNode) - { - fFound = true; - fConnected = true; - node.push_back(Pair("connected", pnode->fInbound ? "inbound" : "outbound")); - break; - } - } - if (!fFound) - node.push_back(Pair("connected", "false")); - addresses.push_back(node); + if (info.fConnected) { + UniValue address(UniValue::VOBJ); + address.push_back(Pair("address", info.resolvedAddress.ToString())); + address.push_back(Pair("connected", info.fInbound ? "inbound" : "outbound")); + addresses.push_back(address); } - obj.push_back(Pair("connected", fConnected)); obj.push_back(Pair("addresses", addresses)); ret.push_back(obj); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 992914f88c..9723e394d6 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -388,8 +388,13 @@ UniValue createrawtransaction(const UniValue& params, bool fHelp) // set the sequence number if passed in the parameters object const UniValue& sequenceObj = find_value(o, "sequence"); - if (sequenceObj.isNum()) - nSequence = sequenceObj.get_int(); + if (sequenceObj.isNum()) { + int64_t seqNr64 = sequenceObj.get_int64(); + if (seqNr64 < 0 || seqNr64 > std::numeric_limits<uint32_t>::max()) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, sequence number is out of range"); + else + nSequence = (uint32_t)seqNr64; + } CTxIn in(COutPoint(txid, nOutput), CScript(), nSequence); diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 95342498fa..4a373fc60b 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -45,7 +45,7 @@ BOOST_FIXTURE_TEST_SUITE(DoS_tests, TestingSetup) BOOST_AUTO_TEST_CASE(DoS_banning) { CNode::ClearBanned(); - CAddress addr1(ip(0xa0b0c001)); + CAddress addr1(ip(0xa0b0c001), NODE_NONE); CNode dummyNode1(INVALID_SOCKET, addr1, "", true); dummyNode1.nVersion = 1; Misbehaving(dummyNode1.GetId(), 100); // Should get banned @@ -53,7 +53,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) BOOST_CHECK(CNode::IsBanned(addr1)); BOOST_CHECK(!CNode::IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned - CAddress addr2(ip(0xa0b0c002)); + CAddress addr2(ip(0xa0b0c002), NODE_NONE); CNode dummyNode2(INVALID_SOCKET, addr2, "", true); dummyNode2.nVersion = 1; Misbehaving(dummyNode2.GetId(), 50); @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) { CNode::ClearBanned(); mapArgs["-banscore"] = "111"; // because 11 is my favorite number - CAddress addr1(ip(0xa0b0c001)); + CAddress addr1(ip(0xa0b0c001), NODE_NONE); CNode dummyNode1(INVALID_SOCKET, addr1, "", true); dummyNode1.nVersion = 1; Misbehaving(dummyNode1.GetId(), 100); @@ -90,7 +90,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) int64_t nStartTime = GetTime(); SetMockTime(nStartTime); // Overrides future calls to GetTime() - CAddress addr(ip(0xa0b0c001)); + CAddress addr(ip(0xa0b0c001), NODE_NONE); CNode dummyNode(INVALID_SOCKET, addr, "", true); dummyNode.nVersion = 1; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 767b653e47..b6cec24b57 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) // Test 2: Does Addrman::Add work as expected. CService addr1 = CService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); CAddrInfo addr_ret1 = addrman.Select(); BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); @@ -76,14 +76,14 @@ BOOST_AUTO_TEST_CASE(addrman_simple) // Test 3: Does IP address deduplication work correctly. // Expected dup IP should not be added. CService addr1_dup = CService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1_dup), source); + addrman.Add(CAddress(addr1_dup, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); // Test 5: New table has one addr and we add a diff addr we should // have two addrs. CService addr2 = CService("250.1.1.2", 8333); - addrman.Add(CAddress(addr2), source); + addrman.Add(CAddress(addr2, NODE_NONE), source); BOOST_CHECK(addrman.size() == 2); // Test 6: AddrMan::Clear() should empty the new table. @@ -106,18 +106,18 @@ BOOST_AUTO_TEST_CASE(addrman_ports) // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = CService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); CService addr1_port = CService("250.1.1.1", 8334); - addrman.Add(CAddress(addr1_port), source); + addrman.Add(CAddress(addr1_port, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); CAddrInfo addr_ret2 = addrman.Select(); BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333"); // Test 8: Add same IP but diff port to tried table, it doesn't get added. // Perhaps this is not ideal behavior but it is the current behavior. - addrman.Good(CAddress(addr1_port)); + addrman.Good(CAddress(addr1_port, NODE_NONE)); BOOST_CHECK(addrman.size() == 1); bool newOnly = true; CAddrInfo addr_ret3 = addrman.Select(newOnly); @@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE(addrman_select) // Test 9: Select from new with 1 addr in new. CService addr1 = CService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); bool newOnly = true; @@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); // Test 10: move addr to tried, select from new expected nothing returned. - addrman.Good(CAddress(addr1)); + addrman.Good(CAddress(addr1, NODE_NONE)); BOOST_CHECK(addrman.size() == 1); CAddrInfo addr_ret2 = addrman.Select(newOnly); BOOST_CHECK(addr_ret2.ToString() == "[::]:0"); @@ -160,21 +160,21 @@ BOOST_AUTO_TEST_CASE(addrman_select) CService addr3 = CService("250.3.2.2", 9999); CService addr4 = CService("250.3.3.3", 9999); - addrman.Add(CAddress(addr2), CService("250.3.1.1", 8333)); - addrman.Add(CAddress(addr3), CService("250.3.1.1", 8333)); - addrman.Add(CAddress(addr4), CService("250.4.1.1", 8333)); + addrman.Add(CAddress(addr2, NODE_NONE), CService("250.3.1.1", 8333)); + addrman.Add(CAddress(addr3, NODE_NONE), CService("250.3.1.1", 8333)); + addrman.Add(CAddress(addr4, NODE_NONE), CService("250.4.1.1", 8333)); // Add three addresses to tried table. CService addr5 = CService("250.4.4.4", 8333); CService addr6 = CService("250.4.5.5", 7777); CService addr7 = CService("250.4.6.6", 8333); - addrman.Add(CAddress(addr5), CService("250.3.1.1", 8333)); - addrman.Good(CAddress(addr5)); - addrman.Add(CAddress(addr6), CService("250.3.1.1", 8333)); - addrman.Good(CAddress(addr6)); - addrman.Add(CAddress(addr7), CService("250.1.1.3", 8333)); - addrman.Good(CAddress(addr7)); + addrman.Add(CAddress(addr5, NODE_NONE), CService("250.3.1.1", 8333)); + addrman.Good(CAddress(addr5, NODE_NONE)); + addrman.Add(CAddress(addr6, NODE_NONE), CService("250.3.1.1", 8333)); + addrman.Good(CAddress(addr6, NODE_NONE)); + addrman.Add(CAddress(addr7, NODE_NONE), CService("250.1.1.3", 8333)); + addrman.Good(CAddress(addr7, NODE_NONE)); // Test 11: 6 addrs + 1 addr from last test = 7. BOOST_CHECK(addrman.size() == 7); @@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) for (unsigned int i = 1; i < 18; i++) { CService addr = CService("250.1.1." + boost::to_string(i)); - addrman.Add(CAddress(addr), source); + addrman.Add(CAddress(addr, NODE_NONE), source); //Test 13: No collision in new table yet. BOOST_CHECK(addrman.size() == i); @@ -207,11 +207,11 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) //Test 14: new table collision! CService addr1 = CService("250.1.1.18"); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 17); CService addr2 = CService("250.1.1.19"); - addrman.Add(CAddress(addr2), source); + addrman.Add(CAddress(addr2, NODE_NONE), source); BOOST_CHECK(addrman.size() == 18); } @@ -228,8 +228,8 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) for (unsigned int i = 1; i < 80; i++) { CService addr = CService("250.1.1." + boost::to_string(i)); - addrman.Add(CAddress(addr), source); - addrman.Good(CAddress(addr)); + addrman.Add(CAddress(addr, NODE_NONE), source); + addrman.Good(CAddress(addr, NODE_NONE)); //Test 15: No collision in tried table yet. BOOST_TEST_MESSAGE(addrman.size()); @@ -238,11 +238,11 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) //Test 16: tried table collision! CService addr1 = CService("250.1.1.80"); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 79); CService addr2 = CService("250.1.1.81"); - addrman.Add(CAddress(addr2), source); + addrman.Add(CAddress(addr2, NODE_NONE), source); BOOST_CHECK(addrman.size() == 80); } @@ -255,9 +255,9 @@ BOOST_AUTO_TEST_CASE(addrman_find) BOOST_CHECK(addrman.size() == 0); - CAddress addr1 = CAddress(CService("250.1.2.1", 8333)); - CAddress addr2 = CAddress(CService("250.1.2.1", 9999)); - CAddress addr3 = CAddress(CService("251.255.2.1", 8333)); + CAddress addr1 = CAddress(CService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(CService("250.1.2.1", 9999), NODE_NONE); + CAddress addr3 = CAddress(CService("251.255.2.1", 8333), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.2.1"); CNetAddr source2 = CNetAddr("250.1.2.2"); @@ -294,7 +294,7 @@ BOOST_AUTO_TEST_CASE(addrman_create) BOOST_CHECK(addrman.size() == 0); - CAddress addr1 = CAddress(CService("250.1.2.1", 8333)); + CAddress addr1 = CAddress(CService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.2.1"); int nId; @@ -317,7 +317,7 @@ BOOST_AUTO_TEST_CASE(addrman_delete) BOOST_CHECK(addrman.size() == 0); - CAddress addr1 = CAddress(CService("250.1.2.1", 8333)); + CAddress addr1 = CAddress(CService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.2.1"); int nId; @@ -344,15 +344,15 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) vector<CAddress> vAddr1 = addrman.GetAddr(); BOOST_CHECK(vAddr1.size() == 0); - CAddress addr1 = CAddress(CService("250.250.2.1", 8333)); + CAddress addr1 = CAddress(CService("250.250.2.1", 8333), NODE_NONE); addr1.nTime = GetAdjustedTime(); // Set time so isTerrible = false - CAddress addr2 = CAddress(CService("250.251.2.2", 9999)); + CAddress addr2 = CAddress(CService("250.251.2.2", 9999), NODE_NONE); addr2.nTime = GetAdjustedTime(); - CAddress addr3 = CAddress(CService("251.252.2.3", 8333)); + CAddress addr3 = CAddress(CService("251.252.2.3", 8333), NODE_NONE); addr3.nTime = GetAdjustedTime(); - CAddress addr4 = CAddress(CService("252.253.3.4", 8333)); + CAddress addr4 = CAddress(CService("252.253.3.4", 8333), NODE_NONE); addr4.nTime = GetAdjustedTime(); - CAddress addr5 = CAddress(CService("252.254.4.5", 8333)); + CAddress addr5 = CAddress(CService("252.254.4.5", 8333), NODE_NONE); addr5.nTime = GetAdjustedTime(); CNetAddr source1 = CNetAddr("250.1.2.1"); CNetAddr source2 = CNetAddr("250.2.3.3"); @@ -368,8 +368,8 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_CHECK(addrman.GetAddr().size() == 1); // Test 24: Ensure GetAddr works with new and tried addresses. - addrman.Good(CAddress(addr1)); - addrman.Good(CAddress(addr2)); + addrman.Good(CAddress(addr1, NODE_NONE)); + addrman.Good(CAddress(addr2, NODE_NONE)); BOOST_CHECK(addrman.GetAddr().size() == 1); // Test 25: Ensure GetAddr still returns 23% when addrman has many addrs. @@ -378,7 +378,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) int octet2 = (i / 256) % 256; int octet3 = (i / (256 * 2)) % 256; string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + "." + boost::to_string(octet3) + ".23"; - CAddress addr = CAddress(CService(strAddr)); + CAddress addr = CAddress(CService(strAddr), NODE_NONE); // Ensure that for all addrs in addrman, isTerrible == false. addr.nTime = GetAdjustedTime(); @@ -403,8 +403,8 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - CAddress addr1 = CAddress(CService("250.1.1.1", 8333)); - CAddress addr2 = CAddress(CService("250.1.1.1", 9999)); + CAddress addr1 = CAddress(CService("250.1.1.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(CService("250.1.1.1", 9999), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.1.1"); @@ -431,7 +431,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( - CAddress(CService("250.1.1." + boost::to_string(i))), + CAddress(CService("250.1.1." + boost::to_string(i)), NODE_NONE), CNetAddr("250.1.1." + boost::to_string(i))); int bucket = infoi.GetTriedBucket(nKey1); buckets.insert(bucket); @@ -443,7 +443,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) buckets.clear(); for (int j = 0; j < 255; j++) { CAddrInfo infoj = CAddrInfo( - CAddress(CService("250." + boost::to_string(j) + ".1.1")), + CAddress(CService("250." + boost::to_string(j) + ".1.1"), NODE_NONE), CNetAddr("250." + boost::to_string(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1); buckets.insert(bucket); @@ -460,8 +460,8 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - CAddress addr1 = CAddress(CService("250.1.2.1", 8333)); - CAddress addr2 = CAddress(CService("250.1.2.1", 9999)); + CAddress addr1 = CAddress(CService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(CService("250.1.2.1", 9999), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.2.1"); @@ -484,7 +484,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( - CAddress(CService("250.1.1." + boost::to_string(i))), + CAddress(CService("250.1.1." + boost::to_string(i)), NODE_NONE), CNetAddr("250.1.1." + boost::to_string(i))); int bucket = infoi.GetNewBucket(nKey1); buckets.insert(bucket); @@ -497,7 +497,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) for (int j = 0; j < 4 * 255; j++) { CAddrInfo infoj = CAddrInfo(CAddress( CService( - boost::to_string(250 + (j / 255)) + "." + boost::to_string(j % 256) + ".1.1")), + boost::to_string(250 + (j / 255)) + "." + boost::to_string(j % 256) + ".1.1"), NODE_NONE), CNetAddr("251.4.1.1")); int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); @@ -509,7 +509,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int p = 0; p < 255; p++) { CAddrInfo infoj = CAddrInfo( - CAddress(CService("250.1.1.1")), + CAddress(CService("250.1.1.1"), NODE_NONE), CNetAddr("250." + boost::to_string(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b38d61f330..d005d6a163 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -51,7 +51,7 @@ public: int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); s << nUBuckets; - CAddress addr = CAddress(CService("252.1.1.1", 7777)); + CAddress addr = CAddress(CService("252.1.1.1", 7777), NODE_NONE); CAddrInfo info = CAddrInfo(addr, CNetAddr("252.2.2.2")); s << info; } @@ -79,9 +79,9 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) CService addr3 = CService("250.7.3.3", 9999); // Add three addresses to new table. - addrmanUncorrupted.Add(CAddress(addr1), CService("252.5.1.1", 8333)); - addrmanUncorrupted.Add(CAddress(addr2), CService("252.5.1.1", 8333)); - addrmanUncorrupted.Add(CAddress(addr3), CService("252.5.1.1", 8333)); + addrmanUncorrupted.Add(CAddress(addr1, NODE_NONE), CService("252.5.1.1", 8333)); + addrmanUncorrupted.Add(CAddress(addr2, NODE_NONE), CService("252.5.1.1", 8333)); + addrmanUncorrupted.Add(CAddress(addr3, NODE_NONE), CService("252.5.1.1", 8333)); // Test that the de-serialization does not throw an exception. CDataStream ssPeers1 = AddrmanToStream(addrmanUncorrupted); diff --git a/src/univalue/Makefile.am b/src/univalue/Makefile.am index 34fe9e3f13..6c1ec81e63 100644 --- a/src/univalue/Makefile.am +++ b/src/univalue/Makefile.am @@ -3,7 +3,7 @@ ACLOCAL_AMFLAGS = -I build-aux/m4 .INTERMEDIATE: $(GENBIN) include_HEADERS = include/univalue.h -noinst_HEADERS = lib/univalue_escapes.h +noinst_HEADERS = lib/univalue_escapes.h lib/univalue_utffilter.h lib_LTLIBRARIES = libunivalue.la @@ -73,6 +73,10 @@ TEST_FILES = \ $(TEST_DATA_DIR)/fail35.json \ $(TEST_DATA_DIR)/fail36.json \ $(TEST_DATA_DIR)/fail37.json \ + $(TEST_DATA_DIR)/fail38.json \ + $(TEST_DATA_DIR)/fail39.json \ + $(TEST_DATA_DIR)/fail40.json \ + $(TEST_DATA_DIR)/fail41.json \ $(TEST_DATA_DIR)/fail3.json \ $(TEST_DATA_DIR)/fail4.json \ $(TEST_DATA_DIR)/fail5.json \ @@ -83,6 +87,7 @@ TEST_FILES = \ $(TEST_DATA_DIR)/pass1.json \ $(TEST_DATA_DIR)/pass2.json \ $(TEST_DATA_DIR)/pass3.json \ - $(TEST_DATA_DIR)/round1.json + $(TEST_DATA_DIR)/round1.json \ + $(TEST_DATA_DIR)/round2.json EXTRA_DIST=$(TEST_FILES) $(GEN_SRCS) diff --git a/src/univalue/configure.ac b/src/univalue/configure.ac index 0515b632bd..93d3ba945d 100644 --- a/src/univalue/configure.ac +++ b/src/univalue/configure.ac @@ -1,7 +1,7 @@ m4_define([libunivalue_major_version], [1]) m4_define([libunivalue_minor_version], [1]) -m4_define([libunivalue_micro_version], [1]) -m4_define([libunivalue_interface_age], [1]) +m4_define([libunivalue_micro_version], [2]) +m4_define([libunivalue_interface_age], [2]) # If you need a modifier for the version number. # Normally empty, but can be used to make "fixup" releases. m4_define([libunivalue_extraversion], []) @@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()]) -AC_INIT([univalue], [1.0.1], +AC_INIT([univalue], [1.0.2], [http://github.com/jgarzik/univalue/]) dnl make the compilation flags quiet unless V=1 is used diff --git a/src/univalue/lib/univalue_read.cpp b/src/univalue/lib/univalue_read.cpp index c7516b9628..95bac6958d 100644 --- a/src/univalue/lib/univalue_read.cpp +++ b/src/univalue/lib/univalue_read.cpp @@ -6,6 +6,7 @@ #include <vector> #include <stdio.h> #include "univalue.h" +#include "univalue_utffilter.h" using namespace std; @@ -174,41 +175,31 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, raw++; // skip " string valStr; + JSONUTF8StringFilter writer(valStr); while (*raw) { - if (*raw < 0x20) + if ((unsigned char)*raw < 0x20) return JTOK_ERR; else if (*raw == '\\') { raw++; // skip backslash switch (*raw) { - case '"': valStr += "\""; break; - case '\\': valStr += "\\"; break; - case '/': valStr += "/"; break; - case 'b': valStr += "\b"; break; - case 'f': valStr += "\f"; break; - case 'n': valStr += "\n"; break; - case 'r': valStr += "\r"; break; - case 't': valStr += "\t"; break; + case '"': writer.push_back('\"'); break; + case '\\': writer.push_back('\\'); break; + case '/': writer.push_back('/'); break; + case 'b': writer.push_back('\b'); break; + case 'f': writer.push_back('\f'); break; + case 'n': writer.push_back('\n'); break; + case 'r': writer.push_back('\r'); break; + case 't': writer.push_back('\t'); break; case 'u': { unsigned int codepoint; if (hatoui(raw + 1, raw + 1 + 4, codepoint) != raw + 1 + 4) return JTOK_ERR; - - if (codepoint <= 0x7f) - valStr.push_back((char)codepoint); - else if (codepoint <= 0x7FF) { - valStr.push_back((char)(0xC0 | (codepoint >> 6))); - valStr.push_back((char)(0x80 | (codepoint & 0x3F))); - } else if (codepoint <= 0xFFFF) { - valStr.push_back((char)(0xE0 | (codepoint >> 12))); - valStr.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); - valStr.push_back((char)(0x80 | (codepoint & 0x3F))); - } - + writer.push_back_u(codepoint); raw += 4; break; } @@ -226,11 +217,13 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, } else { - valStr += *raw; + writer.push_back(*raw); raw++; } } + if (!writer.finalize()) + return JTOK_ERR; tokenVal = valStr; consumed = (raw - rawStart); return JTOK_STRING; diff --git a/src/univalue/lib/univalue_utffilter.h b/src/univalue/lib/univalue_utffilter.h new file mode 100644 index 0000000000..0e330dce9c --- /dev/null +++ b/src/univalue/lib/univalue_utffilter.h @@ -0,0 +1,119 @@ +// Copyright 2016 Wladimir J. van der Laan +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef UNIVALUE_UTFFILTER_H +#define UNIVALUE_UTFFILTER_H + +#include <string> + +/** + * Filter that generates and validates UTF-8, as well as collates UTF-16 + * surrogate pairs as specified in RFC4627. + */ +class JSONUTF8StringFilter +{ +public: + JSONUTF8StringFilter(std::string &s): + str(s), is_valid(true), codepoint(0), state(0), surpair(0) + { + } + // Write single 8-bit char (may be part of UTF-8 sequence) + void push_back(unsigned char ch) + { + if (state == 0) { + if (ch < 0x80) // 7-bit ASCII, fast direct pass-through + str.push_back(ch); + else if (ch < 0xc0) // Mid-sequence character, invalid in this state + is_valid = false; + else if (ch < 0xe0) { // Start of 2-byte sequence + codepoint = (ch & 0x1f) << 6; + state = 6; + } else if (ch < 0xf0) { // Start of 3-byte sequence + codepoint = (ch & 0x0f) << 12; + state = 12; + } else if (ch < 0xf8) { // Start of 4-byte sequence + codepoint = (ch & 0x07) << 18; + state = 18; + } else // Reserved, invalid + is_valid = false; + } else { + if ((ch & 0xc0) != 0x80) // Not a continuation, invalid + is_valid = false; + state -= 6; + codepoint |= (ch & 0x3f) << state; + if (state == 0) + push_back_u(codepoint); + } + } + // Write codepoint directly, possibly collating surrogate pairs + void push_back_u(unsigned int codepoint) + { + if (state) // Only accept full codepoints in open state + is_valid = false; + if (codepoint >= 0xD800 && codepoint < 0xDC00) { // First half of surrogate pair + if (surpair) // Two subsequent surrogate pair openers - fail + is_valid = false; + else + surpair = codepoint; + } else if (codepoint >= 0xDC00 && codepoint < 0xE000) { // Second half of surrogate pair + if (surpair) { // Open surrogate pair, expect second half + // Compute code point from UTF-16 surrogate pair + append_codepoint(0x10000 | ((surpair - 0xD800)<<10) | (codepoint - 0xDC00)); + surpair = 0; + } else // Second half doesn't follow a first half - fail + is_valid = false; + } else { + if (surpair) // First half of surrogate pair not followed by second - fail + is_valid = false; + else + append_codepoint(codepoint); + } + } + // Check that we're in a state where the string can be ended + // No open sequences, no open surrogate pairs, etc + bool finalize() + { + if (state || surpair) + is_valid = false; + return is_valid; + } +private: + std::string &str; + bool is_valid; + // Current UTF-8 decoding state + unsigned int codepoint; + int state; // Top bit to be filled in for next UTF-8 byte, or 0 + + // Keep track of the following state to handle the following section of + // RFC4627: + // + // To escape an extended character that is not in the Basic Multilingual + // Plane, the character is represented as a twelve-character sequence, + // encoding the UTF-16 surrogate pair. So, for example, a string + // containing only the G clef character (U+1D11E) may be represented as + // "\uD834\uDD1E". + // + // Two subsequent \u.... may have to be replaced with one actual codepoint. + unsigned int surpair; // First half of open UTF-16 surrogate pair, or 0 + + void append_codepoint(unsigned int codepoint) + { + if (codepoint <= 0x7f) + str.push_back((char)codepoint); + else if (codepoint <= 0x7FF) { + str.push_back((char)(0xC0 | (codepoint >> 6))); + str.push_back((char)(0x80 | (codepoint & 0x3F))); + } else if (codepoint <= 0xFFFF) { + str.push_back((char)(0xE0 | (codepoint >> 12))); + str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); + str.push_back((char)(0x80 | (codepoint & 0x3F))); + } else if (codepoint <= 0x1FFFFF) { + str.push_back((char)(0xF0 | (codepoint >> 18))); + str.push_back((char)(0x80 | ((codepoint >> 12) & 0x3F))); + str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); + str.push_back((char)(0x80 | (codepoint & 0x3F))); + } + } +}; + +#endif diff --git a/src/univalue/lib/univalue_write.cpp b/src/univalue/lib/univalue_write.cpp index ceb4cc9166..cfbdad3284 100644 --- a/src/univalue/lib/univalue_write.cpp +++ b/src/univalue/lib/univalue_write.cpp @@ -8,8 +8,6 @@ #include "univalue.h" #include "univalue_escapes.h" -// TODO: Using UTF8 - using namespace std; static string json_escape(const string& inS) @@ -23,15 +21,8 @@ static string json_escape(const string& inS) if (escStr) outS += escStr; - - else if (ch < 0x80) + else outS += ch; - - else { // TODO handle UTF-8 properly - char tmpesc[16]; - sprintf(tmpesc, "\\u%04x", ch); - outS += tmpesc; - } } return outS; diff --git a/src/univalue/test/fail38.json b/src/univalue/test/fail38.json new file mode 100644 index 0000000000..b245e2e46c --- /dev/null +++ b/src/univalue/test/fail38.json @@ -0,0 +1 @@ +["\ud834"] diff --git a/src/univalue/test/fail39.json b/src/univalue/test/fail39.json new file mode 100644 index 0000000000..7c9e263f27 --- /dev/null +++ b/src/univalue/test/fail39.json @@ -0,0 +1 @@ +["\udd61"] diff --git a/src/univalue/test/fail40.json b/src/univalue/test/fail40.json new file mode 100644 index 0000000000..664dc9e245 --- /dev/null +++ b/src/univalue/test/fail40.json @@ -0,0 +1 @@ +["
ก"]
\ No newline at end of file diff --git a/src/univalue/test/fail41.json b/src/univalue/test/fail41.json new file mode 100644 index 0000000000..0de342a2b5 --- /dev/null +++ b/src/univalue/test/fail41.json @@ -0,0 +1 @@ +["๐
"]
\ No newline at end of file diff --git a/src/univalue/test/round2.json b/src/univalue/test/round2.json new file mode 100644 index 0000000000..b766cccc68 --- /dev/null +++ b/src/univalue/test/round2.json @@ -0,0 +1 @@ +["aยงโ ๐๐
ก"] diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 5a052fe92c..05f3842cd1 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -22,6 +22,7 @@ string srcdir(JSON_TEST_SRC); static bool test_failed = false; #define d_assert(expr) { if (!(expr)) { test_failed = true; fprintf(stderr, "%s failed\n", filename.c_str()); } } +#define f_assert(expr) { if (!(expr)) { test_failed = true; fprintf(stderr, "%s failed\n", __func__); } } static std::string rtrim(std::string s) { @@ -108,6 +109,10 @@ static const char *filenames[] = { "fail35.json", "fail36.json", "fail37.json", + "fail38.json", // invalid unicode: only first half of surrogate pair + "fail39.json", // invalid unicode: only second half of surrogate pair + "fail40.json", // invalid unicode: broken UTF-8 + "fail41.json", // invalid unicode: unfinished UTF-8 "fail3.json", "fail4.json", // extra comma "fail5.json", @@ -119,14 +124,40 @@ static const char *filenames[] = { "pass2.json", "pass3.json", "round1.json", // round-trip test + "round2.json", // unicode }; +// Test \u handling +void unescape_unicode_test() +{ + UniValue val; + bool testResult; + // Escaped ASCII (quote) + testResult = val.read("[\"\\u0022\"]"); + f_assert(testResult); + f_assert(val[0].get_str() == "\""); + // Escaped Basic Plane character, two-byte UTF-8 + testResult = val.read("[\"\\u0191\"]"); + f_assert(testResult); + f_assert(val[0].get_str() == "\xc6\x91"); + // Escaped Basic Plane character, three-byte UTF-8 + testResult = val.read("[\"\\u2191\"]"); + f_assert(testResult); + f_assert(val[0].get_str() == "\xe2\x86\x91"); + // Escaped Supplementary Plane character U+1d161 + testResult = val.read("[\"\\ud834\\udd61\"]"); + f_assert(testResult); + f_assert(val[0].get_str() == "\xf0\x9d\x85\xa1"); +} + int main (int argc, char *argv[]) { for (unsigned int fidx = 0; fidx < ARRAY_SIZE(filenames); fidx++) { runtest_file(filenames[fidx]); } + unescape_unicode_test(); + return test_failed ? 1 : 0; } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 9faf21591f..723b2eceff 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -42,6 +42,7 @@ bool bSpendZeroConfChange = DEFAULT_SPEND_ZEROCONF_CHANGE; bool fSendFreeTransactions = DEFAULT_SEND_FREE_TRANSACTIONS; const char * DEFAULT_WALLET_DAT = "wallet.dat"; +const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; /** * Fees smaller than this (in satoshi) are considered zero fee (for transaction creation) @@ -91,7 +92,51 @@ CPubKey CWallet::GenerateNewKey() bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets CKey secret; - secret.MakeNewKey(fCompressed); + + // Create new metadata + int64_t nCreationTime = GetTime(); + CKeyMetadata metadata(nCreationTime); + + // use HD key derivation if HD was enabled during wallet creation + if (!hdChain.masterKeyID.IsNull()) { + // for now we use a fixed keypath scheme of m/0'/0'/k + CKey key; //master key seed (256bit) + CExtKey masterKey; //hd master key + CExtKey accountKey; //key at m/0' + CExtKey externalChainChildKey; //key at m/0'/0' + CExtKey childKey; //key at m/0'/0'/<n>' + + // try to get the master key + if (!GetKey(hdChain.masterKeyID, key)) + throw std::runtime_error("CWallet::GenerateNewKey(): Master key not found"); + + masterKey.SetMaster(key.begin(), key.size()); + + // derive m/0' + // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) + masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); + + // derive m/0'/0' + accountKey.Derive(externalChainChildKey, BIP32_HARDENED_KEY_LIMIT); + + // derive child key at next index, skip keys already known to the wallet + do + { + // always derive hardened keys + // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range + // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 + externalChainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + // increment childkey index + hdChain.nExternalChainCounter++; + } while(HaveKey(childKey.key.GetPubKey().GetID())); + secret = childKey.key; + + // update the chain model in the database + if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) + throw std::runtime_error("CWallet::GenerateNewKey(): Writing HD chain model failed"); + } else { + secret.MakeNewKey(fCompressed); + } // Compressed public keys were introduced in version 0.6.0 if (fCompressed) @@ -100,9 +145,7 @@ CPubKey CWallet::GenerateNewKey() CPubKey pubkey = secret.GetPubKey(); assert(secret.VerifyPubKey(pubkey)); - // Create new metadata - int64_t nCreationTime = GetTime(); - mapKeyMetadata[pubkey.GetID()] = CKeyMetadata(nCreationTime); + mapKeyMetadata[pubkey.GetID()] = metadata; if (!nTimeFirstKey || nCreationTime < nTimeFirstKey) nTimeFirstKey = nCreationTime; @@ -1121,6 +1164,37 @@ CAmount CWallet::GetChange(const CTransaction& tx) const return nChange; } +bool CWallet::SetHDMasterKey(const CKey& key) +{ + LOCK(cs_wallet); + + // store the key as normal "key"/"ckey" object + // in the database + // key metadata is not required + CPubKey pubkey = key.GetPubKey(); + if (!AddKeyPubKey(key, pubkey)) + throw std::runtime_error("CWallet::GenerateNewKey(): AddKey failed"); + + // store the keyid (hash160) together with + // the child index counter in the database + // as a hdchain object + CHDChain newHdChain; + newHdChain.masterKeyID = pubkey.GetID(); + SetHDChain(newHdChain, false); + + return true; +} + +bool CWallet::SetHDChain(const CHDChain& chain, bool memonly) +{ + LOCK(cs_wallet); + if (!memonly && !CWalletDB(strWalletFile).WriteHDChain(chain)) + throw runtime_error("AddHDChain(): writing chain failed"); + + hdChain = chain; + return true; +} + int64_t CWalletTx::GetTxTime() const { int64_t n = nTimeSmart; @@ -3135,6 +3209,7 @@ std::string CWallet::GetWalletHelpString(bool showDebug) strUsage += HelpMessageOpt("-sendfreetransactions", strprintf(_("Send transactions as zero-fee transactions if possible (default: %u)"), DEFAULT_SEND_FREE_TRANSACTIONS)); strUsage += HelpMessageOpt("-spendzeroconfchange", strprintf(_("Spend unconfirmed change when sending transactions (default: %u)"), DEFAULT_SPEND_ZEROCONF_CHANGE)); strUsage += HelpMessageOpt("-txconfirmtarget=<n>", strprintf(_("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)"), DEFAULT_TX_CONFIRM_TARGET)); + strUsage += HelpMessageOpt("-usehd", _("Use hierarchical deterministic key generation (HD) after bip32. Only has effect during wallet creation/first start") + " " + strprintf(_("(default: %u)"), DEFAULT_USE_HD_WALLET)); strUsage += HelpMessageOpt("-upgradewallet", _("Upgrade wallet to latest format on startup")); strUsage += HelpMessageOpt("-wallet=<file>", _("Specify wallet file (within data directory)") + " " + strprintf(_("(default: %s)"), DEFAULT_WALLET_DAT)); strUsage += HelpMessageOpt("-walletbroadcast", _("Make the wallet broadcast transactions") + " " + strprintf(_("(default: %u)"), DEFAULT_WALLETBROADCAST)); @@ -3222,6 +3297,13 @@ bool CWallet::InitLoadWallet() if (fFirstRun) { // Create new keyUser and set as default key + if (GetBoolArg("-usehd", DEFAULT_USE_HD_WALLET)) { + // generate a new master key + CKey key; + key.MakeNewKey(true); + if (!walletInstance->SetHDMasterKey(key)) + throw std::runtime_error("CWallet::GenerateNewKey(): Storing master key failed"); + } CPubKey newDefaultKey; if (walletInstance->GetKeyFromPool(newDefaultKey)) { walletInstance->SetDefaultKey(newDefaultKey); @@ -3231,6 +3313,13 @@ bool CWallet::InitLoadWallet() walletInstance->SetBestChain(chainActive.GetLocator()); } + else if (mapArgs.count("-usehd")) { + bool useHD = GetBoolArg("-usehd", DEFAULT_USE_HD_WALLET); + if (!walletInstance->hdChain.masterKeyID.IsNull() && !useHD) + return InitError(strprintf(_("Error loading %s: You can't disable HD on a already existing HD wallet"), walletFile)); + if (walletInstance->hdChain.masterKeyID.IsNull() && useHD) + return InitError(strprintf(_("Error loading %s: You can't enable HD on a already existing non-HD wallet"), walletFile)); + } LogPrintf(" wallet %15dms\n", GetTimeMillis() - nStart); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 683c901444..7fc6ce5de5 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -57,6 +57,9 @@ static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 2; static const unsigned int MAX_FREE_TRANSACTION_CREATE_SIZE = 1000; static const bool DEFAULT_WALLETBROADCAST = true; +//! if set, all keys will be derived by using BIP32 +static const bool DEFAULT_USE_HD_WALLET = true; + extern const char * DEFAULT_WALLET_DAT; class CBlockIndex; @@ -574,6 +577,9 @@ private: void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>); + /* the hd chain data model (external chain counters) */ + CHDChain hdChain; + public: /* * Main wallet lock. @@ -889,6 +895,12 @@ public: static bool ParameterInteraction(); bool BackupWallet(const std::string& strDest); + + /* Set the hd chain model (chain child index counters) */ + bool SetHDChain(const CHDChain& chain, bool memonly); + + /* Set the current hd master key (will reset the chain child index counters) */ + bool SetHDMasterKey(const CKey& key); }; /** A key allocated from the key pool. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index b5037c9a65..7bfd490950 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -599,6 +599,16 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, return false; } } + else if (strType == "hdchain") + { + CHDChain chain; + ssValue >> chain; + if (!pwallet->SetHDChain(chain, true)) + { + strErr = "Error reading wallet database: SetHDChain failed"; + return false; + } + } } catch (...) { return false; @@ -1003,3 +1013,10 @@ bool CWalletDB::EraseDestData(const std::string &address, const std::string &key nWalletDBUpdated++; return Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); } + + +bool CWalletDB::WriteHDChain(const CHDChain& chain) +{ + nWalletDBUpdated++; + return Write(std::string("hdchain"), chain); +} diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 00c10ea70f..71b0ff26db 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -40,6 +40,35 @@ enum DBErrors DB_NEED_REWRITE }; +/* simple hd chain data model */ +class CHDChain +{ +public: + uint32_t nExternalChainCounter; + CKeyID masterKeyID; //!< master key hash160 + + static const int CURRENT_VERSION = 1; + int nVersion; + + CHDChain() { SetNull(); } + ADD_SERIALIZE_METHODS; + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + { + READWRITE(this->nVersion); + nVersion = this->nVersion; + READWRITE(nExternalChainCounter); + READWRITE(masterKeyID); + } + + void SetNull() + { + nVersion = CHDChain::CURRENT_VERSION; + nExternalChainCounter = 0; + masterKeyID.SetNull(); + } +}; + class CKeyMetadata { public: @@ -134,6 +163,9 @@ public: static bool Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys); static bool Recover(CDBEnv& dbenv, const std::string& filename); + //! write the hdchain model (external chain child index counter) + bool WriteHDChain(const CHDChain& chain); + private: CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); |