diff options
author | Gleb Naumenko <naumenko.gs@gmail.com> | 2019-12-24 13:18:44 -0500 |
---|---|---|
committer | Gleb Naumenko <naumenko.gs@gmail.com> | 2019-12-25 08:59:08 -0500 |
commit | ec45646de9e62b3d42c85716bfeb06d8f2b507dc (patch) | |
tree | 2d3717818d8ea1b9a53f9b53af2390f7aee9ac21 /src | |
parent | 8feb4e4b667361bf23344149c01594abebd56fdb (diff) |
Integrate ASN bucketing in Addrman and add tests
Instead of using /16 netgroups to bucket nodes in Addrman for connection
diversification, ASN, which better represents an actor in terms
of network-layer infrastructure, is used.
For testing, asmap.raw is used. It represents a minimal
asmap needed for testing purposes.
Diffstat (limited to 'src')
-rw-r--r-- | src/Makefile.test.include | 12 | ||||
-rw-r--r-- | src/addrman.cpp | 47 | ||||
-rw-r--r-- | src/addrman.h | 101 | ||||
-rw-r--r-- | src/init.cpp | 26 | ||||
-rw-r--r-- | src/net.cpp | 6 | ||||
-rw-r--r-- | src/net.h | 3 | ||||
-rw-r--r-- | src/netaddress.cpp | 24 | ||||
-rw-r--r-- | src/netaddress.h | 3 | ||||
-rw-r--r-- | src/test/addrman_tests.cpp | 301 | ||||
-rw-r--r-- | src/test/data/asmap.raw | bin | 0 -> 59 bytes | |||
-rw-r--r-- | src/test/netbase_tests.cpp | 28 |
11 files changed, 473 insertions, 78 deletions
diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b4ca01b1fe..f3fc6c706b 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -53,7 +53,8 @@ JSON_TEST_FILES = \ test/data/tx_invalid.json \ test/data/tx_valid.json -RAW_TEST_FILES = +RAW_TEST_FILES = \ + test/data/asmap.raw GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) @@ -429,3 +430,12 @@ endif echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" + +%.raw.h: %.raw + @$(MKDIR_P) $(@D) + @{ \ + echo "static unsigned const char $(*F)_raw[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "};"; \ + } > "$@.new" && mv -f "$@.new" "$@" + @echo "Generated $@" diff --git a/src/addrman.cpp b/src/addrman.cpp index 32676f8fa5..2516c6dc7c 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -8,17 +8,17 @@ #include <hash.h> #include <serialize.h> -int CAddrInfo::GetTriedBucket(const uint256& nKey) const +int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src) const +int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool> &asmap) const { - std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetCheapHash(); + std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } @@ -153,7 +153,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -169,7 +169,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey); + int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -233,7 +233,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime return; // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey); + int tried_bucket = info.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); // Will moving this address into tried evict another entry? @@ -301,7 +301,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP fNew = true; } - int nUBucket = pinfo->GetNewBucket(nKey, source); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); if (vvNew[nUBucket][nUBucketPos] != nId) { bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; @@ -439,7 +439,7 @@ int CAddrMan::Check_() if (vvTried[n][i] != -1) { if (!setTried.count(vvTried[n][i])) return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey) != n) + if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) return -17; if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) return -18; @@ -545,7 +545,7 @@ void CAddrMan::ResolveCollisions_() CAddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey); + int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -609,10 +609,33 @@ CAddrInfo CAddrMan::SelectTriedCollision_() CAddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); int id_old = vvTried[tried_bucket][tried_bucket_pos]; return mapInfo[id_old]; } + +std::vector<bool> CAddrMan::DecodeAsmap(fs::path path) +{ + std::vector<bool> bits; + FILE *filestr = fsbridge::fopen(path, "rb"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open asmap file from disk.\n"); + return bits; + } + fseek(filestr, 0, SEEK_END); + int length = ftell(filestr); + LogPrintf("Opened asmap file %s (%d bytes) from disk.\n", path, length); + fseek(filestr, 0, SEEK_SET); + char cur_byte; + for (int i = 0; i < length; ++i) { + file >> cur_byte; + for (int bit = 0; bit < 8; ++bit) { + bits.push_back((cur_byte >> bit) & 1); + } + } + return bits; +} diff --git a/src/addrman.h b/src/addrman.h index e54184ce35..40ecef4625 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -12,11 +12,17 @@ #include <sync.h> #include <timedata.h> #include <util/system.h> +#include <clientversion.h> #include <map> #include <set> #include <stdint.h> #include <vector> +#include <iostream> +#include <streams.h> +#include <fs.h> +#include <hash.h> + /** * Extended statistics about a CAddress @@ -72,15 +78,15 @@ public: } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey) const; + int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src) const; + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey) const + int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const { - return GetNewBucket(nKey, source); + return GetNewBucket(nKey, source, asmap); } //! Calculate in which position of a bucket to store this entry. @@ -174,6 +180,7 @@ static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes */ class CAddrMan { +friend class CAddrManTest; protected: //! critical section to protect the inner data structures mutable CCriticalSection cs; @@ -268,9 +275,29 @@ protected: void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); public: + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + std::vector<bool> m_asmap; + + // Read asmap from provided binary file + static std::vector<bool> DecodeAsmap(fs::path path); + + /** * serialized format: - * * version byte (currently 1) + * * version byte (1 for pre-asmap files, 2 for files including asmap version) * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) * * nNew * * nTried @@ -302,7 +329,7 @@ public: { LOCK(cs); - unsigned char nVersion = 1; + unsigned char nVersion = 2; s << nVersion; s << ((unsigned char)32); s << nKey; @@ -345,6 +372,13 @@ public: } } } + // Store asmap version after bucket entries so that it + // can be ignored by older clients for backward compatibility. + uint256 asmap_version; + if (m_asmap.size() != 0) { + asmap_version = SerializeHash(m_asmap); + } + s << asmap_version; } template<typename Stream> @@ -353,7 +387,6 @@ public: LOCK(cs); Clear(); - unsigned char nVersion; s >> nVersion; unsigned char nKeySize; @@ -383,16 +416,6 @@ public: mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); - if (nVersion != 1 || nUBuckets != ADDRMAN_NEW_BUCKET_COUNT) { - // In case the new table data cannot be used (nVersion unknown, or bucket count wrong), - // immediately try to give them a reference based on their primary source address. - int nUBucket = info.GetNewBucket(nKey); - int nUBucketPos = info.GetBucketPosition(nKey, true, nUBucket); - if (vvNew[nUBucket][nUBucketPos] == -1) { - vvNew[nUBucket][nUBucketPos] = n; - info.nRefCount++; - } - } } nIdCount = nNew; @@ -401,7 +424,7 @@ public: for (int n = 0; n < nTried; n++) { CAddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (vvTried[nKBucket][nKBucketPos] == -1) { info.nRandomPos = vRandom.size(); @@ -417,7 +440,9 @@ public: } nTried -= nLost; - // Deserialize positions in the new table (if possible). + // Store positions in the new table buckets to apply later (if possible). + std::map<int, int> entryToBucket; // Represents which entry belonged to which bucket when serializing + for (int bucket = 0; bucket < nUBuckets; bucket++) { int nSize = 0; s >> nSize; @@ -425,12 +450,38 @@ public: int nIndex = 0; s >> nIndex; if (nIndex >= 0 && nIndex < nNew) { - CAddrInfo &info = mapInfo[nIndex]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (nVersion == 1 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS) { - info.nRefCount++; - vvNew[bucket][nUBucketPos] = nIndex; - } + entryToBucket[nIndex] = bucket; + } + } + } + + uint256 supplied_asmap_version; + if (m_asmap.size() != 0) { + supplied_asmap_version = SerializeHash(m_asmap); + } + uint256 serialized_asmap_version; + if (nVersion > 1) { + s >> serialized_asmap_version; + } + + for (int n = 0; n < nNew; n++) { + CAddrInfo &info = mapInfo[n]; + int bucket = entryToBucket[n]; + int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && + info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; + } else { + // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + bucket = info.GetNewBucket(nKey, m_asmap); + nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][nUBucketPos] == -1) { + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; } } } diff --git a/src/init.cpp b/src/init.cpp index e7dda59590..842eea526b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -52,7 +52,11 @@ #include <util/threadnames.h> #include <util/translation.h> #include <util/validation.h> +#include <util/asmap.h> #include <validation.h> +#include <hash.h> + + #include <validationinterface.h> #include <walletinitinterface.h> @@ -97,6 +101,8 @@ static constexpr int DUMP_BANS_INTERVAL = 60 * 15; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; + /** * The PID file facilities. */ @@ -426,6 +432,7 @@ void SetupServerArgs() gArgs.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-asmap=<file>", "Specify asn mapping used for bucketing of the peers. Path should be relative to the -datadir path.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #ifdef USE_UPNP #if USE_UPNP gArgs.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1807,6 +1814,25 @@ bool AppInitMain(NodeContext& node) return false; } + // Read asmap file if configured + if (gArgs.IsArgSet("-asmap")) { + std::string asmap_file = gArgs.GetArg("-asmap", ""); + if (asmap_file.empty()) { + asmap_file = DEFAULT_ASMAP_FILENAME; + } + const fs::path asmap_path = GetDataDir() / asmap_file; + std::vector<bool> asmap = CAddrMan::DecodeAsmap(asmap_path); + if (asmap.size() == 0) { + InitError(strprintf(_("Could not find or parse specified asmap: '%s'").translated, asmap_path)); + return false; + } + node.connman->SetAsmap(asmap); + const uint256 asmap_version = SerializeHash(asmap); + LogPrintf("Using asmap version %s for IP bucketing.\n", asmap_version.ToString()); + } else { + LogPrintf("Using /16 prefix for IP bucketing.\n"); + } + // ********************************************************* Step 13: finished SetRPCWarmupFinished(); diff --git a/src/net.cpp b/src/net.cpp index 84692d2a79..004f1de583 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1758,7 +1758,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // but inbound and addnode peers do not use our outbound slots. Inbound peers // also have the added issue that they're attacker controlled and could be used // to prevent us from connecting to particular hosts if we used them here. - setConnected.insert(pnode->addr.GetGroup()); + setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); if (pnode->m_tx_relay == nullptr) { nOutboundBlockRelay++; } else if (!pnode->fFeeler) { @@ -1806,7 +1806,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(addr.GetGroup())) { + if (!fFeeler && setConnected.count(addr.GetGroup(addrman.m_asmap))) { break; } @@ -2778,7 +2778,7 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const { - std::vector<unsigned char> vchNetGroup(ad.GetGroup()); + std::vector<unsigned char> vchNetGroup(ad.GetGroup(addrman.m_asmap)); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } @@ -153,6 +153,7 @@ public: bool m_use_addrman_outgoing = true; std::vector<std::string> m_specified_outgoing; std::vector<std::string> m_added_nodes; + std::vector<bool> m_asmap; }; void Init(const Options& connOptions) { @@ -330,6 +331,8 @@ public: */ int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); + void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = asmap; } + private: struct ListenSocket { public: diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 4fbfa2b5c8..20b6a85905 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -6,6 +6,7 @@ #include <netaddress.h> #include <hash.h> #include <util/strencodings.h> +#include <util/asmap.h> #include <tinyformat.h> static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; @@ -410,7 +411,7 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const * @note No two connections will be attempted to addresses with the same network * group. */ -std::vector<unsigned char> CNetAddr::GetGroup() const +std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) const { std::vector<unsigned char> vchRet; int nClass = NET_IPV6; @@ -470,6 +471,27 @@ std::vector<unsigned char> CNetAddr::GetGroup() const else nBits = 32; + // If asmap is supplied and the address is IPv4/IPv6, + // ignore nBits and use 32/128 bits to obtain ASN from asmap. + // ASN is then returned to be used for bucketing. + if (asmap.size() != 0 && (nClass == NET_IPV4 || nClass == NET_IPV6)) { + nClass = NET_IPV6; + std::vector<bool> ip_bits(128); + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = GetByte(15 - byte_i); + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } + } + + uint32_t asn = Interpret(asmap, ip_bits); + vchRet.push_back(nClass); + for (int i = 0; i < 4; i++) { + vchRet.push_back((asn >> (8 * i)) & 0xFF); + } + return vchRet; + } + vchRet.push_back(nClass); // push our ip onto vchRet byte by byte... diff --git a/src/netaddress.h b/src/netaddress.h index fbb1553338..b1a6054e48 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -78,7 +78,8 @@ class CNetAddr unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; - std::vector<unsigned char> GetGroup() const; + std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; + int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index c034216bc1..27831dfa9e 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -5,6 +5,8 @@ #include <test/util/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> +#include <util/asmap.h> +#include <test/data/asmap.raw.h> #include <hash.h> #include <netbase.h> @@ -12,13 +14,18 @@ class CAddrManTest : public CAddrMan { +private: + bool deterministic; public: - explicit CAddrManTest(bool makeDeterministic = true) + explicit CAddrManTest(bool makeDeterministic = true, + std::vector<bool> asmap = std::vector<bool>()) { if (makeDeterministic) { // Set addrman addr placement to be deterministic. MakeDeterministic(); } + deterministic = makeDeterministic; + m_asmap = asmap; } //! Ensure that bucket placement is always the same for testing purposes. @@ -46,6 +53,21 @@ public: CAddrMan::Delete(nId); } + // Used to test deserialization + std::pair<int, int> GetBucketAndEntry(const CAddress& addr) + { + LOCK(cs); + int nId = mapAddr[addr]; + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { + for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { + if (nId == vvNew[bucket][entry]) { + return std::pair<int, int>(bucket, entry); + } + } + } + return std::pair<int, int>(-1, -1); + } + // Simulates connection failure so that we can test eviction of offline nodes void SimConnFail(CService& addr) { @@ -57,6 +79,16 @@ public: int64_t nLastTry = GetAdjustedTime()-61; Attempt(addr, count_failure, nLastTry); } + + void Clear() + { + CAddrMan::Clear(); + if (deterministic) { + nKey.SetNull(); + insecure_rand = FastRandomContext(true); + } + } + }; static CNetAddr ResolveIP(const char* ip) @@ -83,6 +115,18 @@ static CService ResolveService(std::string ip, int port = 0) return ResolveService(ip.c_str(), port); } +static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) { + std::vector<bool> result(vector_size); + for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) { + unsigned char cur_byte = source[byte_i]; + for (int bit_i = 0; bit_i < 8; ++bit_i) { + result[byte_i * 8 + bit_i] = (cur_byte >> bit_i) & 1; + } + } + return result; +} + + BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) @@ -409,7 +453,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) } -BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { CAddrManTest addrman; @@ -424,30 +468,31 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector<bool> asmap; // use /16 - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 40); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info2.GetTriedBucket(nKey1)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetTriedBucket(nKey1); + int bucket = infoi.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same group (\16 prefix for IPv4) should - // never get more than 8 buckets + // Test: IP addresses in the same /16 prefix should + // never get more than 8 buckets with legacy grouping BOOST_CHECK_EQUAL(buckets.size(), 8U); buckets.clear(); @@ -455,15 +500,15 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), ResolveIP("250." + std::to_string(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1); + int bucket = infoj.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different groups should map to more than - // 8 buckets. + // Test: IP addresses in the different /16 prefix should map to more than + // 8 buckets with legacy grouping BOOST_CHECK_EQUAL(buckets.size(), 160U); } -BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) { CAddrManTest addrman; @@ -477,25 +522,27 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector<bool> asmap; // use /16 + // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 786); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetNewBucket(nKey1); + int bucket = infoi.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } // Test: IP addresses in the same group (\16 prefix for IPv4) should @@ -508,10 +555,10 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) ResolveService( std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same source groups should map to no more + // Test: IP addresses in the same source groups should map to NO MORE // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); @@ -520,14 +567,226 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + std::to_string(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different source groups should map to more + // Test: IP addresses in the different source groups should map to MORE // than 64 buckets. BOOST_CHECK(buckets.size() > 64); } +// The following three test cases use asmap.raw +// We use an artificial minimal mock mapping +// 250.0.0.0/8 AS1000 +// 101.1.0.0/16 AS1 +// 101.2.0.0/16 AS2 +// 101.3.0.0/16 AS3 +// 101.4.0.0/16 AS4 +// 101.5.0.0/16 AS5 +// 101.6.0.0/16 AS6 +// 101.7.0.0/16 AS7 +// 101.8.0.0/16 AS8 +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.1.1"); + + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 236); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + + // Test: Two addresses with same IP but different ports can map to + // different buckets because they have different keys. + CAddrInfo info2 = CAddrInfo(addr2, source1); + + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + + std::set<int> buckets; + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("101." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix MAY map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() > 8); + + buckets.clear(); + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix MAY NOT map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() == 8); +} + +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.2.1"); + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 795); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + + // Test: Ports should not affect bucket placement in the addr + CAddrInfo info2 = CAddrInfo(addr2, source1); + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + + std::set<int> buckets; + for (int i = 0; i < 255; i++) { + CAddrInfo infoi = CAddrInfo( + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same /16 prefix + // usually map to the same bucket. + BOOST_CHECK_EQUAL(buckets.size(), 1U); + + buckets.clear(); + for (int j = 0; j < 4 * 255; j++) { + CAddrInfo infoj = CAddrInfo(CAddress( + ResolveService( + std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), + ResolveIP("251.4.1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same source /16 prefix should not map to more + // than 64 buckets. + BOOST_CHECK(buckets.size() <= 64); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes usually map to MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() > 1); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() == 1); + +} + +BOOST_AUTO_TEST_CASE(addrman_serialization) +{ + std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + CAddrManTest addrman_asmap1(true, asmap1); + CAddrManTest addrman_asmap1_dup(true, asmap1); + CAddrManTest addrman_noasmap; + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CNetAddr default_source; + + + addrman_asmap1.Add(addr, default_source); + + stream << addrman_asmap1; + // serizalizing/deserializing addrman with the same asmap + stream >> addrman_asmap1_dup; + + std::pair<int, int> bucketAndEntry_asmap1 = addrman_asmap1.GetBucketAndEntry(addr); + std::pair<int, int> bucketAndEntry_asmap1_dup = addrman_asmap1_dup.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_dup.second != -1); + + BOOST_CHECK(bucketAndEntry_asmap1.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1.second == bucketAndEntry_asmap1_dup.second); + + // deserializing asmaped peers.dat to non-asmaped addrman + stream << addrman_asmap1; + stream >> addrman_noasmap; + std::pair<int, int> bucketAndEntry_noasmap = addrman_noasmap.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_noasmap.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); + + // deserializing non-asmaped peers.dat to asmaped addrman + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + addrman_noasmap.Add(addr, default_source); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser = addrman_asmap1.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); + + // used to map to different buckets, now maps to the same bucket. + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); + addrman_noasmap.Add(addr, default_source); + addrman_noasmap.Add(addr2, default_source); + std::pair<int, int> bucketAndEntry_noasmap_addr1 = addrman_noasmap.GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_noasmap_addr2 = addrman_noasmap.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.first != bucketAndEntry_noasmap_addr2.first); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.second != bucketAndEntry_noasmap_addr2.second); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1.GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.first == bucketAndEntry_asmap1_deser_addr2.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); +} + BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { diff --git a/src/test/data/asmap.raw b/src/test/data/asmap.raw Binary files differnew file mode 100644 index 0000000000..3dcf1f3940 --- /dev/null +++ b/src/test/data/asmap.raw diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 78c11ff202..46ca1c8c68 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -285,23 +285,23 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_AUTO_TEST_CASE(netbase_getgroup) { - - BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // Local -> !Routable() - BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // !Valid -> !Routable() - BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() - BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup() == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() - BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 - BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 - BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 - BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 - BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 - BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_ONION, 239})); // Tor - BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net - BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + std::vector<bool> asmap; // use /16 + BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // Local -> !Routable() + BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // !Valid -> !Routable() + BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() + BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() + BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 + BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 + BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 + BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 + BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 + BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_ONION, 239})); // Tor + BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net + BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 std::vector<unsigned char> internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; - BOOST_CHECK(CreateInternal("baz.net").GetGroup() == internal_group); + BOOST_CHECK(CreateInternal("baz.net").GetGroup(asmap) == internal_group); } BOOST_AUTO_TEST_CASE(netbase_parsenetwork) |