From 3ca574cef0b4423f21b2c3efd8f5c9f71d52f219 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Feb 2020 19:49:10 -0800 Subject: Convert CCompactSize to proper formatter --- src/serialize.h | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index cee7225bcb..e0e29bcf4a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -495,7 +495,7 @@ static inline Wrapper Using(T&& t) { return Wrapper>(obj) #define VARINT(obj) Using>(obj) -#define COMPACTSIZE(obj) CCompactSize(REF(obj)) +#define COMPACTSIZE(obj) Using(obj) #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) /** Serialization wrapper class for integers in VarInt format. */ @@ -547,21 +547,26 @@ public: } }; -class CCompactSize +/** Formatter for integers in CompactSize format. */ +struct CompactSizeFormatter { -protected: - uint64_t &n; -public: - explicit CCompactSize(uint64_t& nIn) : n(nIn) { } - - template - void Serialize(Stream &s) const { - WriteCompactSize(s, n); + template + void Unser(Stream& s, I& v) + { + uint64_t n = ReadCompactSize(s); + if (n < std::numeric_limits::min() || n > std::numeric_limits::max()) { + throw std::ios_base::failure("CompactSize exceeds limit of type"); + } + v = n; } - template - void Unserialize(Stream& s) { - n = ReadCompactSize(s); + template + void Ser(Stream& s, I v) + { + static_assert(std::is_unsigned::value, "CompactSize only supported for unsigned integers"); + static_assert(std::numeric_limits::max() <= std::numeric_limits::max(), "CompactSize only supports 64-bit integers and below"); + + WriteCompactSize(s, v); } }; -- cgit v1.2.3 From 56dd9f04c701aa3ac340e95065bf83de20373c8b Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Sat, 15 Feb 2020 19:09:09 -0800 Subject: Make VectorFormatter support stateful formatters --- src/prevector.h | 9 +++++++-- src/serialize.h | 11 ++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/prevector.h b/src/prevector.h index 09debedc4f..6d690e7f96 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -424,15 +424,20 @@ public: return first; } - void push_back(const T& value) { + template + void emplace_back(Args&&... args) { size_type new_size = size() + 1; if (capacity() < new_size) { change_capacity(new_size + (new_size >> 1)); } - new(item_ptr(size())) T(value); + new(item_ptr(size())) T(std::forward(args)...); _size++; } + void push_back(const T& value) { + emplace_back(value); + } + void pop_back() { erase(end() - 1, end()); } diff --git a/src/serialize.h b/src/serialize.h index e0e29bcf4a..d7d1be7c4b 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -613,7 +613,7 @@ BigEndian WrapBigEndian(I& n) { return BigEndian(n); } * as a vector of VarInt-encoded integers. * * V is not required to be an std::vector type. It works for any class that - * exposes a value_type, size, reserve, push_back, and const iterators. + * exposes a value_type, size, reserve, emplace_back, back, and const iterators. */ template struct VectorFormatter @@ -621,15 +621,17 @@ struct VectorFormatter template void Ser(Stream& s, const V& v) { + Formatter formatter; WriteCompactSize(s, v.size()); for (const typename V::value_type& elem : v) { - s << Using(elem); + formatter.Ser(s, elem); } } template void Unser(Stream& s, V& v) { + Formatter formatter; v.clear(); size_t size = ReadCompactSize(s); size_t allocated = 0; @@ -641,9 +643,8 @@ struct VectorFormatter allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type)); v.reserve(allocated); while (v.size() < allocated) { - typename V::value_type val; - s >> Using(val); - v.push_back(std::move(val)); + v.emplace_back(); + formatter.Unser(s, v.back()); } } }; -- cgit v1.2.3 From 10633398f2dddf929d3f535aa48d138ad5e6c50f Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Sat, 15 Feb 2020 19:05:11 -0800 Subject: Add DifferenceFormatter --- src/blockencodings.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/blockencodings.h b/src/blockencodings.h index 55ed8989bb..606c11b417 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -25,6 +25,28 @@ public: } }; +class DifferenceFormatter +{ + uint64_t m_shift = 0; + +public: + template + void Ser(Stream& s, I v) + { + if (v < m_shift || v >= std::numeric_limits::max()) throw std::ios_base::failure("differential value overflow"); + WriteCompactSize(s, v - m_shift); + m_shift = uint64_t(v) + 1; + } + template + void Unser(Stream& s, I& v) + { + uint64_t n = ReadCompactSize(s); + m_shift += n; + if (m_shift < n || m_shift >= std::numeric_limits::max() || m_shift < std::numeric_limits::min() || m_shift > std::numeric_limits::max()) throw std::ios_base::failure("differential value overflow"); + v = I(m_shift++); + } +}; + class BlockTransactionsRequest { public: // A BlockTransactionsRequest message -- cgit v1.2.3 From e574fff53eec4a27c83b765cb69e31d8399047ea Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 15 Feb 2020 19:48:42 -0800 Subject: Add CustomUintFormatter --- src/serialize.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index d7d1be7c4b..2e4b14e586 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -513,6 +513,28 @@ struct VarIntFormatter } }; +template +struct CustomUintFormatter +{ + static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range"); + static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes)); + + template void Ser(Stream& s, I v) + { + if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); + uint64_t raw = htole64(v); + s.write((const char*)&raw, Bytes); + } + + template void Unser(Stream& s, I& v) + { + static_assert(std::numeric_limits::max() >= MAX && std::numeric_limits::min() <= 0, "CustomUintFormatter type too small"); + uint64_t raw = 0; + s.read((char*)&raw, Bytes); + v = le64toh(raw); + } +}; + /** Serialization wrapper class for big-endian integers. * * Use this wrapper around integer types that are stored in memory in native -- cgit v1.2.3 From 353f376277ad9b87e03c9ccbc1028c4b6d12e8ea Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Feb 2020 19:41:42 -0800 Subject: Convert blockencodings.h to new serialization framework --- src/blockencodings.h | 126 +++++++-------------------------------------------- 1 file changed, 16 insertions(+), 110 deletions(-) diff --git a/src/blockencodings.h b/src/blockencodings.h index 606c11b417..be50166cfc 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -10,20 +10,9 @@ class CTxMemPool; -// Dumb helper to handle CTransaction compression at serialize-time -struct TransactionCompressor { -private: - CTransactionRef& tx; -public: - explicit TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {} - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(tx); //TODO: Compress tx encoding - } -}; +// Transaction compression schemes for compact block relay can be introduced by writing +// an actual formatter here. +using TransactionCompression = DefaultFormatter; class DifferenceFormatter { @@ -53,39 +42,9 @@ public: uint256 blockhash; std::vector indexes; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(blockhash); - uint64_t indexes_size = (uint64_t)indexes.size(); - READWRITE(COMPACTSIZE(indexes_size)); - if (ser_action.ForRead()) { - size_t i = 0; - while (indexes.size() < indexes_size) { - indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size)); - for (; i < indexes.size(); i++) { - uint64_t index = 0; - READWRITE(COMPACTSIZE(index)); - if (index > std::numeric_limits::max()) - throw std::ios_base::failure("index overflowed 16 bits"); - indexes[i] = index; - } - } - - int32_t offset = 0; - for (size_t j = 0; j < indexes.size(); j++) { - if (int32_t(indexes[j]) + offset > std::numeric_limits::max()) - throw std::ios_base::failure("indexes overflowed 16 bits"); - indexes[j] = indexes[j] + offset; - offset = int32_t(indexes[j]) + 1; - } - } else { - for (size_t i = 0; i < indexes.size(); i++) { - uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1)); - READWRITE(COMPACTSIZE(index)); - } - } + SERIALIZE_METHODS(BlockTransactionsRequest, obj) + { + READWRITE(obj.blockhash, Using>(obj.indexes)); } }; @@ -99,24 +58,9 @@ public: explicit BlockTransactions(const BlockTransactionsRequest& req) : blockhash(req.blockhash), txn(req.indexes.size()) {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(blockhash); - uint64_t txn_size = (uint64_t)txn.size(); - READWRITE(COMPACTSIZE(txn_size)); - if (ser_action.ForRead()) { - size_t i = 0; - while (txn.size() < txn_size) { - txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size)); - for (; i < txn.size(); i++) - READWRITE(TransactionCompressor(txn[i])); - } - } else { - for (size_t i = 0; i < txn.size(); i++) - READWRITE(TransactionCompressor(txn[i])); - } + SERIALIZE_METHODS(BlockTransactions, obj) + { + READWRITE(obj.blockhash, Using>(obj.txn)); } }; @@ -127,17 +71,7 @@ struct PrefilledTransaction { uint16_t index; CTransactionRef tx; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - uint64_t idx = index; - READWRITE(COMPACTSIZE(idx)); - if (idx > std::numeric_limits::max()) - throw std::ios_base::failure("index overflowed 16-bits"); - index = idx; - READWRITE(TransactionCompressor(tx)); - } + SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), Using(obj.tx)); } }; typedef enum ReadStatus_t @@ -175,43 +109,15 @@ public: size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(header); - READWRITE(nonce); - - uint64_t shorttxids_size = (uint64_t)shorttxids.size(); - READWRITE(COMPACTSIZE(shorttxids_size)); + SERIALIZE_METHODS(CBlockHeaderAndShortTxIDs, obj) + { + READWRITE(obj.header, obj.nonce, Using>>(obj.shorttxids), obj.prefilledtxn); if (ser_action.ForRead()) { - size_t i = 0; - while (shorttxids.size() < shorttxids_size) { - shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size)); - for (; i < shorttxids.size(); i++) { - uint32_t lsb = 0; uint16_t msb = 0; - READWRITE(lsb); - READWRITE(msb); - shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); - static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids"); - } - } - } else { - for (size_t i = 0; i < shorttxids.size(); i++) { - uint32_t lsb = shorttxids[i] & 0xffffffff; - uint16_t msb = (shorttxids[i] >> 32) & 0xffff; - READWRITE(lsb); - READWRITE(msb); + if (obj.BlockTxCount() > std::numeric_limits::max()) { + throw std::ios_base::failure("indexes overflowed 16 bits"); } + obj.FillShortTxIDSelector(); } - - READWRITE(prefilledtxn); - - if (BlockTxCount() > std::numeric_limits::max()) - throw std::ios_base::failure("indexes overflowed 16 bits"); - - if (ser_action.ForRead()) - FillShortTxIDSelector(); } }; -- cgit v1.2.3