Merge #18112: Serialization improvements step 5 (blockencodings)

353f376277ad9b87e03c9ccbc1028c4b6d12e8ea Convert blockencodings.h to new serialization framework (Pieter Wuille)
e574fff53eec4a27c83b765cb69e31d8399047ea Add CustomUintFormatter (Pieter Wuille)
10633398f2dddf929d3f535aa48d138ad5e6c50f Add DifferenceFormatter (Russell Yanofsky)
56dd9f04c701aa3ac340e95065bf83de20373c8b Make VectorFormatter support stateful formatters (Russell Yanofsky)
3ca574cef0b4423f21b2c3efd8f5c9f71d52f219 Convert CCompactSize to proper formatter (Pieter Wuille)

Pull request description:

  This is probably the most involved change in the sequence of changes extracted from #10785.

  In order to implement the differential encoding of BIP152, this change changes `VectorFormatter` to permit a stateful sub-formatter, which is then used by `DifferenceFormatter`. A `CustomUintFormatter` is added as well to do the 48-bit serialization of short ids.

ACKs for top commit:
  laanwj:
    ACK 353f376277ad9b87e03c9ccbc1028c4b6d12e8ea, nice change
  ryanofsky:
    Code review ACK 353f376277ad9b87e03c9ccbc1028c4b6d12e8ea. Only changes since last review are suggested assert change and MASK->MAX rename

Tree-SHA512: 976618991a8be62ba0738725b7cfa166a56cde998ebf1031ba6f28557032f1577b666ac7ae25cd498c0e1e740108c3c56a342620b724df41d6cc9d8bdafac037

1 files changed, 46 insertions, 18 deletions

diff --git a/src/serialize.h b/src/serialize.h
index 0a43913f83..5045cb3c7f 100644
--- a/src/serialize.h
+++ b/src/serialize.h
@@ -500,7 +500,7 @@ static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&
 
 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
-#define COMPACTSIZE(obj) CCompactSize(REF(obj))
+#define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj)
 #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj))
 
 /** Serialization wrapper class for integers in VarInt format. */
@@ -518,6 +518,28 @@ struct VarIntFormatter
     }
 };
 
+template<int Bytes>
+struct CustomUintFormatter
+{
+    static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
+    static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
+
+    template <typename Stream, typename I> void Ser(Stream& s, I v)
+    {
+        if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
+        uint64_t raw = htole64(v);
+        s.write((const char*)&raw, Bytes);
+    }
+
+    template <typename Stream, typename I> void Unser(Stream& s, I& v)
+    {
+        static_assert(std::numeric_limits<I>::max() >= MAX && std::numeric_limits<I>::min() <= 0, "CustomUintFormatter type too small");
+        uint64_t raw = 0;
+        s.read((char*)&raw, Bytes);
+        v = le64toh(raw);
+    }
+};
+
 /** Serialization wrapper class for big-endian integers.
  *
  * Use this wrapper around integer types that are stored in memory in native
@@ -552,21 +574,26 @@ public:
     }
 };
 
-class CCompactSize
+/** Formatter for integers in CompactSize format. */
+struct CompactSizeFormatter
 {
-protected:
-    uint64_t &n;
-public:
-    explicit CCompactSize(uint64_t& nIn) : n(nIn) { }
-
-    template<typename Stream>
-    void Serialize(Stream &s) const {
-        WriteCompactSize<Stream>(s, n);
+    template<typename Stream, typename I>
+    void Unser(Stream& s, I& v)
+    {
+        uint64_t n = ReadCompactSize<Stream>(s);
+        if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
+            throw std::ios_base::failure("CompactSize exceeds limit of type");
+        }
+        v = n;
     }
 
-    template<typename Stream>
-    void Unserialize(Stream& s) {
-        n = ReadCompactSize<Stream>(s);
+    template<typename Stream, typename I>
+    void Ser(Stream& s, I v)
+    {
+        static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
+        static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
+
+        WriteCompactSize<Stream>(s, v);
     }
 };
 
@@ -613,7 +640,7 @@ BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
  * as a vector of VarInt-encoded integers.
  *
  * V is not required to be an std::vector type. It works for any class that
- * exposes a value_type, size, reserve, push_back, and const iterators.
+ * exposes a value_type, size, reserve, emplace_back, back, and const iterators.
  */
 template<class Formatter>
 struct VectorFormatter
@@ -621,15 +648,17 @@ struct VectorFormatter
     template<typename Stream, typename V>
     void Ser(Stream& s, const V& v)
     {
+        Formatter formatter;
         WriteCompactSize(s, v.size());
         for (const typename V::value_type& elem : v) {
-            s << Using<Formatter>(elem);
+            formatter.Ser(s, elem);
         }
     }
 
     template<typename Stream, typename V>
     void Unser(Stream& s, V& v)
     {
+        Formatter formatter;
         v.clear();
         size_t size = ReadCompactSize(s);
         size_t allocated = 0;
@@ -641,9 +670,8 @@ struct VectorFormatter
             allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
             v.reserve(allocated);
             while (v.size() < allocated) {
-                typename V::value_type val;
-                s >> Using<Formatter>(val);
-                v.push_back(std::move(val));
+                v.emplace_back();
+                formatter.Unser(s, v.back());
             }
         }
     };