diff options
Diffstat (limited to 'src/serialize.h')
-rw-r--r-- | src/serialize.h | 242 |
1 files changed, 237 insertions, 5 deletions
diff --git a/src/serialize.h b/src/serialize.h index 63df3160a6..f2626281c1 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -50,10 +50,6 @@ enum SER_NETWORK = (1 << 0), SER_DISK = (1 << 1), SER_GETHASH = (1 << 2), - - // modifiers - SER_SKIPSIG = (1 << 16), - SER_BLOCKHEADERONLY = (1 << 17), }; #define IMPLEMENT_SERIALIZE(statements) \ @@ -238,9 +234,75 @@ uint64 ReadCompactSize(Stream& is) return nSizeRet; } +// Variable-length integers: bytes are a MSB base-128 encoding of the number. +// The high bit in each byte signifies whether another digit follows. To make +// the encoding is one-to-one, one is subtracted from all but the last digit. +// Thus, the byte sequence a[] with length len, where all but the last byte +// has bit 128 set, encodes the number: +// +// (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1)) +// +// Properties: +// * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes) +// * Every integer has exactly one encoding +// * Encoding does not depend on size of original integer type +// * No redundancy: every (infinite) byte sequence corresponds to a list +// of encoded integers. +// +// 0: [0x00] 256: [0x81 0x00] +// 1: [0x01] 16383: [0xFE 0x7F] +// 127: [0x7F] 16384: [0xFF 0x00] +// 128: [0x80 0x00] 16511: [0x80 0xFF 0x7F] +// 255: [0x80 0x7F] 65535: [0x82 0xFD 0x7F] +// 2^32: [0x8E 0xFE 0xFE 0xFF 0x00] + +template<typename I> +inline unsigned int GetSizeOfVarInt(I n) +{ + int nRet = 0; + while(true) { + nRet++; + if (n <= 0x7F) + break; + n = (n >> 7) - 1; + } + return nRet; +} + +template<typename Stream, typename I> +void WriteVarInt(Stream& os, I n) +{ + unsigned char tmp[(sizeof(n)*8+6)/7]; + int len=0; + while(true) { + tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00); + if (n <= 0x7F) + break; + n = (n >> 7) - 1; + len++; + } + do { + WRITEDATA(os, tmp[len]); + } while(len--); +} +template<typename Stream, typename I> +I ReadVarInt(Stream& is) +{ + I n = 0; + while(true) { + unsigned char chData; + READDATA(is, chData); + n = (n << 7) | (chData & 0x7F); + if (chData & 0x80) + n++; + else + return n; + } +} -#define FLATDATA(obj) REF(CFlatData((char*)&(obj), (char*)&(obj) + sizeof(obj))) +#define FLATDATA(obj) REF(CFlatData((char*)&(obj), (char*)&(obj) + sizeof(obj))) +#define VARINT(obj) REF(WrapVarInt(REF(obj))) /** Wrapper for serializing arrays and POD. */ @@ -274,6 +336,32 @@ public: } }; +template<typename I> +class CVarInt +{ +protected: + I &n; +public: + CVarInt(I& nIn) : n(nIn) { } + + unsigned int GetSerializeSize(int, int) const { + return GetSizeOfVarInt<I>(n); + } + + template<typename Stream> + void Serialize(Stream &s, int, int) const { + WriteVarInt<Stream,I>(s, n); + } + + template<typename Stream> + void Unserialize(Stream& s, int, int) { + n = ReadVarInt<Stream,I>(s); + } +}; + +template<typename I> +CVarInt<I> WrapVarInt(I& n) { return CVarInt<I>(n); } + // // Forward declarations // @@ -1133,4 +1221,148 @@ public: } }; +/** Wrapper around a FILE* that implements a ring buffer to + * deserialize from. It guarantees the ability to rewind + * a given number of bytes. */ +class CBufferedFile +{ +private: + FILE *src; // source file + uint64 nSrcPos; // how many bytes have been read from source + uint64 nReadPos; // how many bytes have been read from this + uint64 nReadLimit; // up to which position we're allowed to read + uint64 nRewind; // how many bytes we guarantee to rewind + std::vector<char> vchBuf; // the buffer + + short state; + short exceptmask; + +protected: + void setstate(short bits, const char *psz) { + state |= bits; + if (state & exceptmask) + throw std::ios_base::failure(psz); + } + + // read data from the source to fill the buffer + bool Fill() { + unsigned int pos = nSrcPos % vchBuf.size(); + unsigned int readNow = vchBuf.size() - pos; + unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind; + if (nAvail < readNow) + readNow = nAvail; + if (readNow == 0) + return false; + size_t read = fread((void*)&vchBuf[pos], 1, readNow, src); + if (read == 0) { + setstate(std::ios_base::failbit, feof(src) ? "CBufferedFile::Fill : end of file" : "CBufferedFile::Fill : fread failed"); + return false; + } else { + nSrcPos += read; + return true; + } + } + +public: + int nType; + int nVersion; + + CBufferedFile(FILE *fileIn, uint64 nBufSize, uint64 nRewindIn, int nTypeIn, int nVersionIn) : + src(fileIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0), + state(0), exceptmask(std::ios_base::badbit | std::ios_base::failbit), nType(nTypeIn), nVersion(nVersionIn) { + } + + // check whether no error occurred + bool good() const { + return state == 0; + } + + // check whether we're at the end of the source file + bool eof() const { + return nReadPos == nSrcPos && feof(src); + } + + // read a number of bytes + CBufferedFile& read(char *pch, size_t nSize) { + if (nSize + nReadPos > nReadLimit) + throw std::ios_base::failure("Read attempted past buffer limit"); + if (nSize + nRewind > vchBuf.size()) + throw std::ios_base::failure("Read larger than buffer size"); + while (nSize > 0) { + if (nReadPos == nSrcPos) + Fill(); + unsigned int pos = nReadPos % vchBuf.size(); + size_t nNow = nSize; + if (nNow + pos > vchBuf.size()) + nNow = vchBuf.size() - pos; + if (nNow + nReadPos > nSrcPos) + nNow = nSrcPos - nReadPos; + memcpy(pch, &vchBuf[pos], nNow); + nReadPos += nNow; + pch += nNow; + nSize -= nNow; + } + return (*this); + } + + // return the current reading position + uint64 GetPos() { + return nReadPos; + } + + // rewind to a given reading position + bool SetPos(uint64 nPos) { + nReadPos = nPos; + if (nReadPos + nRewind < nSrcPos) { + nReadPos = nSrcPos - nRewind; + return false; + } else if (nReadPos > nSrcPos) { + nReadPos = nSrcPos; + return false; + } else { + return true; + } + } + + bool Seek(uint64 nPos) { + long nLongPos = nPos; + if (nPos != (uint64)nLongPos) + return false; + if (fseek(src, nLongPos, SEEK_SET)) + return false; + nLongPos = ftell(src); + nSrcPos = nLongPos; + nReadPos = nLongPos; + state = 0; + return true; + } + + // prevent reading beyond a certain position + // no argument removes the limit + bool SetLimit(uint64 nPos = (uint64)(-1)) { + if (nPos < nReadPos) + return false; + nReadLimit = nPos; + return true; + } + + template<typename T> + CBufferedFile& operator>>(T& obj) { + // Unserialize from this stream + ::Unserialize(*this, obj, nType, nVersion); + return (*this); + } + + // search for a given byte in the stream, and remain positioned on it + void FindByte(char ch) { + while (true) { + if (nReadPos == nSrcPos) + Fill(); + if (vchBuf[nReadPos % vchBuf.size()] == ch) + break; + nReadPos++; + } + } +}; + #endif |