boost: split stream classes out of serialize.h

serialization now has no dependencies.

16 files changed, 589 insertions, 553 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index 155adfef7d..77443686db 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -107,6 +107,7 @@ BITCOIN_CORE_H = \
   script/sign.h \
   script/standard.h \
   serialize.h \
+  streams.h \
   sync.h \
   threadsafety.h \
   timedata.h \
diff --git a/src/addrman.cpp b/src/addrman.cpp
index 7b674a66e7..3c6f8d9e72 100644
--- a/src/addrman.cpp
+++ b/src/addrman.cpp
@@ -6,6 +6,7 @@
 
 #include "hash.h"
 #include "serialize.h"
+#include "streams.h"
 
 using namespace std;
 
diff --git a/src/allocators.h b/src/allocators.h
index 6b69e7ae69..78a3b76d0c 100644
--- a/src/allocators.h
+++ b/src/allocators.h
@@ -9,6 +9,7 @@
 #include <map>
 #include <string>
 #include <string.h>
+#include <vector>
 
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/once.hpp>
@@ -261,4 +262,7 @@ struct zero_after_free_allocator : public std::allocator<T> {
 // This is exactly like std::string, but with a custom allocator.
 typedef std::basic_string<char, std::char_traits<char>, secure_allocator<char> > SecureString;
 
+// Byte-vector that clears its contents before deletion.
+typedef std::vector<char, zero_after_free_allocator<char> > CSerializeData;
+
 #endif // BITCOIN_ALLOCATORS_H
diff --git a/src/bloom.cpp b/src/bloom.cpp
index cef74a3a54..cac71fdbbf 100644
--- a/src/bloom.cpp
+++ b/src/bloom.cpp
@@ -7,6 +7,7 @@
 #include "core.h"
 #include "script/script.h"
 #include "script/standard.h"
+#include "streams.h"
 
 #include <math.h>
 #include <stdlib.h>
diff --git a/src/core_read.cpp b/src/core_read.cpp
index 8b85a03c54..dcbcf4b4f7 100644
--- a/src/core_read.cpp
+++ b/src/core_read.cpp
@@ -7,6 +7,7 @@
 #include "core.h"
 #include "script/script.h"
 #include "serialize.h"
+#include "streams.h"
 #include "univalue/univalue.h"
 #include "util.h"
 #include "utilstrencodings.h"
diff --git a/src/core_write.cpp b/src/core_write.cpp
index e42e0b62a9..b2b29fb367 100644
--- a/src/core_write.cpp
+++ b/src/core_write.cpp
@@ -9,6 +9,7 @@
 #include "script/script.h"
 #include "script/standard.h"
 #include "serialize.h"
+#include "streams.h"
 #include "univalue/univalue.h"
 #include "util.h"
 #include "utilmoneystr.h"
diff --git a/src/db.h b/src/db.h
index d202399383..0cbdd8b91b 100644
--- a/src/db.h
+++ b/src/db.h
@@ -7,6 +7,7 @@
 #define BITCOIN_DB_H
 
 #include "serialize.h"
+#include "streams.h"
 #include "sync.h"
 #include "version.h"
 
diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h
index da5ba61c7b..d997d56e00 100644
--- a/src/leveldbwrapper.h
+++ b/src/leveldbwrapper.h
@@ -6,6 +6,7 @@
 #define BITCOIN_LEVELDBWRAPPER_H
 
 #include "serialize.h"
+#include "streams.h"
 #include "util.h"
 #include "version.h"
 
diff --git a/src/net.h b/src/net.h
index 18da24183f..340158512d 100644
--- a/src/net.h
+++ b/src/net.h
@@ -14,6 +14,7 @@
 #include "netbase.h"
 #include "protocol.h"
 #include "random.h"
+#include "streams.h"
 #include "sync.h"
 #include "uint256.h"
 #include "utilstrencodings.h"
diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp
index 0e5802922c..5deac8007c 100644
--- a/src/qt/recentrequeststablemodel.cpp
+++ b/src/qt/recentrequeststablemodel.cpp
@@ -7,6 +7,7 @@
 #include "bitcoinunits.h"
 #include "guiutil.h"
 #include "optionsmodel.h"
+#include "streams.h"
 
 #include <boost/foreach.hpp>
 
diff --git a/src/serialize.h b/src/serialize.h
index b9d5f95463..877ef8640a 100644
--- a/src/serialize.h
+++ b/src/serialize.h
@@ -6,8 +6,6 @@
 #ifndef BITCOIN_SERIALIZE_H
 #define BITCOIN_SERIALIZE_H
 
-#include "allocators.h"
-
 #include <algorithm>
 #include <assert.h>
 #include <ios>
@@ -20,8 +18,6 @@
 #include <utility>
 #include <vector>
 
-class CAutoFile;
-class CDataStream;
 class CScript;
 
 static const unsigned int MAX_SIZE = 0x02000000;
@@ -761,8 +757,6 @@ inline void SerReadWrite(Stream& s, T& obj, int nType, int nVersion, CSerActionU
 
 
 
-typedef std::vector<char, zero_after_free_allocator<char> > CSerializeData;
-
 class CSizeComputer
 {
 protected:
@@ -792,551 +786,4 @@ public:
     }
 };
 
-/** Double ended buffer combining vector and stream-like interfaces.
- *
- * >> and << read and write unformatted data using the above serialization templates.
- * Fills with data in linear time; some stringstream implementations take N^2 time.
- */
-class CDataStream
-{
-protected:
-    typedef CSerializeData vector_type;
-    vector_type vch;
-    unsigned int nReadPos;
-public:
-    int nType;
-    int nVersion;
-
-    typedef vector_type::allocator_type   allocator_type;
-    typedef vector_type::size_type        size_type;
-    typedef vector_type::difference_type  difference_type;
-    typedef vector_type::reference        reference;
-    typedef vector_type::const_reference  const_reference;
-    typedef vector_type::value_type       value_type;
-    typedef vector_type::iterator         iterator;
-    typedef vector_type::const_iterator   const_iterator;
-    typedef vector_type::reverse_iterator reverse_iterator;
-
-    explicit CDataStream(int nTypeIn, int nVersionIn)
-    {
-        Init(nTypeIn, nVersionIn);
-    }
-
-    CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
-    {
-        Init(nTypeIn, nVersionIn);
-    }
-
-#if !defined(_MSC_VER) || _MSC_VER >= 1300
-    CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
-    {
-        Init(nTypeIn, nVersionIn);
-    }
-#endif
-
-    CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
-    {
-        Init(nTypeIn, nVersionIn);
-    }
-
-    CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
-    {
-        Init(nTypeIn, nVersionIn);
-    }
-
-    CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
-    {
-        Init(nTypeIn, nVersionIn);
-    }
-
-    void Init(int nTypeIn, int nVersionIn)
-    {
-        nReadPos = 0;
-        nType = nTypeIn;
-        nVersion = nVersionIn;
-    }
-
-    CDataStream& operator+=(const CDataStream& b)
-    {
-        vch.insert(vch.end(), b.begin(), b.end());
-        return *this;
-    }
-
-    friend CDataStream operator+(const CDataStream& a, const CDataStream& b)
-    {
-        CDataStream ret = a;
-        ret += b;
-        return (ret);
-    }
-
-    std::string str() const
-    {
-        return (std::string(begin(), end()));
-    }
-
-
-    //
-    // Vector subset
-    //
-    const_iterator begin() const                     { return vch.begin() + nReadPos; }
-    iterator begin()                                 { return vch.begin() + nReadPos; }
-    const_iterator end() const                       { return vch.end(); }
-    iterator end()                                   { return vch.end(); }
-    size_type size() const                           { return vch.size() - nReadPos; }
-    bool empty() const                               { return vch.size() == nReadPos; }
-    void resize(size_type n, value_type c=0)         { vch.resize(n + nReadPos, c); }
-    void reserve(size_type n)                        { vch.reserve(n + nReadPos); }
-    const_reference operator[](size_type pos) const  { return vch[pos + nReadPos]; }
-    reference operator[](size_type pos)              { return vch[pos + nReadPos]; }
-    void clear()                                     { vch.clear(); nReadPos = 0; }
-    iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); }
-    void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); }
-
-    void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last)
-    {
-        assert(last - first >= 0);
-        if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
-        {
-            // special case for inserting at the front when there's room
-            nReadPos -= (last - first);
-            memcpy(&vch[nReadPos], &first[0], last - first);
-        }
-        else
-            vch.insert(it, first, last);
-    }
-
-#if !defined(_MSC_VER) || _MSC_VER >= 1300
-    void insert(iterator it, const char* first, const char* last)
-    {
-        assert(last - first >= 0);
-        if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
-        {
-            // special case for inserting at the front when there's room
-            nReadPos -= (last - first);
-            memcpy(&vch[nReadPos], &first[0], last - first);
-        }
-        else
-            vch.insert(it, first, last);
-    }
-#endif
-
-    iterator erase(iterator it)
-    {
-        if (it == vch.begin() + nReadPos)
-        {
-            // special case for erasing from the front
-            if (++nReadPos >= vch.size())
-            {
-                // whenever we reach the end, we take the opportunity to clear the buffer
-                nReadPos = 0;
-                return vch.erase(vch.begin(), vch.end());
-            }
-            return vch.begin() + nReadPos;
-        }
-        else
-            return vch.erase(it);
-    }
-
-    iterator erase(iterator first, iterator last)
-    {
-        if (first == vch.begin() + nReadPos)
-        {
-            // special case for erasing from the front
-            if (last == vch.end())
-            {
-                nReadPos = 0;
-                return vch.erase(vch.begin(), vch.end());
-            }
-            else
-            {
-                nReadPos = (last - vch.begin());
-                return last;
-            }
-        }
-        else
-            return vch.erase(first, last);
-    }
-
-    inline void Compact()
-    {
-        vch.erase(vch.begin(), vch.begin() + nReadPos);
-        nReadPos = 0;
-    }
-
-    bool Rewind(size_type n)
-    {
-        // Rewind by n characters if the buffer hasn't been compacted yet
-        if (n > nReadPos)
-            return false;
-        nReadPos -= n;
-        return true;
-    }
-
-
-    //
-    // Stream subset
-    //
-    bool eof() const             { return size() == 0; }
-    CDataStream* rdbuf()         { return this; }
-    int in_avail()               { return size(); }
-
-    void SetType(int n)          { nType = n; }
-    int GetType()                { return nType; }
-    void SetVersion(int n)       { nVersion = n; }
-    int GetVersion()             { return nVersion; }
-    void ReadVersion()           { *this >> nVersion; }
-    void WriteVersion()          { *this << nVersion; }
-
-    CDataStream& read(char* pch, size_t nSize)
-    {
-        // Read from the beginning of the buffer
-        unsigned int nReadPosNext = nReadPos + nSize;
-        if (nReadPosNext >= vch.size())
-        {
-            if (nReadPosNext > vch.size())
-            {
-                throw std::ios_base::failure("CDataStream::read() : end of data");
-            }
-            memcpy(pch, &vch[nReadPos], nSize);
-            nReadPos = 0;
-            vch.clear();
-            return (*this);
-        }
-        memcpy(pch, &vch[nReadPos], nSize);
-        nReadPos = nReadPosNext;
-        return (*this);
-    }
-
-    CDataStream& ignore(int nSize)
-    {
-        // Ignore from the beginning of the buffer
-        assert(nSize >= 0);
-        unsigned int nReadPosNext = nReadPos + nSize;
-        if (nReadPosNext >= vch.size())
-        {
-            if (nReadPosNext > vch.size())
-                throw std::ios_base::failure("CDataStream::ignore() : end of data");
-            nReadPos = 0;
-            vch.clear();
-            return (*this);
-        }
-        nReadPos = nReadPosNext;
-        return (*this);
-    }
-
-    CDataStream& write(const char* pch, size_t nSize)
-    {
-        // Write to the end of the buffer
-        vch.insert(vch.end(), pch, pch + nSize);
-        return (*this);
-    }
-
-    template<typename Stream>
-    void Serialize(Stream& s, int nType, int nVersion) const
-    {
-        // Special case: stream << stream concatenates like stream += stream
-        if (!vch.empty())
-            s.write((char*)&vch[0], vch.size() * sizeof(vch[0]));
-    }
-
-    template<typename T>
-    unsigned int GetSerializeSize(const T& obj)
-    {
-        // Tells the size of the object if serialized to this stream
-        return ::GetSerializeSize(obj, nType, nVersion);
-    }
-
-    template<typename T>
-    CDataStream& operator<<(const T& obj)
-    {
-        // Serialize to this stream
-        ::Serialize(*this, obj, nType, nVersion);
-        return (*this);
-    }
-
-    template<typename T>
-    CDataStream& operator>>(T& obj)
-    {
-        // Unserialize from this stream
-        ::Unserialize(*this, obj, nType, nVersion);
-        return (*this);
-    }
-
-    void GetAndClear(CSerializeData &data) {
-        data.insert(data.end(), begin(), end());
-        clear();
-    }
-};
-
-
-
-
-
-
-
-
-
-
-/** Non-refcounted RAII wrapper for FILE*
- *
- * Will automatically close the file when it goes out of scope if not null.
- * If you're returning the file pointer, return file.release().
- * If you need to close the file early, use file.fclose() instead of fclose(file).
- */
-class CAutoFile
-{
-private:
-    // Disallow copies
-    CAutoFile(const CAutoFile&);
-    CAutoFile& operator=(const CAutoFile&);
-
-    int nType;
-    int nVersion;
-	
-    FILE* file;	
-
-public:
-    CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn)
-    {
-        file = filenew;
-        nType = nTypeIn;
-        nVersion = nVersionIn;
-    }
-
-    ~CAutoFile()
-    {
-        fclose();
-    }
-
-    void fclose()
-    {
-        if (file) {
-            ::fclose(file);
-            file = NULL;
-        }
-    }
-
-    /** Get wrapped FILE* with transfer of ownership.
-     * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller
-     * of this function to clean up the returned FILE*.
-     */
-    FILE* release()             { FILE* ret = file; file = NULL; return ret; }
-
-    /** Get wrapped FILE* without transfer of ownership.
-     * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the
-     * CAutoFile outlives use of the passed pointer.
-     */
-    FILE* Get() const           { return file; }
-
-    /** Return true if the wrapped FILE* is NULL, false otherwise.
-     */
-    bool IsNull() const         { return (file == NULL); }
-
-    //
-    // Stream subset
-    //
-    void SetType(int n)          { nType = n; }
-    int GetType()                { return nType; }
-    void SetVersion(int n)       { nVersion = n; }
-    int GetVersion()             { return nVersion; }
-    void ReadVersion()           { *this >> nVersion; }
-    void WriteVersion()          { *this << nVersion; }
-
-    CAutoFile& read(char* pch, size_t nSize)
-    {
-        if (!file)
-            throw std::ios_base::failure("CAutoFile::read : file handle is NULL");
-        if (fread(pch, 1, nSize, file) != nSize)
-            throw std::ios_base::failure(feof(file) ? "CAutoFile::read : end of file" : "CAutoFile::read : fread failed");
-        return (*this);
-    }
-
-    CAutoFile& write(const char* pch, size_t nSize)
-    {
-        if (!file)
-            throw std::ios_base::failure("CAutoFile::write : file handle is NULL");
-        if (fwrite(pch, 1, nSize, file) != nSize)
-            throw std::ios_base::failure("CAutoFile::write : write failed");
-        return (*this);
-    }
-
-    template<typename T>
-    unsigned int GetSerializeSize(const T& obj)
-    {
-        // Tells the size of the object if serialized to this stream
-        return ::GetSerializeSize(obj, nType, nVersion);
-    }
-
-    template<typename T>
-    CAutoFile& operator<<(const T& obj)
-    {
-        // Serialize to this stream
-        if (!file)
-            throw std::ios_base::failure("CAutoFile::operator<< : file handle is NULL");
-        ::Serialize(*this, obj, nType, nVersion);
-        return (*this);
-    }
-
-    template<typename T>
-    CAutoFile& operator>>(T& obj)
-    {
-        // Unserialize from this stream
-        if (!file)
-            throw std::ios_base::failure("CAutoFile::operator>> : file handle is NULL");
-        ::Unserialize(*this, obj, nType, nVersion);
-        return (*this);
-    }
-};
-
-/** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to
- *  deserialize from. It guarantees the ability to rewind a given number of bytes.
- *
- *  Will automatically close the file when it goes out of scope if not null.
- *  If you need to close the file early, use file.fclose() instead of fclose(file).
- */
-class CBufferedFile
-{
-private:
-    // Disallow copies
-    CBufferedFile(const CBufferedFile&);
-    CBufferedFile& operator=(const CBufferedFile&);
-
-    int nType;
-    int nVersion;
-
-    FILE *src;            // source file
-    uint64_t nSrcPos;     // how many bytes have been read from source
-    uint64_t nReadPos;    // how many bytes have been read from this
-    uint64_t nReadLimit;  // up to which position we're allowed to read
-    uint64_t nRewind;     // how many bytes we guarantee to rewind
-    std::vector<char> vchBuf; // the buffer
-
-protected:
-    // read data from the source to fill the buffer
-    bool Fill() {
-        unsigned int pos = nSrcPos % vchBuf.size();
-        unsigned int readNow = vchBuf.size() - pos;
-        unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind;
-        if (nAvail < readNow)
-            readNow = nAvail;
-        if (readNow == 0)
-            return false;
-        size_t read = fread((void*)&vchBuf[pos], 1, readNow, src);
-        if (read == 0) {
-            throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill : end of file" : "CBufferedFile::Fill : fread failed");
-        } else {
-            nSrcPos += read;
-            return true;
-        }
-    }
-
-public:
-    CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) :
-        nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
-    {
-        src = fileIn;
-        nType = nTypeIn;
-        nVersion = nVersionIn;
-    }
-
-    ~CBufferedFile()
-    {
-        fclose();
-    }
-
-    void fclose()
-    {
-        if (src) {
-            ::fclose(src);
-            src = NULL;
-        }
-    }
-
-    // check whether we're at the end of the source file
-    bool eof() const {
-        return nReadPos == nSrcPos && feof(src);
-    }
-
-    // read a number of bytes
-    CBufferedFile& read(char *pch, size_t nSize) {
-        if (nSize + nReadPos > nReadLimit)
-            throw std::ios_base::failure("Read attempted past buffer limit");
-        if (nSize + nRewind > vchBuf.size())
-            throw std::ios_base::failure("Read larger than buffer size");
-        while (nSize > 0) {
-            if (nReadPos == nSrcPos)
-                Fill();
-            unsigned int pos = nReadPos % vchBuf.size();
-            size_t nNow = nSize;
-            if (nNow + pos > vchBuf.size())
-                nNow = vchBuf.size() - pos;
-            if (nNow + nReadPos > nSrcPos)
-                nNow = nSrcPos - nReadPos;
-            memcpy(pch, &vchBuf[pos], nNow);
-            nReadPos += nNow;
-            pch += nNow;
-            nSize -= nNow;
-        }
-        return (*this);
-    }
-
-    // return the current reading position
-    uint64_t GetPos() {
-        return nReadPos;
-    }
-
-    // rewind to a given reading position
-    bool SetPos(uint64_t nPos) {
-        nReadPos = nPos;
-        if (nReadPos + nRewind < nSrcPos) {
-            nReadPos = nSrcPos - nRewind;
-            return false;
-        } else if (nReadPos > nSrcPos) {
-            nReadPos = nSrcPos;
-            return false;
-        } else {
-            return true;
-        }
-    }
-
-    bool Seek(uint64_t nPos) {
-        long nLongPos = nPos;
-        if (nPos != (uint64_t)nLongPos)
-            return false;
-        if (fseek(src, nLongPos, SEEK_SET))
-            return false;
-        nLongPos = ftell(src);
-        nSrcPos = nLongPos;
-        nReadPos = nLongPos;
-        return true;
-    }
-
-    // prevent reading beyond a certain position
-    // no argument removes the limit
-    bool SetLimit(uint64_t nPos = (uint64_t)(-1)) {
-        if (nPos < nReadPos)
-            return false;
-        nReadLimit = nPos;
-        return true;
-    }
-
-    template<typename T>
-    CBufferedFile& operator>>(T& obj) {
-        // Unserialize from this stream
-        ::Unserialize(*this, obj, nType, nVersion);
-        return (*this);
-    }
-
-    // search for a given byte in the stream, and remain positioned on it
-    void FindByte(char ch) {
-        while (true) {
-            if (nReadPos == nSrcPos)
-                Fill();
-            if (vchBuf[nReadPos % vchBuf.size()] == ch)
-                break;
-            nReadPos++;
-        }
-    }
-};
-
 #endif // BITCOIN_SERIALIZE_H
diff --git a/src/streams.h b/src/streams.h
new file mode 100644
index 0000000000..b07b11eb3d
--- /dev/null
+++ b/src/streams.h
@@ -0,0 +1,571 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2013 The Bitcoin developers
+// Distributed under the MIT/X11 software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_STREAMS_H
+#define BITCOIN_STREAMS_H
+
+#include "allocators.h"
+#include "serialize.h"
+
+#include <algorithm>
+#include <assert.h>
+#include <ios>
+#include <limits>
+#include <map>
+#include <set>
+#include <stdint.h>
+#include <string>
+#include <string.h>
+#include <utility>
+#include <vector>
+
+/** Double ended buffer combining vector and stream-like interfaces.
+ *
+ * >> and << read and write unformatted data using the above serialization templates.
+ * Fills with data in linear time; some stringstream implementations take N^2 time.
+ */
+class CDataStream
+{
+protected:
+    typedef CSerializeData vector_type;
+    vector_type vch;
+    unsigned int nReadPos;
+public:
+    int nType;
+    int nVersion;
+
+    typedef vector_type::allocator_type   allocator_type;
+    typedef vector_type::size_type        size_type;
+    typedef vector_type::difference_type  difference_type;
+    typedef vector_type::reference        reference;
+    typedef vector_type::const_reference  const_reference;
+    typedef vector_type::value_type       value_type;
+    typedef vector_type::iterator         iterator;
+    typedef vector_type::const_iterator   const_iterator;
+    typedef vector_type::reverse_iterator reverse_iterator;
+
+    explicit CDataStream(int nTypeIn, int nVersionIn)
+    {
+        Init(nTypeIn, nVersionIn);
+    }
+
+    CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
+    {
+        Init(nTypeIn, nVersionIn);
+    }
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1300
+    CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
+    {
+        Init(nTypeIn, nVersionIn);
+    }
+#endif
+
+    CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
+    {
+        Init(nTypeIn, nVersionIn);
+    }
+
+    CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
+    {
+        Init(nTypeIn, nVersionIn);
+    }
+
+    CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
+    {
+        Init(nTypeIn, nVersionIn);
+    }
+
+    void Init(int nTypeIn, int nVersionIn)
+    {
+        nReadPos = 0;
+        nType = nTypeIn;
+        nVersion = nVersionIn;
+    }
+
+    CDataStream& operator+=(const CDataStream& b)
+    {
+        vch.insert(vch.end(), b.begin(), b.end());
+        return *this;
+    }
+
+    friend CDataStream operator+(const CDataStream& a, const CDataStream& b)
+    {
+        CDataStream ret = a;
+        ret += b;
+        return (ret);
+    }
+
+    std::string str() const
+    {
+        return (std::string(begin(), end()));
+    }
+
+
+    //
+    // Vector subset
+    //
+    const_iterator begin() const                     { return vch.begin() + nReadPos; }
+    iterator begin()                                 { return vch.begin() + nReadPos; }
+    const_iterator end() const                       { return vch.end(); }
+    iterator end()                                   { return vch.end(); }
+    size_type size() const                           { return vch.size() - nReadPos; }
+    bool empty() const                               { return vch.size() == nReadPos; }
+    void resize(size_type n, value_type c=0)         { vch.resize(n + nReadPos, c); }
+    void reserve(size_type n)                        { vch.reserve(n + nReadPos); }
+    const_reference operator[](size_type pos) const  { return vch[pos + nReadPos]; }
+    reference operator[](size_type pos)              { return vch[pos + nReadPos]; }
+    void clear()                                     { vch.clear(); nReadPos = 0; }
+    iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); }
+    void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); }
+
+    void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last)
+    {
+        assert(last - first >= 0);
+        if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
+        {
+            // special case for inserting at the front when there's room
+            nReadPos -= (last - first);
+            memcpy(&vch[nReadPos], &first[0], last - first);
+        }
+        else
+            vch.insert(it, first, last);
+    }
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1300
+    void insert(iterator it, const char* first, const char* last)
+    {
+        assert(last - first >= 0);
+        if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
+        {
+            // special case for inserting at the front when there's room
+            nReadPos -= (last - first);
+            memcpy(&vch[nReadPos], &first[0], last - first);
+        }
+        else
+            vch.insert(it, first, last);
+    }
+#endif
+
+    iterator erase(iterator it)
+    {
+        if (it == vch.begin() + nReadPos)
+        {
+            // special case for erasing from the front
+            if (++nReadPos >= vch.size())
+            {
+                // whenever we reach the end, we take the opportunity to clear the buffer
+                nReadPos = 0;
+                return vch.erase(vch.begin(), vch.end());
+            }
+            return vch.begin() + nReadPos;
+        }
+        else
+            return vch.erase(it);
+    }
+
+    iterator erase(iterator first, iterator last)
+    {
+        if (first == vch.begin() + nReadPos)
+        {
+            // special case for erasing from the front
+            if (last == vch.end())
+            {
+                nReadPos = 0;
+                return vch.erase(vch.begin(), vch.end());
+            }
+            else
+            {
+                nReadPos = (last - vch.begin());
+                return last;
+            }
+        }
+        else
+            return vch.erase(first, last);
+    }
+
+    inline void Compact()
+    {
+        vch.erase(vch.begin(), vch.begin() + nReadPos);
+        nReadPos = 0;
+    }
+
+    bool Rewind(size_type n)
+    {
+        // Rewind by n characters if the buffer hasn't been compacted yet
+        if (n > nReadPos)
+            return false;
+        nReadPos -= n;
+        return true;
+    }
+
+
+    //
+    // Stream subset
+    //
+    bool eof() const             { return size() == 0; }
+    CDataStream* rdbuf()         { return this; }
+    int in_avail()               { return size(); }
+
+    void SetType(int n)          { nType = n; }
+    int GetType()                { return nType; }
+    void SetVersion(int n)       { nVersion = n; }
+    int GetVersion()             { return nVersion; }
+    void ReadVersion()           { *this >> nVersion; }
+    void WriteVersion()          { *this << nVersion; }
+
+    CDataStream& read(char* pch, size_t nSize)
+    {
+        // Read from the beginning of the buffer
+        unsigned int nReadPosNext = nReadPos + nSize;
+        if (nReadPosNext >= vch.size())
+        {
+            if (nReadPosNext > vch.size())
+            {
+                throw std::ios_base::failure("CDataStream::read() : end of data");
+            }
+            memcpy(pch, &vch[nReadPos], nSize);
+            nReadPos = 0;
+            vch.clear();
+            return (*this);
+        }
+        memcpy(pch, &vch[nReadPos], nSize);
+        nReadPos = nReadPosNext;
+        return (*this);
+    }
+
+    CDataStream& ignore(int nSize)
+    {
+        // Ignore from the beginning of the buffer
+        assert(nSize >= 0);
+        unsigned int nReadPosNext = nReadPos + nSize;
+        if (nReadPosNext >= vch.size())
+        {
+            if (nReadPosNext > vch.size())
+                throw std::ios_base::failure("CDataStream::ignore() : end of data");
+            nReadPos = 0;
+            vch.clear();
+            return (*this);
+        }
+        nReadPos = nReadPosNext;
+        return (*this);
+    }
+
+    CDataStream& write(const char* pch, size_t nSize)
+    {
+        // Write to the end of the buffer
+        vch.insert(vch.end(), pch, pch + nSize);
+        return (*this);
+    }
+
+    template<typename Stream>
+    void Serialize(Stream& s, int nType, int nVersion) const
+    {
+        // Special case: stream << stream concatenates like stream += stream
+        if (!vch.empty())
+            s.write((char*)&vch[0], vch.size() * sizeof(vch[0]));
+    }
+
+    template<typename T>
+    unsigned int GetSerializeSize(const T& obj)
+    {
+        // Tells the size of the object if serialized to this stream
+        return ::GetSerializeSize(obj, nType, nVersion);
+    }
+
+    template<typename T>
+    CDataStream& operator<<(const T& obj)
+    {
+        // Serialize to this stream
+        ::Serialize(*this, obj, nType, nVersion);
+        return (*this);
+    }
+
+    template<typename T>
+    CDataStream& operator>>(T& obj)
+    {
+        // Unserialize from this stream
+        ::Unserialize(*this, obj, nType, nVersion);
+        return (*this);
+    }
+
+    void GetAndClear(CSerializeData &data) {
+        data.insert(data.end(), begin(), end());
+        clear();
+    }
+};
+
+
+
+
+
+
+
+
+
+
+/** Non-refcounted RAII wrapper for FILE*
+ *
+ * Will automatically close the file when it goes out of scope if not null.
+ * If you're returning the file pointer, return file.release().
+ * If you need to close the file early, use file.fclose() instead of fclose(file).
+ */
+class CAutoFile
+{
+private:
+    // Disallow copies
+    CAutoFile(const CAutoFile&);
+    CAutoFile& operator=(const CAutoFile&);
+
+    int nType;
+    int nVersion;
+	
+    FILE* file;	
+
+public:
+    CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn)
+    {
+        file = filenew;
+        nType = nTypeIn;
+        nVersion = nVersionIn;
+    }
+
+    ~CAutoFile()
+    {
+        fclose();
+    }
+
+    void fclose()
+    {
+        if (file) {
+            ::fclose(file);
+            file = NULL;
+        }
+    }
+
+    /** Get wrapped FILE* with transfer of ownership.
+     * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller
+     * of this function to clean up the returned FILE*.
+     */
+    FILE* release()             { FILE* ret = file; file = NULL; return ret; }
+
+    /** Get wrapped FILE* without transfer of ownership.
+     * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the
+     * CAutoFile outlives use of the passed pointer.
+     */
+    FILE* Get() const           { return file; }
+
+    /** Return true if the wrapped FILE* is NULL, false otherwise.
+     */
+    bool IsNull() const         { return (file == NULL); }
+
+    //
+    // Stream subset
+    //
+    void SetType(int n)          { nType = n; }
+    int GetType()                { return nType; }
+    void SetVersion(int n)       { nVersion = n; }
+    int GetVersion()             { return nVersion; }
+    void ReadVersion()           { *this >> nVersion; }
+    void WriteVersion()          { *this << nVersion; }
+
+    CAutoFile& read(char* pch, size_t nSize)
+    {
+        if (!file)
+            throw std::ios_base::failure("CAutoFile::read : file handle is NULL");
+        if (fread(pch, 1, nSize, file) != nSize)
+            throw std::ios_base::failure(feof(file) ? "CAutoFile::read : end of file" : "CAutoFile::read : fread failed");
+        return (*this);
+    }
+
+    CAutoFile& write(const char* pch, size_t nSize)
+    {
+        if (!file)
+            throw std::ios_base::failure("CAutoFile::write : file handle is NULL");
+        if (fwrite(pch, 1, nSize, file) != nSize)
+            throw std::ios_base::failure("CAutoFile::write : write failed");
+        return (*this);
+    }
+
+    template<typename T>
+    unsigned int GetSerializeSize(const T& obj)
+    {
+        // Tells the size of the object if serialized to this stream
+        return ::GetSerializeSize(obj, nType, nVersion);
+    }
+
+    template<typename T>
+    CAutoFile& operator<<(const T& obj)
+    {
+        // Serialize to this stream
+        if (!file)
+            throw std::ios_base::failure("CAutoFile::operator<< : file handle is NULL");
+        ::Serialize(*this, obj, nType, nVersion);
+        return (*this);
+    }
+
+    template<typename T>
+    CAutoFile& operator>>(T& obj)
+    {
+        // Unserialize from this stream
+        if (!file)
+            throw std::ios_base::failure("CAutoFile::operator>> : file handle is NULL");
+        ::Unserialize(*this, obj, nType, nVersion);
+        return (*this);
+    }
+};
+
+/** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to
+ *  deserialize from. It guarantees the ability to rewind a given number of bytes.
+ *
+ *  Will automatically close the file when it goes out of scope if not null.
+ *  If you need to close the file early, use file.fclose() instead of fclose(file).
+ */
+class CBufferedFile
+{
+private:
+    // Disallow copies
+    CBufferedFile(const CBufferedFile&);
+    CBufferedFile& operator=(const CBufferedFile&);
+
+    int nType;
+    int nVersion;
+
+    FILE *src;            // source file
+    uint64_t nSrcPos;     // how many bytes have been read from source
+    uint64_t nReadPos;    // how many bytes have been read from this
+    uint64_t nReadLimit;  // up to which position we're allowed to read
+    uint64_t nRewind;     // how many bytes we guarantee to rewind
+    std::vector<char> vchBuf; // the buffer
+
+protected:
+    // read data from the source to fill the buffer
+    bool Fill() {
+        unsigned int pos = nSrcPos % vchBuf.size();
+        unsigned int readNow = vchBuf.size() - pos;
+        unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind;
+        if (nAvail < readNow)
+            readNow = nAvail;
+        if (readNow == 0)
+            return false;
+        size_t read = fread((void*)&vchBuf[pos], 1, readNow, src);
+        if (read == 0) {
+            throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill : end of file" : "CBufferedFile::Fill : fread failed");
+        } else {
+            nSrcPos += read;
+            return true;
+        }
+    }
+
+public:
+    CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) :
+        nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
+    {
+        src = fileIn;
+        nType = nTypeIn;
+        nVersion = nVersionIn;
+    }
+
+    ~CBufferedFile()
+    {
+        fclose();
+    }
+
+    void fclose()
+    {
+        if (src) {
+            ::fclose(src);
+            src = NULL;
+        }
+    }
+
+    // check whether we're at the end of the source file
+    bool eof() const {
+        return nReadPos == nSrcPos && feof(src);
+    }
+
+    // read a number of bytes
+    CBufferedFile& read(char *pch, size_t nSize) {
+        if (nSize + nReadPos > nReadLimit)
+            throw std::ios_base::failure("Read attempted past buffer limit");
+        if (nSize + nRewind > vchBuf.size())
+            throw std::ios_base::failure("Read larger than buffer size");
+        while (nSize > 0) {
+            if (nReadPos == nSrcPos)
+                Fill();
+            unsigned int pos = nReadPos % vchBuf.size();
+            size_t nNow = nSize;
+            if (nNow + pos > vchBuf.size())
+                nNow = vchBuf.size() - pos;
+            if (nNow + nReadPos > nSrcPos)
+                nNow = nSrcPos - nReadPos;
+            memcpy(pch, &vchBuf[pos], nNow);
+            nReadPos += nNow;
+            pch += nNow;
+            nSize -= nNow;
+        }
+        return (*this);
+    }
+
+    // return the current reading position
+    uint64_t GetPos() {
+        return nReadPos;
+    }
+
+    // rewind to a given reading position
+    bool SetPos(uint64_t nPos) {
+        nReadPos = nPos;
+        if (nReadPos + nRewind < nSrcPos) {
+            nReadPos = nSrcPos - nRewind;
+            return false;
+        } else if (nReadPos > nSrcPos) {
+            nReadPos = nSrcPos;
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    bool Seek(uint64_t nPos) {
+        long nLongPos = nPos;
+        if (nPos != (uint64_t)nLongPos)
+            return false;
+        if (fseek(src, nLongPos, SEEK_SET))
+            return false;
+        nLongPos = ftell(src);
+        nSrcPos = nLongPos;
+        nReadPos = nLongPos;
+        return true;
+    }
+
+    // prevent reading beyond a certain position
+    // no argument removes the limit
+    bool SetLimit(uint64_t nPos = (uint64_t)(-1)) {
+        if (nPos < nReadPos)
+            return false;
+        nReadLimit = nPos;
+        return true;
+    }
+
+    template<typename T>
+    CBufferedFile& operator>>(T& obj) {
+        // Unserialize from this stream
+        ::Unserialize(*this, obj, nType, nVersion);
+        return (*this);
+    }
+
+    // search for a given byte in the stream, and remain positioned on it
+    void FindByte(char ch) {
+        while (true) {
+            if (nReadPos == nSrcPos)
+                Fill();
+            if (vchBuf[nReadPos % vchBuf.size()] == ch)
+                break;
+            nReadPos++;
+        }
+    }
+};
+
+#endif // BITCOIN_STREAMS_H
diff --git a/src/test/alert_tests.cpp b/src/test/alert_tests.cpp
index 4af87cf8ef..28610f0d2e 100644
--- a/src/test/alert_tests.cpp
+++ b/src/test/alert_tests.cpp
@@ -10,6 +10,7 @@
 #include "data/alertTests.raw.h"
 
 #include "serialize.h"
+#include "streams.h"
 #include "util.h"
 #include "utilstrencodings.h"
 #include "version.h"
diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp
index 867a7df888..59e95f2fd1 100644
--- a/src/test/serialize_tests.cpp
+++ b/src/test/serialize_tests.cpp
@@ -3,6 +3,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "serialize.h"
+#include "streams.h"
 
 #include <stdint.h>
 
diff --git a/src/txmempool.cpp b/src/txmempool.cpp
index 4522c63617..b0d6b4aefa 100644
--- a/src/txmempool.cpp
+++ b/src/txmempool.cpp
@@ -6,6 +6,7 @@
 #include "txmempool.h"
 
 #include "core.h"
+#include "streams.h"
 #include "util.h"
 #include "utilmoneystr.h"
 #include "version.h"
diff --git a/src/txmempool.h b/src/txmempool.h
index ad190eea9d..85cf5310ff 100644
--- a/src/txmempool.h
+++ b/src/txmempool.h
@@ -12,6 +12,8 @@
 #include "core.h"
 #include "sync.h"
 
+class CAutoFile;
+
 inline bool AllowFree(double dPriority)
 {
     // Large (in bytes) low-priority (new, small-coin) transactions