One file per block

Refactor of the block storage code, which now stores one file per block.
This will allow easier pruning, as blocks can be removed individually.

1 files changed, 118 insertions, 43 deletions

diff --git a/src/main.h b/src/main.h
index 0c85286391..a1d6334746 100644
--- a/src/main.h
+++ b/src/main.h
@@ -80,14 +80,14 @@ static const uint64 nMinDiskSpace = 52428800;
 class CReserveKey;
 class CTxDB;
 class CTxIndex;
+class CDiskBlockPos;
 
 void RegisterWallet(CWallet* pwalletIn);
 void UnregisterWallet(CWallet* pwalletIn);
 void SyncWithWallets(const CTransaction& tx, const CBlock* pblock = NULL, bool fUpdate = false);
 bool ProcessBlock(CNode* pfrom, CBlock* pblock);
 bool CheckDiskSpace(uint64 nAdditionalBytes=0);
-FILE* OpenBlockFile(unsigned int nFile, unsigned int nBlockPos, const char* pszMode="rb");
-FILE* AppendBlockFile(unsigned int& nFileRet);
+FILE* OpenBlockFile(const CDiskBlockPos &pos, const char* pszMode="rb");
 bool LoadBlockIndex(bool fAllowNew=true);
 void PrintBlockTree();
 CBlockIndex* FindBlockByHeight(int nHeight);
@@ -118,34 +118,115 @@ bool GetTransaction(const uint256 &hash, CTransaction &tx, uint256 &hashBlock);
 
 bool GetWalletFile(CWallet* pwallet, std::string &strWalletFileOut);
 
+class CDiskBlockPos
+{
+public:
+    int nHeight;
+    int nAlternative;
+
+    CDiskBlockPos() {
+        SetNull();
+    }
+
+    CDiskBlockPos(int nHeightIn, int nAlternativeIn = 0) {
+        nHeight = nHeightIn;
+        nAlternative = nAlternativeIn;
+    }
+
+    std::string GetAlternative() const {
+        char c[9]={0,0,0,0,0,0,0,0,0};
+        char *cp = &c[8];
+        unsigned int n = nAlternative;
+        while (n > 0 && cp>c) {
+            n--;
+            *(--cp) = 'a' + (n % 26);
+            n /= 26;
+        }
+        return std::string(cp);
+    }
+
+    boost::filesystem::path GetDirectory(const boost::filesystem::path &base) const {
+        assert(nHeight != -1);
+        return base / strprintf("era%02u", nHeight / 210000) / 
+                      strprintf("cycle%04u", nHeight / 2016);
+    }
+
+    boost::filesystem::path GetFileName(const boost::filesystem::path &base) const {
+        return GetDirectory(base) / strprintf("%08u%s.blk", nHeight, GetAlternative().c_str());
+    }
+
+    // TODO: make thread-safe (lockfile, atomic file creation, ...?)
+    void MakeUnique(const boost::filesystem::path &base) {
+        while (boost::filesystem::exists(GetFileName(base)))
+            nAlternative++;
+    }
+
+    IMPLEMENT_SERIALIZE(({
+        CDiskBlockPos *me = const_cast<CDiskBlockPos*>(this);
+        if (!fRead) {
+            unsigned int nCode = (nHeight + 1) * 2 + (nAlternative > 0);
+            READWRITE(VARINT(nCode));
+            if (nAlternative > 0) {
+                unsigned int nAlt = nAlternative - 1;
+                READWRITE(VARINT(nAlt));
+            }
+        } else {
+            unsigned int nCode = 0;
+            READWRITE(VARINT(nCode));
+            me->nHeight = (nCode / 2) - 1;
+            if (nCode & 1) {
+                unsigned int nAlt = 0;
+                READWRITE(VARINT(nAlt));
+                me->nAlternative = 1 + nAlt;
+            } else {
+                me->nAlternative = 0;
+            }
+        }
+    });)
+
+    friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) {
+        return ((a.nHeight == b.nHeight) && (a.nAlternative == b.nAlternative));
+    }
+
+    friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) {
+        return !(a == b);
+    }
+
+    void SetNull() { nHeight = -1; nAlternative = 0; }
+    bool IsNull() const { return ((nHeight == -1) && (nAlternative == 0)); }
+
+    void SetMemPool() { nHeight = -1; nAlternative = -1; }
+    bool IsMemPool() const { return ((nHeight == -1) && (nAlternative == -1)); }
+};
+
 /** Position on disk for a particular transaction. */
 class CDiskTxPos
 {
 public:
-    unsigned int nFile;
-    unsigned int nBlockPos;
+    CDiskBlockPos blockPos;
     unsigned int nTxPos;
 
-    CDiskTxPos()
+    CDiskTxPos(bool fInMemPool = false)
     {
         SetNull();
+        if (fInMemPool)
+            blockPos.SetMemPool();
     }
 
-    CDiskTxPos(unsigned int nFileIn, unsigned int nBlockPosIn, unsigned int nTxPosIn)
-    {
-        nFile = nFileIn;
-        nBlockPos = nBlockPosIn;
-        nTxPos = nTxPosIn;
-    }
+    CDiskTxPos(const CDiskBlockPos &block, unsigned int nTxPosIn) : blockPos(block), nTxPos(nTxPosIn) { }
 
-    IMPLEMENT_SERIALIZE( READWRITE(FLATDATA(*this)); )
-    void SetNull() { nFile = (unsigned int) -1; nBlockPos = 0; nTxPos = 0; }
-    bool IsNull() const { return (nFile == (unsigned int) -1); }
+    IMPLEMENT_SERIALIZE( 
+        READWRITE(blockPos);
+        READWRITE(VARINT(nTxPos));
+    )
+
+    void SetNull() { blockPos.SetNull(); nTxPos = 0; }
+    bool IsNull() const { return blockPos.IsNull(); }
+    bool IsMemPool() const { return blockPos.IsMemPool(); }
 
     friend bool operator==(const CDiskTxPos& a, const CDiskTxPos& b)
     {
-        return (a.nFile     == b.nFile &&
-                a.nBlockPos == b.nBlockPos &&
+        return (a.blockPos  == b.blockPos &&
                 a.nTxPos    == b.nTxPos);
     }
 
@@ -158,8 +239,10 @@ public:
     {
         if (IsNull())
             return "null";
+        else if (blockPos.IsMemPool())
+            return "mempool";
         else
-            return strprintf("(nFile=%u, nBlockPos=%u, nTxPos=%u)", nFile, nBlockPos, nTxPos);
+            return strprintf("(%s, nTxPos=%u)", blockPos.GetFileName("").string().c_str(), nTxPos);
     }
 
     void print() const
@@ -545,7 +628,7 @@ public:
 
     bool ReadFromDisk(CDiskTxPos pos, FILE** pfileRet=NULL)
     {
-        CAutoFile filein = CAutoFile(OpenBlockFile(pos.nFile, 0, pfileRet ? "rb+" : "rb"), SER_DISK, CLIENT_VERSION);
+        CAutoFile filein = CAutoFile(OpenBlockFile(pos.blockPos, pfileRet ? "rb+" : "rb"), SER_DISK, CLIENT_VERSION);
         if (!filein)
             return error("CTransaction::ReadFromDisk() : OpenBlockFile failed");
 
@@ -1215,22 +1298,15 @@ public:
     }
 
 
-    bool WriteToDisk(unsigned int& nFileRet, unsigned int& nBlockPosRet)
+    bool WriteToDisk(CDiskBlockPos &pos)
     {
         // Open history file to append
-        CAutoFile fileout = CAutoFile(AppendBlockFile(nFileRet), SER_DISK, CLIENT_VERSION);
+        pos.MakeUnique(GetDataDir());
+        CAutoFile fileout = CAutoFile(OpenBlockFile(pos, "ab"), SER_DISK, CLIENT_VERSION);
         if (!fileout)
             return error("CBlock::WriteToDisk() : AppendBlockFile failed");
 
-        // Write index header
-        unsigned int nSize = fileout.GetSerializeSize(*this);
-        fileout << FLATDATA(pchMessageStart) << nSize;
-
         // Write block
-        long fileOutPos = ftell(fileout);
-        if (fileOutPos < 0)
-            return error("CBlock::WriteToDisk() : ftell failed");
-        nBlockPosRet = fileOutPos;
         fileout << *this;
 
         // Flush stdio buffers and commit to disk before returning
@@ -1241,12 +1317,12 @@ public:
         return true;
     }
 
-    bool ReadFromDisk(unsigned int nFile, unsigned int nBlockPos, bool fReadTransactions=true)
+    bool ReadFromDisk(const CDiskBlockPos &pos, bool fReadTransactions = true)
     {
         SetNull();
 
         // Open history file to read
-        CAutoFile filein = CAutoFile(OpenBlockFile(nFile, nBlockPos, "rb"), SER_DISK, CLIENT_VERSION);
+        CAutoFile filein = CAutoFile(OpenBlockFile(pos, "rb"), SER_DISK, CLIENT_VERSION);
         if (!filein)
             return error("CBlock::ReadFromDisk() : OpenBlockFile failed");
         if (!fReadTransactions)
@@ -1294,7 +1370,7 @@ public:
     bool ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck=false);
     bool ReadFromDisk(const CBlockIndex* pindex, bool fReadTransactions=true);
     bool SetBestChain(CTxDB& txdb, CBlockIndex* pindexNew);
-    bool AddToBlockIndex(unsigned int nFile, unsigned int nBlockPos);
+    bool AddToBlockIndex(const CDiskBlockPos &pos);
     bool CheckBlock(bool fCheckPOW=true, bool fCheckMerkleRoot=true) const;
     bool AcceptBlock();
 
@@ -1320,9 +1396,8 @@ public:
     const uint256* phashBlock;
     CBlockIndex* pprev;
     CBlockIndex* pnext;
-    unsigned int nFile;
-    unsigned int nBlockPos;
     int nHeight;
+    unsigned int nAlternative;
     CBigNum bnChainWork;
 
     // block header
@@ -1338,10 +1413,9 @@ public:
         phashBlock = NULL;
         pprev = NULL;
         pnext = NULL;
-        nFile = 0;
-        nBlockPos = 0;
         nHeight = 0;
         bnChainWork = 0;
+        nAlternative = 0;
 
         nVersion       = 0;
         hashMerkleRoot = 0;
@@ -1350,15 +1424,14 @@ public:
         nNonce         = 0;
     }
 
-    CBlockIndex(unsigned int nFileIn, unsigned int nBlockPosIn, CBlock& block)
+    CBlockIndex(CBlock& block)
     {
         phashBlock = NULL;
         pprev = NULL;
         pnext = NULL;
-        nFile = nFileIn;
-        nBlockPos = nBlockPosIn;
         nHeight = 0;
         bnChainWork = 0;
+        nAlternative = 0;
 
         nVersion       = block.nVersion;
         hashMerkleRoot = block.hashMerkleRoot;
@@ -1367,6 +1440,10 @@ public:
         nNonce         = block.nNonce;
     }
 
+    CDiskBlockPos GetBlockPos() const {
+        return CDiskBlockPos(nHeight, nAlternative);
+    }
+
     CBlock GetBlockHeader() const
     {
         CBlock block;
@@ -1444,11 +1521,10 @@ public:
     static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart,
                                 unsigned int nRequired, unsigned int nToCheck);
 
-
     std::string ToString() const
     {
-        return strprintf("CBlockIndex(pprev=%p, pnext=%p, nFile=%u, nBlockPos=%-6u nHeight=%d, merkle=%s, hashBlock=%s)",
-            pprev, pnext, nFile, nBlockPos, nHeight,
+        return strprintf("CBlockIndex(pprev=%p, pnext=%p, nHeight=%d, merkle=%s, hashBlock=%s)",
+            pprev, pnext, nHeight,
             hashMerkleRoot.ToString().substr(0,10).c_str(),
             GetBlockHash().ToString().substr(0,20).c_str());
     }
@@ -1486,9 +1562,8 @@ public:
             READWRITE(nVersion);
 
         READWRITE(hashNext);
-        READWRITE(nFile);
-        READWRITE(nBlockPos);
         READWRITE(nHeight);
+        READWRITE(nAlternative);
 
         // block header
         READWRITE(this->nVersion);