diff options
| -rw-r--r-- | src/Makefile.am | 2 | ||||
| -rw-r--r-- | src/Makefile.test.include | 1 | ||||
| -rw-r--r-- | src/chain.h | 49 | ||||
| -rw-r--r-- | src/flatfile.cpp | 98 | ||||
| -rw-r--r-- | src/flatfile.h | 96 | ||||
| -rw-r--r-- | src/index/txindex.cpp | 8 | ||||
| -rw-r--r-- | src/init.cpp | 8 | ||||
| -rw-r--r-- | src/test/flatfile_tests.cpp | 123 | ||||
| -rw-r--r-- | src/util/system.cpp | 8 | ||||
| -rw-r--r-- | src/util/system.h | 1 | ||||
| -rw-r--r-- | src/validation.cpp | 189 | ||||
| -rw-r--r-- | src/validation.h | 15 |
12 files changed, 417 insertions, 181 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index d491530ca1..2e297d3dac 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -128,6 +128,7 @@ BITCOIN_CORE_H = \ core_io.h \ core_memusage.h \ cuckoocache.h \ + flatfile.h \ fs.h \ httprpc.h \ httpserver.h \ @@ -247,6 +248,7 @@ libbitcoin_server_a_SOURCES = \ chain.cpp \ checkpoints.cpp \ consensus/tx_verify.cpp \ + flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 84bc326cfe..09fef5a1dd 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -83,6 +83,7 @@ BITCOIN_TESTS =\ test/cuckoocache_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ + test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ diff --git a/src/chain.h b/src/chain.h index 5a6f10b84f..2b6d2d082c 100644 --- a/src/chain.h +++ b/src/chain.h @@ -8,6 +8,7 @@ #include <arith_uint256.h> #include <consensus/params.h> +#include <flatfile.h> #include <primitives/block.h> #include <tinyformat.h> #include <uint256.h> @@ -90,46 +91,6 @@ public: } }; -struct CDiskBlockPos -{ - int nFile; - unsigned int nPos; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); - } - - CDiskBlockPos() { - SetNull(); - } - - CDiskBlockPos(int nFileIn, unsigned int nPosIn) { - nFile = nFileIn; - nPos = nPosIn; - } - - friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return (a.nFile == b.nFile && a.nPos == b.nPos); - } - - friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return !(a == b); - } - - void SetNull() { nFile = -1; nPos = 0; } - bool IsNull() const { return (nFile == -1); } - - std::string ToString() const - { - return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos); - } - -}; - enum BlockStatus: uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, @@ -266,8 +227,8 @@ public: nNonce = block.nNonce; } - CDiskBlockPos GetBlockPos() const { - CDiskBlockPos ret; + FlatFilePos GetBlockPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_DATA) { ret.nFile = nFile; ret.nPos = nDataPos; @@ -275,8 +236,8 @@ public: return ret; } - CDiskBlockPos GetUndoPos() const { - CDiskBlockPos ret; + FlatFilePos GetUndoPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_UNDO) { ret.nFile = nFile; ret.nPos = nUndoPos; diff --git a/src/flatfile.cpp b/src/flatfile.cpp new file mode 100644 index 0000000000..8a8f7b681c --- /dev/null +++ b/src/flatfile.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <stdexcept> + +#include <flatfile.h> +#include <logging.h> +#include <tinyformat.h> +#include <util/system.h> + +FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : + m_dir(std::move(dir)), + m_prefix(prefix), + m_chunk_size(chunk_size) +{ + if (chunk_size == 0) { + throw std::invalid_argument("chunk_size must be positive"); + } +} + +std::string FlatFilePos::ToString() const +{ + return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos); +} + +fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const +{ + return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); +} + +FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) +{ + if (pos.IsNull()) { + return nullptr; + } + fs::path path = FileName(pos); + fs::create_directories(path.parent_path()); + FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+"); + if (!file && !read_only) + file = fsbridge::fopen(path, "wb+"); + if (!file) { + LogPrintf("Unable to open file %s\n", path.string()); + return nullptr; + } + if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) { + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + fclose(file); + return nullptr; + } + return file; +} + +size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) +{ + out_of_space = false; + + unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size; + unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size; + if (n_new_chunks > n_old_chunks) { + size_t old_size = pos.nPos; + size_t new_size = n_new_chunks * m_chunk_size; + size_t inc_size = new_size - old_size; + + if (CheckDiskSpace(m_dir, inc_size)) { + FILE *file = Open(pos); + if (file) { + LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); + AllocateFileRange(file, pos.nPos, inc_size); + fclose(file); + return inc_size; + } + } else { + out_of_space = true; + } + } + return 0; +} + +bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) +{ + FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos + if (!file) { + return error("%s: failed to open file %d", __func__, pos.nFile); + } + if (finalize && !TruncateFile(file, pos.nPos)) { + fclose(file); + return error("%s: failed to truncate file %d", __func__, pos.nFile); + } + if (!FileCommit(file)) { + fclose(file); + return error("%s: failed to commit file %d", __func__, pos.nFile); + } + + fclose(file); + return true; +} diff --git a/src/flatfile.h b/src/flatfile.h new file mode 100644 index 0000000000..374ceff411 --- /dev/null +++ b/src/flatfile.h @@ -0,0 +1,96 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FLATFILE_H +#define BITCOIN_FLATFILE_H + +#include <string> + +#include <fs.h> +#include <serialize.h> + +struct FlatFilePos +{ + int nFile; + unsigned int nPos; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(nPos)); + } + + FlatFilePos() : nFile(-1), nPos(0) {} + + FlatFilePos(int nFileIn, unsigned int nPosIn) : + nFile(nFileIn), + nPos(nPosIn) + {} + + friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { + return (a.nFile == b.nFile && a.nPos == b.nPos); + } + + friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) { + return !(a == b); + } + + void SetNull() { nFile = -1; nPos = 0; } + bool IsNull() const { return (nFile == -1); } + + std::string ToString() const; +}; + +/** + * FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates + * access to and efficient management of these files. + */ +class FlatFileSeq +{ +private: + const fs::path m_dir; + const char* const m_prefix; + const size_t m_chunk_size; + +public: + /** + * Constructor + * + * @param dir The base directory that all files live in. + * @param prefix A short prefix given to all file names. + * @param chunk_size Disk space is pre-allocated in multiples of this amount. + */ + FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size); + + /** Get the name of the file at the given position. */ + fs::path FileName(const FlatFilePos& pos) const; + + /** Open a handle to the file at the given position. */ + FILE* Open(const FlatFilePos& pos, bool read_only = false); + + /** + * Allocate additional space in a file after the given starting position. The amount allocated + * will be the minimum multiple of the sequence chunk size greater than add_size. + * + * @param[in] pos The starting position that bytes will be allocated after. + * @param[in] add_size The minimum number of bytes to be allocated. + * @param[out] out_of_space Whether the allocation failed due to insufficient disk space. + * @return The number of bytes successfully allocated. + */ + size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space); + + /** + * Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final. + * + * @param[in] pos The first unwritten position in the file to be flushed. + * @param[in] finalize True if no more data will be written to this file. + * @return true on success, false on failure. + */ + bool Flush(const FlatFilePos& pos, bool finalize = false); +}; + +#endif // BITCOIN_FLATFILE_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 10bc8419dd..7367ec7cb6 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -16,7 +16,7 @@ constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr<TxIndex> g_txindex; -struct CDiskTxPos : public CDiskBlockPos +struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header @@ -24,11 +24,11 @@ struct CDiskTxPos : public CDiskBlockPos template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); + READWRITEAS(FlatFilePos, *this); READWRITE(VARINT(nTxOffset)); } - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { } CDiskTxPos() { @@ -36,7 +36,7 @@ struct CDiskTxPos : public CDiskBlockPos } void SetNull() { - CDiskBlockPos::SetNull(); + FlatFilePos::SetNull(); nTxOffset = 0; } }; diff --git a/src/init.cpp b/src/init.cpp index e0da962030..ee9a451d0b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -668,8 +668,8 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) if (fReindex) { int nFile = 0; while (true) { - CDiskBlockPos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos, "blk"))) + FlatFilePos pos(nFile, 0); + if (!fs::exists(GetBlockPosFilename(pos))) break; // No block files left to reindex FILE *file = OpenBlockFile(pos, true); if (!file) @@ -1669,11 +1669,11 @@ bool AppInitMain(InitInterfaces& interfaces) // ********************************************************* Step 11: import blocks - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) { + if (!CheckDiskSpace(GetDataDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir())); return false; } - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) { + if (!CheckDiskSpace(GetBlocksDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir())); return false; } diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp new file mode 100644 index 0000000000..079a09f8f9 --- /dev/null +++ b/src/test/flatfile_tests.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <flatfile.h> +#include <test/test_bitcoin.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(flatfile_filename) +{ + auto data_dir = SetDataDir("flatfile_test"); + + FlatFilePos pos(456, 789); + + FlatFileSeq seq1(data_dir, "a", 16 * 1024); + BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat"); + + FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024); + BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat"); +} + +BOOST_AUTO_TEST_CASE(flatfile_open) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 16 * 1024); + + std::string line1("A purely peer-to-peer version of electronic cash would allow online " + "payments to be sent directly from one party to another without going " + "through a financial institution."); + std::string line2("Digital signatures provide part of the solution, but the main benefits are " + "lost if a trusted third party is still required to prevent double-spending."); + + size_t pos1 = 0; + size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION); + + // Write first line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line1, 256); + } + + // Attempt to append to file opened in read-only mode. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure); + } + + // Append second line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line2, 256); + } + + // Read text from file in read-only mode. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line1); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Read text from file with position offset. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Ensure another file in the sequence has no data. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure); + } +} + +BOOST_AUTO_TEST_CASE(flatfile_allocate) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200); + BOOST_CHECK(!out_of_space); +} + +BOOST_AUTO_TEST_CASE(flatfile_flush) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + seq.Allocate(FlatFilePos(0, 0), 1, out_of_space); + + // Flush without finalize should not truncate file. + seq.Flush(FlatFilePos(0, 1)); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100); + + // Flush with finalize should truncate file. + seq.Flush(FlatFilePos(0, 1), true); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/system.cpp b/src/util/system.cpp index 5ae0604894..9594dd81bf 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -135,6 +135,14 @@ bool DirIsWritable(const fs::path& directory) return true; } +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes) +{ + constexpr uint64_t min_disk_space = 52428800; // 50 MiB + + uint64_t free_bytes_available = fs::space(dir).available; + return free_bytes_available >= min_disk_space + additional_bytes; +} + /** * Interpret a string argument as a boolean. * diff --git a/src/util/system.h b/src/util/system.h index 6899e38c9e..54eb88e261 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -72,6 +72,7 @@ bool RenameOver(fs::path src, fs::path dest); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name); bool DirIsWritable(const fs::path& directory); +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); /** Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. diff --git a/src/validation.cpp b/src/validation.cpp index f3d34dca70..358992b74d 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -15,6 +15,7 @@ #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <cuckoocache.h> +#include <flatfile.h> #include <hash.h> #include <index/txindex.h> #include <policy/fees.h> @@ -165,7 +166,7 @@ public: * that it doesn't descend from an invalid block, and then add it to mapBlockIndex. */ bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); @@ -204,7 +205,7 @@ private: void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -317,7 +318,9 @@ static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState & static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CTransaction &tx, int flags) { @@ -1042,7 +1045,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus // CBlock and CBlockIndex // -static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) +static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -1063,7 +1066,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes return true; } -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); @@ -1089,7 +1092,7 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus: bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - CDiskBlockPos blockPos; + FlatFilePos blockPos; { LOCK(cs_main); blockPos = pindex->GetBlockPos(); @@ -1103,9 +1106,9 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus return true; } -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start) +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos hpos = pos; + FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { @@ -1140,7 +1143,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos block_pos; + FlatFilePos block_pos; { LOCK(cs_main); block_pos = pindex->GetBlockPos(); @@ -1452,7 +1455,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi namespace { -bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); @@ -1481,7 +1484,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex) { - CDiskBlockPos pos = pindex->GetUndoPos(); + FlatFilePos pos = pindex->GetUndoPos(); if (pos.IsNull()) { return error("%s: no undo data available", __func__); } @@ -1627,37 +1630,24 @@ void static FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); - CDiskBlockPos posOld(nLastBlockFile, 0); - bool status = true; - - FILE *fileOld = OpenBlockFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } - - fileOld = OpenUndoFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); + bool status = true; + status &= BlockFileSeq().Flush(block_pos_old, fFinalize); + status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); if (!status) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) { // Write undo information to disk if (pindex->GetUndoPos().IsNull()) { - CDiskBlockPos _pos; + FlatFilePos _pos; if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) return error("ConnectBlock(): FindUndoPos failed"); if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) @@ -2134,8 +2124,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index - if (!CheckDiskSpace(0, true)) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetBlocksDir())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // First make sure all block and undo data is flushed to disk. FlushBlockFile(); // Then update all block file information (which may refer to block and undo files). @@ -2168,8 +2159,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * pcoinsTip->GetCacheSize())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); @@ -2922,7 +2914,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2968,7 +2960,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -3003,21 +2995,13 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { - unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) { - FILE *file = OpenBlockFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } } @@ -3025,32 +3009,23 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int return true; } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) { pos.nFile = nFile; LOCK(cs_LastBlockFile); - unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; - nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; setDirtyFileInfo.insert(nFile); - unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) { - FILE *file = OpenUndoFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return state.Error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } return true; @@ -3435,26 +3410,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) { +static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); - CDiskBlockPos blockPos; + FlatFilePos blockPos; if (dbp != nullptr) blockPos = *dbp; if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { error("%s: FindBlockPos failed", __func__); - return CDiskBlockPos(); + return FlatFilePos(); } if (dbp == nullptr) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); - return CDiskBlockPos(); + return FlatFilePos(); } } return blockPos; } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3516,7 +3491,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali // Write block to history file if (fNewBlock) *fNewBlock = true; try { - CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); + FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -3647,9 +3622,9 @@ void PruneOneBlockFile(const int fileNumber) void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) { for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - CDiskBlockPos pos(*it, 0); - fs::remove(GetBlockPosFilename(pos, "blk")); - fs::remove(GetBlockPosFilename(pos, "rev")); + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -3757,52 +3732,28 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte nLastBlockWeCanPrune, count); } -bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir) +static FlatFileSeq BlockFileSeq() { - uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available; - - // Check for nMinDiskSpace bytes (currently 50MB) - if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) - return AbortNode("Disk space is low!", _("Error: Disk space is low!")); - - return true; + return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); } -static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) +static FlatFileSeq UndoFileSeq() { - if (pos.IsNull()) - return nullptr; - fs::path path = GetBlockPosFilename(pos, prefix); - fs::create_directories(path.parent_path()); - FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); - if (!file && !fReadOnly) - file = fsbridge::fopen(path, "wb+"); - if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); - return nullptr; - } - if (pos.nPos) { - if (fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); - fclose(file); - return nullptr; - } - } - return file; + return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "blk", fReadOnly); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { + return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "rev", fReadOnly); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { + return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +fs::path GetBlockPosFilename(const FlatFilePos &pos) { - return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile); + return BlockFileSeq().FileName(pos); } CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) @@ -3909,7 +3860,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE } for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { - CDiskBlockPos pos(*it, 0); + FlatFilePos pos(*it, 0); if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { return false; } @@ -4342,7 +4293,7 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) try { const CBlock& block = chainparams.GenesisBlock(); - CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); + FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = AddToBlockIndex(block); @@ -4359,10 +4310,10 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return g_chainstate.LoadGenesisBlock(chainparams); } -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) - static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent; + static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); int nLoaded = 0; @@ -4448,9 +4399,9 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB while (!queue.empty()) { uint256 head = queue.front(); queue.pop_front(); - std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head); + std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head); while (range.first != range.second) { - std::multimap<uint256, CDiskBlockPos>::iterator it = range.first; + std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) { diff --git a/src/validation.h b/src/validation.h index 1975846b69..4ef82ee2c7 100644 --- a/src/validation.h +++ b/src/validation.h @@ -181,9 +181,6 @@ extern arith_uint256 nMinimumChainWork; /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; -/** Minimum disk space required - used in CheckDiskSpace() */ -static const uint64_t nMinDiskSpace = 52428800; - /** Pruning-related variables and constants */ /** True if any block files have ever been pruned. */ extern bool fHavePruned; @@ -245,14 +242,12 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons */ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main); -/** Check whether enough disk space is available for an incoming block */ -bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false); /** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); +fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Import blocks from an external file */ -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = nullptr); +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Load the block tree and coins database from disk, @@ -391,9 +386,9 @@ void InitScriptExecutionCache(); /** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start); +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); /** Functions for validating blocks and updating the block tree */ |