From 62e7addb632cad77cbd5fbccbaee51c7b32505d0 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Sat, 1 Sep 2018 15:18:02 -0700 Subject: util: Move CheckDiskSpace to util. --- src/init.cpp | 4 ++-- src/util/system.cpp | 8 ++++++++ src/util/system.h | 1 + src/validation.cpp | 32 ++++++++++++-------------------- src/validation.h | 5 ----- 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 8b831a726f..5111baddd2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1675,11 +1675,11 @@ bool AppInitMain(InitInterfaces& interfaces) // ********************************************************* Step 11: import blocks - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) { + if (!CheckDiskSpace(GetDataDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir())); return false; } - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) { + if (!CheckDiskSpace(GetBlocksDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir())); return false; } diff --git a/src/util/system.cpp b/src/util/system.cpp index 6e82de743a..3e6c2ae5d4 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -135,6 +135,14 @@ bool DirIsWritable(const fs::path& directory) return true; } +bool CheckDiskSpace(const fs::path& dir, uint64_t nAdditionalBytes) +{ + constexpr uint64_t nMinDiskSpace = 52428800; // 50 MiB + + uint64_t nFreeBytesAvailable = fs::space(dir).available; + return nFreeBytesAvailable >= nMinDiskSpace + nAdditionalBytes; +} + /** * Interpret a string argument as a boolean. * diff --git a/src/util/system.h b/src/util/system.h index 69ae11d1ed..2a2ee9bc1d 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -72,6 +72,7 @@ bool RenameOver(fs::path src, fs::path dest); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name); bool DirIsWritable(const fs::path& directory); +bool CheckDiskSpace(const fs::path& dir, uint64_t nAdditionalBytes = 0); /** Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. diff --git a/src/validation.cpp b/src/validation.cpp index 1806bc1268..06e0e99290 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2134,8 +2134,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index - if (!CheckDiskSpace(0, true)) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetBlocksDir())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // First make sure all block and undo data is flushed to disk. FlushBlockFile(); // Then update all block file information (which may refer to block and undo files). @@ -2168,8 +2169,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * pcoinsTip->GetCacheSize())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); @@ -3014,7 +3016,7 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int if (nNewChunks > nOldChunks) { if (fPruneMode) fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) { + if (CheckDiskSpace(GetBlocksDir(), nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenBlockFile(pos); if (file) { LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); @@ -3023,7 +3025,7 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int } } else - return error("out of disk space"); + return AbortNode("Disk space is low!", _("Error: Disk space is low!")); } } @@ -3047,7 +3049,7 @@ static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, if (nNewChunks > nOldChunks) { if (fPruneMode) fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) { + if (CheckDiskSpace(GetBlocksDir(), nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenUndoFile(pos); if (file) { LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); @@ -3055,8 +3057,9 @@ static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, fclose(file); } } - else - return state.Error("out of disk space"); + else { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } } return true; @@ -3763,17 +3766,6 @@ static void FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfte nLastBlockWeCanPrune, count); } -bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir) -{ - uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available; - - // Check for nMinDiskSpace bytes (currently 50MB) - if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) - return AbortNode("Disk space is low!", _("Error: Disk space is low!")); - - return true; -} - static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { if (pos.IsNull()) diff --git a/src/validation.h b/src/validation.h index 1975846b69..1beee7869f 100644 --- a/src/validation.h +++ b/src/validation.h @@ -181,9 +181,6 @@ extern arith_uint256 nMinimumChainWork; /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; -/** Minimum disk space required - used in CheckDiskSpace() */ -static const uint64_t nMinDiskSpace = 52428800; - /** Pruning-related variables and constants */ /** True if any block files have ever been pruned. */ extern bool fHavePruned; @@ -245,8 +242,6 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main); -/** Check whether enough disk space is available for an incoming block */ -bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false); /** Open a block file (blk?????.dat) */ FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -- cgit v1.2.3 From 9183d6ef656c8f3ed406821b99827f9b5f047665 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Sun, 6 Jan 2019 11:06:31 -0800 Subject: validation: Extract basic block file logic into FlatFileSeq class. --- src/Makefile.am | 2 ++ src/flatfile.cpp | 23 +++++++++++++++++++++++ src/flatfile.h | 36 ++++++++++++++++++++++++++++++++++++ src/init.cpp | 2 +- src/validation.cpp | 21 +++++++++++++++++---- src/validation.h | 2 +- 6 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 src/flatfile.cpp create mode 100644 src/flatfile.h diff --git a/src/Makefile.am b/src/Makefile.am index d491530ca1..2e297d3dac 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -128,6 +128,7 @@ BITCOIN_CORE_H = \ core_io.h \ core_memusage.h \ cuckoocache.h \ + flatfile.h \ fs.h \ httprpc.h \ httpserver.h \ @@ -247,6 +248,7 @@ libbitcoin_server_a_SOURCES = \ chain.cpp \ checkpoints.cpp \ consensus/tx_verify.cpp \ + flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ diff --git a/src/flatfile.cpp b/src/flatfile.cpp new file mode 100644 index 0000000000..c9ca9aa869 --- /dev/null +++ b/src/flatfile.cpp @@ -0,0 +1,23 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include + +FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : + m_dir(std::move(dir)), + m_prefix(prefix), + m_chunk_size(chunk_size) +{ + if (chunk_size == 0) { + throw std::invalid_argument("chunk_size must be positive"); + } +} + +fs::path FlatFileSeq::FileName(const CDiskBlockPos& pos) const +{ + return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); +} diff --git a/src/flatfile.h b/src/flatfile.h new file mode 100644 index 0000000000..9c7131d201 --- /dev/null +++ b/src/flatfile.h @@ -0,0 +1,36 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FLATFILE_H +#define BITCOIN_FLATFILE_H + +#include +#include + +/** + * FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates + * access to and efficient management of these files. + */ +class FlatFileSeq +{ +private: + const fs::path m_dir; + const char* const m_prefix; + const size_t m_chunk_size; + +public: + /** + * Constructor + * + * @param dir The base directory that all files live in. + * @param prefix A short prefix given to all file names. + * @param chunk_size Disk space is pre-allocated in multiples of this amount. + */ + FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size); + + /** Get the name of the file at the given position. */ + fs::path FileName(const CDiskBlockPos& pos) const; +}; + +#endif // BITCOIN_FLATFILE_H diff --git a/src/init.cpp b/src/init.cpp index 5111baddd2..66c5995651 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -673,7 +673,7 @@ static void ThreadImport(std::vector vImportFiles) int nFile = 0; while (true) { CDiskBlockPos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos, "blk"))) + if (!fs::exists(GetBlockPosFilename(pos))) break; // No block files left to reindex FILE *file = OpenBlockFile(pos, true); if (!file) diff --git a/src/validation.cpp b/src/validation.cpp index 06e0e99290..3abe8b2fa1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -318,6 +319,8 @@ static void FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPr static void FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfterHeight); bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector *pvChecks = nullptr); static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CTransaction &tx, int flags) { @@ -3657,8 +3660,8 @@ void UnlinkPrunedFiles(const std::set& setFilesToPrune) { for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { CDiskBlockPos pos(*it, 0); - fs::remove(GetBlockPosFilename(pos, "blk")); - fs::remove(GetBlockPosFilename(pos, "rev")); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -3789,6 +3792,16 @@ static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fRe return file; } +static FlatFileSeq BlockFileSeq() +{ + return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); +} + +static FlatFileSeq UndoFileSeq() +{ + return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE); +} + FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { return OpenDiskFile(pos, "blk", fReadOnly); } @@ -3798,9 +3811,9 @@ static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { return OpenDiskFile(pos, "rev", fReadOnly); } -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +fs::path GetBlockPosFilename(const CDiskBlockPos &pos) { - return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile); + return BlockFileSeq().FileName(pos); } CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) diff --git a/src/validation.h b/src/validation.h index 1beee7869f..19f8e8ab7a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -245,7 +245,7 @@ bool ProcessNewBlockHeaders(const std::vector& block, CValidationS /** Open a block file (blk?????.dat) */ FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); +fs::path GetBlockPosFilename(const CDiskBlockPos &pos); /** Import blocks from an external file */ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ -- cgit v1.2.3 From e2d2abb99fe353ffc2ff3bc1ff578fad31065335 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Sun, 6 Jan 2019 11:14:22 -0800 Subject: validation: Refactor OpenDiskFile into method on FlatFileSeq. --- src/flatfile.cpp | 24 ++++++++++++++++++++++++ src/flatfile.h | 3 +++ src/validation.cpp | 27 ++------------------------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/flatfile.cpp b/src/flatfile.cpp index c9ca9aa869..535f4eda91 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -5,6 +5,7 @@ #include #include +#include #include FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : @@ -21,3 +22,26 @@ fs::path FlatFileSeq::FileName(const CDiskBlockPos& pos) const { return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); } + +FILE* FlatFileSeq::Open(const CDiskBlockPos& pos, bool fReadOnly) +{ + if (pos.IsNull()) + return nullptr; + fs::path path = FileName(pos); + fs::create_directories(path.parent_path()); + FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); + if (!file && !fReadOnly) + file = fsbridge::fopen(path, "wb+"); + if (!file) { + LogPrintf("Unable to open file %s\n", path.string()); + return nullptr; + } + if (pos.nPos) { + if (fseek(file, pos.nPos, SEEK_SET)) { + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + fclose(file); + return nullptr; + } + } + return file; +} diff --git a/src/flatfile.h b/src/flatfile.h index 9c7131d201..b1eea93140 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -31,6 +31,9 @@ public: /** Get the name of the file at the given position. */ fs::path FileName(const CDiskBlockPos& pos) const; + + /** Open a handle to the file at the given position. */ + FILE* Open(const CDiskBlockPos& pos, bool fReadOnly = false); }; #endif // BITCOIN_FLATFILE_H diff --git a/src/validation.cpp b/src/validation.cpp index 3abe8b2fa1..4135063505 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3769,29 +3769,6 @@ static void FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfte nLastBlockWeCanPrune, count); } -static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) -{ - if (pos.IsNull()) - return nullptr; - fs::path path = GetBlockPosFilename(pos, prefix); - fs::create_directories(path.parent_path()); - FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); - if (!file && !fReadOnly) - file = fsbridge::fopen(path, "wb+"); - if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); - return nullptr; - } - if (pos.nPos) { - if (fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); - fclose(file); - return nullptr; - } - } - return file; -} - static FlatFileSeq BlockFileSeq() { return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); @@ -3803,12 +3780,12 @@ static FlatFileSeq UndoFileSeq() } FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "blk", fReadOnly); + return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "rev", fReadOnly); + return UndoFileSeq().Open(pos, fReadOnly); } fs::path GetBlockPosFilename(const CDiskBlockPos &pos) -- cgit v1.2.3 From 992404b31ed2f8cabeed59d074552f0ae10fda94 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Sun, 6 Jan 2019 11:27:31 -0800 Subject: validation: Refactor block file pre-allocation into FlatFileSeq. --- src/flatfile.cpp | 27 +++++++++++++++++++++++++++ src/flatfile.h | 11 +++++++++++ src/validation.cpp | 48 +++++++++++++++--------------------------------- 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 535f4eda91..d9fd4041b7 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -7,6 +7,7 @@ #include #include #include +#include FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : m_dir(std::move(dir)), @@ -45,3 +46,29 @@ FILE* FlatFileSeq::Open(const CDiskBlockPos& pos, bool fReadOnly) } return file; } + +size_t FlatFileSeq::Allocate(const CDiskBlockPos& pos, size_t add_size, bool& out_of_space) +{ + out_of_space = false; + + unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size; + unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size; + if (n_new_chunks > n_old_chunks) { + size_t old_size = pos.nPos; + size_t new_size = n_new_chunks * m_chunk_size; + size_t inc_size = new_size - old_size; + + if (CheckDiskSpace(m_dir, inc_size)) { + FILE *file = Open(pos); + if (file) { + LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); + AllocateFileRange(file, pos.nPos, inc_size); + fclose(file); + return inc_size; + } + } else { + out_of_space = true; + } + } + return 0; +} diff --git a/src/flatfile.h b/src/flatfile.h index b1eea93140..1b5d83e8e6 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -34,6 +34,17 @@ public: /** Open a handle to the file at the given position. */ FILE* Open(const CDiskBlockPos& pos, bool fReadOnly = false); + + /** + * Allocate additional space in a file after the given starting position. The amount allocated + * will be the minimum multiple of the sequence chunk size greater than add_size. + * + * @param[in] pos The starting position that bytes will be allocated after. + * @param[in] add_size The minimum number of bytes to be allocated. + * @param[out] out_of_space Whether the allocation failed due to insufficient disk space. + * @return The number of bytes successfully allocated. + */ + size_t Allocate(const CDiskBlockPos& pos, size_t add_size, bool& out_of_space); }; #endif // BITCOIN_FLATFILE_H diff --git a/src/validation.cpp b/src/validation.cpp index 4135063505..aca555ee0a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3014,21 +3014,13 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { - unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(GetBlocksDir(), nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { - FILE *file = OpenBlockFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return AbortNode("Disk space is low!", _("Error: Disk space is low!")); + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } } @@ -3042,27 +3034,17 @@ static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, LOCK(cs_LastBlockFile); - unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; - nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; setDirtyFileInfo.insert(nFile); - unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(GetBlocksDir(), nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { - FILE *file = OpenUndoFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else { - return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); - } + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } return true; -- cgit v1.2.3 From e0380933e3745214331d358bda8c5e79299c84d2 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Sun, 6 Jan 2019 10:14:35 -0800 Subject: validation: Refactor file flush logic into FlatFileSeq. --- src/flatfile.cpp | 19 +++++++++++++++++++ src/flatfile.h | 9 +++++++++ src/validation.cpp | 23 +++++------------------ 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/flatfile.cpp b/src/flatfile.cpp index d9fd4041b7..1cdead6bf5 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -72,3 +72,22 @@ size_t FlatFileSeq::Allocate(const CDiskBlockPos& pos, size_t add_size, bool& ou } return 0; } + +bool FlatFileSeq::Flush(const CDiskBlockPos& pos, bool finalize) +{ + FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos + if (!file) { + return error("%s: failed to open file %d", __func__, pos.nFile); + } + if (finalize && !TruncateFile(file, pos.nPos)) { + fclose(file); + return error("%s: failed to truncate file %d", __func__, pos.nFile); + } + if (!FileCommit(file)) { + fclose(file); + return error("%s: failed to commit file %d", __func__, pos.nFile); + } + + fclose(file); + return true; +} diff --git a/src/flatfile.h b/src/flatfile.h index 1b5d83e8e6..eea41ddf84 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -45,6 +45,15 @@ public: * @return The number of bytes successfully allocated. */ size_t Allocate(const CDiskBlockPos& pos, size_t add_size, bool& out_of_space); + + /** + * Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final. + * + * @param[in] pos The first unwritten position in the file to be flushed. + * @param[in] finalize True if no more data will be written to this file. + * @return true on success, false on failure. + */ + bool Flush(const CDiskBlockPos& pos, bool finalize = false); }; #endif // BITCOIN_FLATFILE_H diff --git a/src/validation.cpp b/src/validation.cpp index aca555ee0a..98d89599f1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1630,25 +1630,12 @@ void static FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); - CDiskBlockPos posOld(nLastBlockFile, 0); - bool status = true; - - FILE *fileOld = OpenBlockFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } - - fileOld = OpenUndoFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } + CDiskBlockPos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + CDiskBlockPos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); + bool status = true; + status &= BlockFileSeq().Flush(block_pos_old, fFinalize); + status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); if (!status) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } -- cgit v1.2.3 From d6d8a78f26f52fdfe43293686135e2fc6919926c Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Sun, 6 Jan 2019 11:43:38 -0800 Subject: Move CDiskBlockPos from chain to flatfile. --- src/chain.h | 41 +---------------------------------------- src/flatfile.cpp | 8 +++++++- src/flatfile.h | 43 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/chain.h b/src/chain.h index 5a6f10b84f..583c46c58c 100644 --- a/src/chain.h +++ b/src/chain.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -90,46 +91,6 @@ public: } }; -struct CDiskBlockPos -{ - int nFile; - unsigned int nPos; - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); - } - - CDiskBlockPos() { - SetNull(); - } - - CDiskBlockPos(int nFileIn, unsigned int nPosIn) { - nFile = nFileIn; - nPos = nPosIn; - } - - friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return (a.nFile == b.nFile && a.nPos == b.nPos); - } - - friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return !(a == b); - } - - void SetNull() { nFile = -1; nPos = 0; } - bool IsNull() const { return (nFile == -1); } - - std::string ToString() const - { - return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos); - } - -}; - enum BlockStatus: uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 1cdead6bf5..b1ea257da1 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -1,4 +1,5 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,6 +20,11 @@ FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : } } +std::string CDiskBlockPos::ToString() const +{ + return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos); +} + fs::path FlatFileSeq::FileName(const CDiskBlockPos& pos) const { return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); diff --git a/src/flatfile.h b/src/flatfile.h index eea41ddf84..d599b8cc42 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -1,12 +1,51 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_FLATFILE_H #define BITCOIN_FLATFILE_H -#include +#include + #include +#include + +struct CDiskBlockPos +{ + int nFile; + unsigned int nPos; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(nPos)); + } + + CDiskBlockPos() { + SetNull(); + } + + CDiskBlockPos(int nFileIn, unsigned int nPosIn) { + nFile = nFileIn; + nPos = nPosIn; + } + + friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { + return (a.nFile == b.nFile && a.nPos == b.nPos); + } + + friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { + return !(a == b); + } + + void SetNull() { nFile = -1; nPos = 0; } + bool IsNull() const { return (nFile == -1); } + + std::string ToString() const; +}; /** * FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates -- cgit v1.2.3 From 65a489e93d181d3c0f7a9cf79f7c11ff8cf2b0f0 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Sun, 6 Jan 2019 11:46:30 -0800 Subject: scripted-diff: Rename CBlockDiskPos to FlatFilePos. -BEGIN VERIFY SCRIPT- sed -i 's/CDiskBlockPos/FlatFilePos/g' $(git ls-files 'src/*.h' 'src/*.cpp') -END VERIFY SCRIPT- --- src/chain.h | 8 +++--- src/flatfile.cpp | 12 ++++----- src/flatfile.h | 18 +++++++------- src/index/txindex.cpp | 8 +++--- src/init.cpp | 2 +- src/validation.cpp | 68 +++++++++++++++++++++++++-------------------------- src/validation.h | 10 ++++---- 7 files changed, 63 insertions(+), 63 deletions(-) diff --git a/src/chain.h b/src/chain.h index 583c46c58c..2b6d2d082c 100644 --- a/src/chain.h +++ b/src/chain.h @@ -227,8 +227,8 @@ public: nNonce = block.nNonce; } - CDiskBlockPos GetBlockPos() const { - CDiskBlockPos ret; + FlatFilePos GetBlockPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_DATA) { ret.nFile = nFile; ret.nPos = nDataPos; @@ -236,8 +236,8 @@ public: return ret; } - CDiskBlockPos GetUndoPos() const { - CDiskBlockPos ret; + FlatFilePos GetUndoPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_UNDO) { ret.nFile = nFile; ret.nPos = nUndoPos; diff --git a/src/flatfile.cpp b/src/flatfile.cpp index b1ea257da1..d2e11825d5 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -20,17 +20,17 @@ FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : } } -std::string CDiskBlockPos::ToString() const +std::string FlatFilePos::ToString() const { - return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos); + return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos); } -fs::path FlatFileSeq::FileName(const CDiskBlockPos& pos) const +fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const { return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); } -FILE* FlatFileSeq::Open(const CDiskBlockPos& pos, bool fReadOnly) +FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool fReadOnly) { if (pos.IsNull()) return nullptr; @@ -53,7 +53,7 @@ FILE* FlatFileSeq::Open(const CDiskBlockPos& pos, bool fReadOnly) return file; } -size_t FlatFileSeq::Allocate(const CDiskBlockPos& pos, size_t add_size, bool& out_of_space) +size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) { out_of_space = false; @@ -79,7 +79,7 @@ size_t FlatFileSeq::Allocate(const CDiskBlockPos& pos, size_t add_size, bool& ou return 0; } -bool FlatFileSeq::Flush(const CDiskBlockPos& pos, bool finalize) +bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) { FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos if (!file) { diff --git a/src/flatfile.h b/src/flatfile.h index d599b8cc42..6051970490 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -11,7 +11,7 @@ #include #include -struct CDiskBlockPos +struct FlatFilePos { int nFile; unsigned int nPos; @@ -24,20 +24,20 @@ struct CDiskBlockPos READWRITE(VARINT(nPos)); } - CDiskBlockPos() { + FlatFilePos() { SetNull(); } - CDiskBlockPos(int nFileIn, unsigned int nPosIn) { + FlatFilePos(int nFileIn, unsigned int nPosIn) { nFile = nFileIn; nPos = nPosIn; } - friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { + friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { return (a.nFile == b.nFile && a.nPos == b.nPos); } - friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { + friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) { return !(a == b); } @@ -69,10 +69,10 @@ public: FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size); /** Get the name of the file at the given position. */ - fs::path FileName(const CDiskBlockPos& pos) const; + fs::path FileName(const FlatFilePos& pos) const; /** Open a handle to the file at the given position. */ - FILE* Open(const CDiskBlockPos& pos, bool fReadOnly = false); + FILE* Open(const FlatFilePos& pos, bool fReadOnly = false); /** * Allocate additional space in a file after the given starting position. The amount allocated @@ -83,7 +83,7 @@ public: * @param[out] out_of_space Whether the allocation failed due to insufficient disk space. * @return The number of bytes successfully allocated. */ - size_t Allocate(const CDiskBlockPos& pos, size_t add_size, bool& out_of_space); + size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space); /** * Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final. @@ -92,7 +92,7 @@ public: * @param[in] finalize True if no more data will be written to this file. * @return true on success, false on failure. */ - bool Flush(const CDiskBlockPos& pos, bool finalize = false); + bool Flush(const FlatFilePos& pos, bool finalize = false); }; #endif // BITCOIN_FLATFILE_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 10bc8419dd..7367ec7cb6 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -16,7 +16,7 @@ constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr g_txindex; -struct CDiskTxPos : public CDiskBlockPos +struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header @@ -24,11 +24,11 @@ struct CDiskTxPos : public CDiskBlockPos template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); + READWRITEAS(FlatFilePos, *this); READWRITE(VARINT(nTxOffset)); } - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { } CDiskTxPos() { @@ -36,7 +36,7 @@ struct CDiskTxPos : public CDiskBlockPos } void SetNull() { - CDiskBlockPos::SetNull(); + FlatFilePos::SetNull(); nTxOffset = 0; } }; diff --git a/src/init.cpp b/src/init.cpp index 66c5995651..8a849d2815 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -672,7 +672,7 @@ static void ThreadImport(std::vector vImportFiles) if (fReindex) { int nFile = 0; while (true) { - CDiskBlockPos pos(nFile, 0); + FlatFilePos pos(nFile, 0); if (!fs::exists(GetBlockPosFilename(pos))) break; // No block files left to reindex FILE *file = OpenBlockFile(pos, true); diff --git a/src/validation.cpp b/src/validation.cpp index 98d89599f1..86775bb391 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -166,7 +166,7 @@ public: * that it doesn't descend from an invalid block, and then add it to mapBlockIndex. */ bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); @@ -205,7 +205,7 @@ private: void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -318,7 +318,7 @@ static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState & static void FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfterHeight); bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector *pvChecks = nullptr); -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); @@ -1045,7 +1045,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus // CBlock and CBlockIndex // -static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) +static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -1066,7 +1066,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes return true; } -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); @@ -1092,7 +1092,7 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus: bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - CDiskBlockPos blockPos; + FlatFilePos blockPos; { LOCK(cs_main); blockPos = pindex->GetBlockPos(); @@ -1106,9 +1106,9 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus return true; } -bool ReadRawBlockFromDisk(std::vector& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start) +bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos hpos = pos; + FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { @@ -1143,7 +1143,7 @@ bool ReadRawBlockFromDisk(std::vector& block, const CDiskBlockPos& pos, bool ReadRawBlockFromDisk(std::vector& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos block_pos; + FlatFilePos block_pos; { LOCK(cs_main); block_pos = pindex->GetBlockPos(); @@ -1455,7 +1455,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi namespace { -bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); @@ -1484,7 +1484,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex) { - CDiskBlockPos pos = pindex->GetUndoPos(); + FlatFilePos pos = pindex->GetUndoPos(); if (pos.IsNull()) { return error("%s: no undo data available", __func__); } @@ -1630,8 +1630,8 @@ void static FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); - CDiskBlockPos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); - CDiskBlockPos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); bool status = true; status &= BlockFileSeq().Flush(block_pos_old, fFinalize); @@ -1641,13 +1641,13 @@ void static FlushBlockFile(bool fFinalize = false) } } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) { // Write undo information to disk if (pindex->GetUndoPos().IsNull()) { - CDiskBlockPos _pos; + FlatFilePos _pos; if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) return error("ConnectBlock(): FindUndoPos failed"); if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) @@ -2920,7 +2920,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2966,7 +2966,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -3015,7 +3015,7 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int return true; } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) { pos.nFile = nFile; @@ -3416,26 +3416,26 @@ bool ProcessNewBlockHeaders(const std::vector& headers, CValidatio } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) { +static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); - CDiskBlockPos blockPos; + FlatFilePos blockPos; if (dbp != nullptr) blockPos = *dbp; if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { error("%s: FindBlockPos failed", __func__); - return CDiskBlockPos(); + return FlatFilePos(); } if (dbp == nullptr) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); - return CDiskBlockPos(); + return FlatFilePos(); } } return blockPos; } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool CChainState::AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) +bool CChainState::AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3497,7 +3497,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr& pblock, CVali // Write block to history file if (fNewBlock) *fNewBlock = true; try { - CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); + FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -3628,7 +3628,7 @@ void PruneOneBlockFile(const int fileNumber) void UnlinkPrunedFiles(const std::set& setFilesToPrune) { for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - CDiskBlockPos pos(*it, 0); + FlatFilePos pos(*it, 0); fs::remove(BlockFileSeq().FileName(pos)); fs::remove(UndoFileSeq().FileName(pos)); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); @@ -3748,16 +3748,16 @@ static FlatFileSeq UndoFileSeq() return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const CDiskBlockPos &pos) +fs::path GetBlockPosFilename(const FlatFilePos &pos) { return BlockFileSeq().FileName(pos); } @@ -3866,7 +3866,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE } for (std::set::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { - CDiskBlockPos pos(*it, 0); + FlatFilePos pos(*it, 0); if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { return false; } @@ -4299,7 +4299,7 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) try { const CBlock& block = chainparams.GenesisBlock(); - CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); + FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = AddToBlockIndex(block); @@ -4316,10 +4316,10 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return g_chainstate.LoadGenesisBlock(chainparams); } -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) - static std::multimap mapBlocksUnknownParent; + static std::multimap mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); int nLoaded = 0; @@ -4405,9 +4405,9 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB while (!queue.empty()) { uint256 head = queue.front(); queue.pop_front(); - std::pair::iterator, std::multimap::iterator> range = mapBlocksUnknownParent.equal_range(head); + std::pair::iterator, std::multimap::iterator> range = mapBlocksUnknownParent.equal_range(head); while (range.first != range.second) { - std::multimap::iterator it = range.first; + std::multimap::iterator it = range.first; std::shared_ptr pblockrecursive = std::make_shared(); if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) { diff --git a/src/validation.h b/src/validation.h index 19f8e8ab7a..4ef82ee2c7 100644 --- a/src/validation.h +++ b/src/validation.h @@ -243,11 +243,11 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main); /** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const CDiskBlockPos &pos); +fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Import blocks from an external file */ -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = nullptr); +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Load the block tree and coins database from disk, @@ -386,9 +386,9 @@ void InitScriptExecutionCache(); /** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start); +bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); bool ReadRawBlockFromDisk(std::vector& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); /** Functions for validating blocks and updating the block tree */ -- cgit v1.2.3 From 4c01e4e159db82ce4b2acce75f709cac996367d7 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Sun, 6 Jan 2019 22:23:34 -0800 Subject: flatfile: Unit tests for FlatFileSeq methods. --- src/Makefile.test.include | 1 + src/test/flatfile_tests.cpp | 123 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/test/flatfile_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 84bc326cfe..09fef5a1dd 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -83,6 +83,7 @@ BITCOIN_TESTS =\ test/cuckoocache_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ + test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp new file mode 100644 index 0000000000..079a09f8f9 --- /dev/null +++ b/src/test/flatfile_tests.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +#include + +BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(flatfile_filename) +{ + auto data_dir = SetDataDir("flatfile_test"); + + FlatFilePos pos(456, 789); + + FlatFileSeq seq1(data_dir, "a", 16 * 1024); + BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat"); + + FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024); + BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat"); +} + +BOOST_AUTO_TEST_CASE(flatfile_open) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 16 * 1024); + + std::string line1("A purely peer-to-peer version of electronic cash would allow online " + "payments to be sent directly from one party to another without going " + "through a financial institution."); + std::string line2("Digital signatures provide part of the solution, but the main benefits are " + "lost if a trusted third party is still required to prevent double-spending."); + + size_t pos1 = 0; + size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION); + + // Write first line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line1, 256); + } + + // Attempt to append to file opened in read-only mode. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure); + } + + // Append second line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line2, 256); + } + + // Read text from file in read-only mode. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line1); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Read text from file with position offset. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Ensure another file in the sequence has no data. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure); + } +} + +BOOST_AUTO_TEST_CASE(flatfile_allocate) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200); + BOOST_CHECK(!out_of_space); +} + +BOOST_AUTO_TEST_CASE(flatfile_flush) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + seq.Allocate(FlatFilePos(0, 0), 1, out_of_space); + + // Flush without finalize should not truncate file. + seq.Flush(FlatFilePos(0, 1)); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100); + + // Flush with finalize should truncate file. + seq.Flush(FlatFilePos(0, 1), true); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1); +} + +BOOST_AUTO_TEST_SUITE_END() -- cgit v1.2.3 From 04cca330944f859b4ed68cb8da8a79f5206fd630 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Thu, 24 Jan 2019 11:20:57 -0800 Subject: Style cleanup. --- src/flatfile.cpp | 19 +++++++++---------- src/flatfile.h | 14 ++++++-------- src/util/system.cpp | 8 ++++---- src/util/system.h | 2 +- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/flatfile.cpp b/src/flatfile.cpp index d2e11825d5..8a8f7b681c 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -30,25 +30,24 @@ fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); } -FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool fReadOnly) +FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) { - if (pos.IsNull()) + if (pos.IsNull()) { return nullptr; + } fs::path path = FileName(pos); fs::create_directories(path.parent_path()); - FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); - if (!file && !fReadOnly) + FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+"); + if (!file && !read_only) file = fsbridge::fopen(path, "wb+"); if (!file) { LogPrintf("Unable to open file %s\n", path.string()); return nullptr; } - if (pos.nPos) { - if (fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); - fclose(file); - return nullptr; - } + if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) { + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + fclose(file); + return nullptr; } return file; } diff --git a/src/flatfile.h b/src/flatfile.h index 6051970490..374ceff411 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -24,14 +24,12 @@ struct FlatFilePos READWRITE(VARINT(nPos)); } - FlatFilePos() { - SetNull(); - } + FlatFilePos() : nFile(-1), nPos(0) {} - FlatFilePos(int nFileIn, unsigned int nPosIn) { - nFile = nFileIn; - nPos = nPosIn; - } + FlatFilePos(int nFileIn, unsigned int nPosIn) : + nFile(nFileIn), + nPos(nPosIn) + {} friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { return (a.nFile == b.nFile && a.nPos == b.nPos); @@ -72,7 +70,7 @@ public: fs::path FileName(const FlatFilePos& pos) const; /** Open a handle to the file at the given position. */ - FILE* Open(const FlatFilePos& pos, bool fReadOnly = false); + FILE* Open(const FlatFilePos& pos, bool read_only = false); /** * Allocate additional space in a file after the given starting position. The amount allocated diff --git a/src/util/system.cpp b/src/util/system.cpp index 3e6c2ae5d4..1f0a097e3d 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -135,12 +135,12 @@ bool DirIsWritable(const fs::path& directory) return true; } -bool CheckDiskSpace(const fs::path& dir, uint64_t nAdditionalBytes) +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes) { - constexpr uint64_t nMinDiskSpace = 52428800; // 50 MiB + constexpr uint64_t min_disk_space = 52428800; // 50 MiB - uint64_t nFreeBytesAvailable = fs::space(dir).available; - return nFreeBytesAvailable >= nMinDiskSpace + nAdditionalBytes; + uint64_t free_bytes_available = fs::space(dir).available; + return free_bytes_available >= min_disk_space + additional_bytes; } /** diff --git a/src/util/system.h b/src/util/system.h index 2a2ee9bc1d..bcfcaaaed0 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -72,7 +72,7 @@ bool RenameOver(fs::path src, fs::path dest); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name); bool DirIsWritable(const fs::path& directory); -bool CheckDiskSpace(const fs::path& dir, uint64_t nAdditionalBytes = 0); +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); /** Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. -- cgit v1.2.3