aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/Makefile.am2
-rw-r--r--src/Makefile.test.include1
-rw-r--r--src/chain.h49
-rw-r--r--src/flatfile.cpp98
-rw-r--r--src/flatfile.h96
-rw-r--r--src/index/txindex.cpp8
-rw-r--r--src/init.cpp8
-rw-r--r--src/test/flatfile_tests.cpp123
-rw-r--r--src/util/system.cpp8
-rw-r--r--src/util/system.h1
-rw-r--r--src/validation.cpp189
-rw-r--r--src/validation.h15
12 files changed, 417 insertions, 181 deletions
diff --git a/src/Makefile.am b/src/Makefile.am
index d491530ca1..2e297d3dac 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -128,6 +128,7 @@ BITCOIN_CORE_H = \
core_io.h \
core_memusage.h \
cuckoocache.h \
+ flatfile.h \
fs.h \
httprpc.h \
httpserver.h \
@@ -247,6 +248,7 @@ libbitcoin_server_a_SOURCES = \
chain.cpp \
checkpoints.cpp \
consensus/tx_verify.cpp \
+ flatfile.cpp \
httprpc.cpp \
httpserver.cpp \
index/base.cpp \
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index 84bc326cfe..09fef5a1dd 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -83,6 +83,7 @@ BITCOIN_TESTS =\
test/cuckoocache_tests.cpp \
test/denialofservice_tests.cpp \
test/descriptor_tests.cpp \
+ test/flatfile_tests.cpp \
test/fs_tests.cpp \
test/getarg_tests.cpp \
test/hash_tests.cpp \
diff --git a/src/chain.h b/src/chain.h
index 5a6f10b84f..2b6d2d082c 100644
--- a/src/chain.h
+++ b/src/chain.h
@@ -8,6 +8,7 @@
#include <arith_uint256.h>
#include <consensus/params.h>
+#include <flatfile.h>
#include <primitives/block.h>
#include <tinyformat.h>
#include <uint256.h>
@@ -90,46 +91,6 @@ public:
}
};
-struct CDiskBlockPos
-{
- int nFile;
- unsigned int nPos;
-
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
- READWRITE(VARINT(nPos));
- }
-
- CDiskBlockPos() {
- SetNull();
- }
-
- CDiskBlockPos(int nFileIn, unsigned int nPosIn) {
- nFile = nFileIn;
- nPos = nPosIn;
- }
-
- friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) {
- return (a.nFile == b.nFile && a.nPos == b.nPos);
- }
-
- friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) {
- return !(a == b);
- }
-
- void SetNull() { nFile = -1; nPos = 0; }
- bool IsNull() const { return (nFile == -1); }
-
- std::string ToString() const
- {
- return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos);
- }
-
-};
-
enum BlockStatus: uint32_t {
//! Unused.
BLOCK_VALID_UNKNOWN = 0,
@@ -266,8 +227,8 @@ public:
nNonce = block.nNonce;
}
- CDiskBlockPos GetBlockPos() const {
- CDiskBlockPos ret;
+ FlatFilePos GetBlockPos() const {
+ FlatFilePos ret;
if (nStatus & BLOCK_HAVE_DATA) {
ret.nFile = nFile;
ret.nPos = nDataPos;
@@ -275,8 +236,8 @@ public:
return ret;
}
- CDiskBlockPos GetUndoPos() const {
- CDiskBlockPos ret;
+ FlatFilePos GetUndoPos() const {
+ FlatFilePos ret;
if (nStatus & BLOCK_HAVE_UNDO) {
ret.nFile = nFile;
ret.nPos = nUndoPos;
diff --git a/src/flatfile.cpp b/src/flatfile.cpp
new file mode 100644
index 0000000000..8a8f7b681c
--- /dev/null
+++ b/src/flatfile.cpp
@@ -0,0 +1,98 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stdexcept>
+
+#include <flatfile.h>
+#include <logging.h>
+#include <tinyformat.h>
+#include <util/system.h>
+
+FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) :
+ m_dir(std::move(dir)),
+ m_prefix(prefix),
+ m_chunk_size(chunk_size)
+{
+ if (chunk_size == 0) {
+ throw std::invalid_argument("chunk_size must be positive");
+ }
+}
+
+std::string FlatFilePos::ToString() const
+{
+ return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos);
+}
+
+fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const
+{
+ return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile);
+}
+
+FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only)
+{
+ if (pos.IsNull()) {
+ return nullptr;
+ }
+ fs::path path = FileName(pos);
+ fs::create_directories(path.parent_path());
+ FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+");
+ if (!file && !read_only)
+ file = fsbridge::fopen(path, "wb+");
+ if (!file) {
+ LogPrintf("Unable to open file %s\n", path.string());
+ return nullptr;
+ }
+ if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) {
+ LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string());
+ fclose(file);
+ return nullptr;
+ }
+ return file;
+}
+
+size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space)
+{
+ out_of_space = false;
+
+ unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size;
+ unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size;
+ if (n_new_chunks > n_old_chunks) {
+ size_t old_size = pos.nPos;
+ size_t new_size = n_new_chunks * m_chunk_size;
+ size_t inc_size = new_size - old_size;
+
+ if (CheckDiskSpace(m_dir, inc_size)) {
+ FILE *file = Open(pos);
+ if (file) {
+ LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile);
+ AllocateFileRange(file, pos.nPos, inc_size);
+ fclose(file);
+ return inc_size;
+ }
+ } else {
+ out_of_space = true;
+ }
+ }
+ return 0;
+}
+
+bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize)
+{
+ FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos
+ if (!file) {
+ return error("%s: failed to open file %d", __func__, pos.nFile);
+ }
+ if (finalize && !TruncateFile(file, pos.nPos)) {
+ fclose(file);
+ return error("%s: failed to truncate file %d", __func__, pos.nFile);
+ }
+ if (!FileCommit(file)) {
+ fclose(file);
+ return error("%s: failed to commit file %d", __func__, pos.nFile);
+ }
+
+ fclose(file);
+ return true;
+}
diff --git a/src/flatfile.h b/src/flatfile.h
new file mode 100644
index 0000000000..374ceff411
--- /dev/null
+++ b/src/flatfile.h
@@ -0,0 +1,96 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_FLATFILE_H
+#define BITCOIN_FLATFILE_H
+
+#include <string>
+
+#include <fs.h>
+#include <serialize.h>
+
+struct FlatFilePos
+{
+ int nFile;
+ unsigned int nPos;
+
+ ADD_SERIALIZE_METHODS;
+
+ template <typename Stream, typename Operation>
+ inline void SerializationOp(Stream& s, Operation ser_action) {
+ READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
+ READWRITE(VARINT(nPos));
+ }
+
+ FlatFilePos() : nFile(-1), nPos(0) {}
+
+ FlatFilePos(int nFileIn, unsigned int nPosIn) :
+ nFile(nFileIn),
+ nPos(nPosIn)
+ {}
+
+ friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) {
+ return (a.nFile == b.nFile && a.nPos == b.nPos);
+ }
+
+ friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) {
+ return !(a == b);
+ }
+
+ void SetNull() { nFile = -1; nPos = 0; }
+ bool IsNull() const { return (nFile == -1); }
+
+ std::string ToString() const;
+};
+
+/**
+ * FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates
+ * access to and efficient management of these files.
+ */
+class FlatFileSeq
+{
+private:
+ const fs::path m_dir;
+ const char* const m_prefix;
+ const size_t m_chunk_size;
+
+public:
+ /**
+ * Constructor
+ *
+ * @param dir The base directory that all files live in.
+ * @param prefix A short prefix given to all file names.
+ * @param chunk_size Disk space is pre-allocated in multiples of this amount.
+ */
+ FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size);
+
+ /** Get the name of the file at the given position. */
+ fs::path FileName(const FlatFilePos& pos) const;
+
+ /** Open a handle to the file at the given position. */
+ FILE* Open(const FlatFilePos& pos, bool read_only = false);
+
+ /**
+ * Allocate additional space in a file after the given starting position. The amount allocated
+ * will be the minimum multiple of the sequence chunk size greater than add_size.
+ *
+ * @param[in] pos The starting position that bytes will be allocated after.
+ * @param[in] add_size The minimum number of bytes to be allocated.
+ * @param[out] out_of_space Whether the allocation failed due to insufficient disk space.
+ * @return The number of bytes successfully allocated.
+ */
+ size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space);
+
+ /**
+ * Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final.
+ *
+ * @param[in] pos The first unwritten position in the file to be flushed.
+ * @param[in] finalize True if no more data will be written to this file.
+ * @return true on success, false on failure.
+ */
+ bool Flush(const FlatFilePos& pos, bool finalize = false);
+};
+
+#endif // BITCOIN_FLATFILE_H
diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp
index 10bc8419dd..7367ec7cb6 100644
--- a/src/index/txindex.cpp
+++ b/src/index/txindex.cpp
@@ -16,7 +16,7 @@ constexpr char DB_TXINDEX_BLOCK = 'T';
std::unique_ptr<TxIndex> g_txindex;
-struct CDiskTxPos : public CDiskBlockPos
+struct CDiskTxPos : public FlatFilePos
{
unsigned int nTxOffset; // after header
@@ -24,11 +24,11 @@ struct CDiskTxPos : public CDiskBlockPos
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITEAS(CDiskBlockPos, *this);
+ READWRITEAS(FlatFilePos, *this);
READWRITE(VARINT(nTxOffset));
}
- CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
+ CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
}
CDiskTxPos() {
@@ -36,7 +36,7 @@ struct CDiskTxPos : public CDiskBlockPos
}
void SetNull() {
- CDiskBlockPos::SetNull();
+ FlatFilePos::SetNull();
nTxOffset = 0;
}
};
diff --git a/src/init.cpp b/src/init.cpp
index e0da962030..ee9a451d0b 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -668,8 +668,8 @@ static void ThreadImport(std::vector<fs::path> vImportFiles)
if (fReindex) {
int nFile = 0;
while (true) {
- CDiskBlockPos pos(nFile, 0);
- if (!fs::exists(GetBlockPosFilename(pos, "blk")))
+ FlatFilePos pos(nFile, 0);
+ if (!fs::exists(GetBlockPosFilename(pos)))
break; // No block files left to reindex
FILE *file = OpenBlockFile(pos, true);
if (!file)
@@ -1669,11 +1669,11 @@ bool AppInitMain(InitInterfaces& interfaces)
// ********************************************************* Step 11: import blocks
- if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) {
+ if (!CheckDiskSpace(GetDataDir())) {
InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir()));
return false;
}
- if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) {
+ if (!CheckDiskSpace(GetBlocksDir())) {
InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir()));
return false;
}
diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp
new file mode 100644
index 0000000000..079a09f8f9
--- /dev/null
+++ b/src/test/flatfile_tests.cpp
@@ -0,0 +1,123 @@
+// Copyright (c) 2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <flatfile.h>
+#include <test/test_bitcoin.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup)
+
+BOOST_AUTO_TEST_CASE(flatfile_filename)
+{
+ auto data_dir = SetDataDir("flatfile_test");
+
+ FlatFilePos pos(456, 789);
+
+ FlatFileSeq seq1(data_dir, "a", 16 * 1024);
+ BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat");
+
+ FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024);
+ BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat");
+}
+
+BOOST_AUTO_TEST_CASE(flatfile_open)
+{
+ auto data_dir = SetDataDir("flatfile_test");
+ FlatFileSeq seq(data_dir, "a", 16 * 1024);
+
+ std::string line1("A purely peer-to-peer version of electronic cash would allow online "
+ "payments to be sent directly from one party to another without going "
+ "through a financial institution.");
+ std::string line2("Digital signatures provide part of the solution, but the main benefits are "
+ "lost if a trusted third party is still required to prevent double-spending.");
+
+ size_t pos1 = 0;
+ size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION);
+
+ // Write first line to file.
+ {
+ CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION);
+ file << LIMITED_STRING(line1, 256);
+ }
+
+ // Attempt to append to file opened in read-only mode.
+ {
+ CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION);
+ BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure);
+ }
+
+ // Append second line to file.
+ {
+ CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION);
+ file << LIMITED_STRING(line2, 256);
+ }
+
+ // Read text from file in read-only mode.
+ {
+ std::string text;
+ CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION);
+
+ file >> LIMITED_STRING(text, 256);
+ BOOST_CHECK_EQUAL(text, line1);
+
+ file >> LIMITED_STRING(text, 256);
+ BOOST_CHECK_EQUAL(text, line2);
+ }
+
+ // Read text from file with position offset.
+ {
+ std::string text;
+ CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION);
+
+ file >> LIMITED_STRING(text, 256);
+ BOOST_CHECK_EQUAL(text, line2);
+ }
+
+ // Ensure another file in the sequence has no data.
+ {
+ std::string text;
+ CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION);
+ BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure);
+ }
+}
+
+BOOST_AUTO_TEST_CASE(flatfile_allocate)
+{
+ auto data_dir = SetDataDir("flatfile_test");
+ FlatFileSeq seq(data_dir, "a", 100);
+
+ bool out_of_space;
+
+ BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100);
+ BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100);
+ BOOST_CHECK(!out_of_space);
+
+ BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0);
+ BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100);
+ BOOST_CHECK(!out_of_space);
+
+ BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101);
+ BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200);
+ BOOST_CHECK(!out_of_space);
+}
+
+BOOST_AUTO_TEST_CASE(flatfile_flush)
+{
+ auto data_dir = SetDataDir("flatfile_test");
+ FlatFileSeq seq(data_dir, "a", 100);
+
+ bool out_of_space;
+ seq.Allocate(FlatFilePos(0, 0), 1, out_of_space);
+
+ // Flush without finalize should not truncate file.
+ seq.Flush(FlatFilePos(0, 1));
+ BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100);
+
+ // Flush with finalize should truncate file.
+ seq.Flush(FlatFilePos(0, 1), true);
+ BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/util/system.cpp b/src/util/system.cpp
index 5ae0604894..9594dd81bf 100644
--- a/src/util/system.cpp
+++ b/src/util/system.cpp
@@ -135,6 +135,14 @@ bool DirIsWritable(const fs::path& directory)
return true;
}
+bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes)
+{
+ constexpr uint64_t min_disk_space = 52428800; // 50 MiB
+
+ uint64_t free_bytes_available = fs::space(dir).available;
+ return free_bytes_available >= min_disk_space + additional_bytes;
+}
+
/**
* Interpret a string argument as a boolean.
*
diff --git a/src/util/system.h b/src/util/system.h
index 6899e38c9e..54eb88e261 100644
--- a/src/util/system.h
+++ b/src/util/system.h
@@ -72,6 +72,7 @@ bool RenameOver(fs::path src, fs::path dest);
bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false);
void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name);
bool DirIsWritable(const fs::path& directory);
+bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0);
/** Release all directory locks. This is used for unit testing only, at runtime
* the global destructor will take care of the locks.
diff --git a/src/validation.cpp b/src/validation.cpp
index f3d34dca70..358992b74d 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -15,6 +15,7 @@
#include <consensus/tx_verify.h>
#include <consensus/validation.h>
#include <cuckoocache.h>
+#include <flatfile.h>
#include <hash.h>
#include <index/txindex.h>
#include <policy/fees.h>
@@ -165,7 +166,7 @@ public:
* that it doesn't descend from an invalid block, and then add it to mapBlockIndex.
*/
bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
- bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
+ bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
// Block (dis)connection on a given view:
DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view);
@@ -204,7 +205,7 @@ private:
void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main);
- void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
+ void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
@@ -317,7 +318,9 @@ static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState &
static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight);
static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight);
bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr);
-static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false);
+static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false);
+static FlatFileSeq BlockFileSeq();
+static FlatFileSeq UndoFileSeq();
bool CheckFinalTx(const CTransaction &tx, int flags)
{
@@ -1042,7 +1045,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus
// CBlock and CBlockIndex
//
-static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart)
+static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart)
{
// Open history file to append
CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION);
@@ -1063,7 +1066,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes
return true;
}
-bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams)
+bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams)
{
block.SetNull();
@@ -1089,7 +1092,7 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus:
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams)
{
- CDiskBlockPos blockPos;
+ FlatFilePos blockPos;
{
LOCK(cs_main);
blockPos = pindex->GetBlockPos();
@@ -1103,9 +1106,9 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus
return true;
}
-bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start)
+bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start)
{
- CDiskBlockPos hpos = pos;
+ FlatFilePos hpos = pos;
hpos.nPos -= 8; // Seek back 8 bytes for meta header
CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION);
if (filein.IsNull()) {
@@ -1140,7 +1143,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos,
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start)
{
- CDiskBlockPos block_pos;
+ FlatFilePos block_pos;
{
LOCK(cs_main);
block_pos = pindex->GetBlockPos();
@@ -1452,7 +1455,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi
namespace {
-bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
+bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
{
// Open history file to append
CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION);
@@ -1481,7 +1484,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint
static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex)
{
- CDiskBlockPos pos = pindex->GetUndoPos();
+ FlatFilePos pos = pindex->GetUndoPos();
if (pos.IsNull()) {
return error("%s: no undo data available", __func__);
}
@@ -1627,37 +1630,24 @@ void static FlushBlockFile(bool fFinalize = false)
{
LOCK(cs_LastBlockFile);
- CDiskBlockPos posOld(nLastBlockFile, 0);
- bool status = true;
-
- FILE *fileOld = OpenBlockFile(posOld);
- if (fileOld) {
- if (fFinalize)
- status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize);
- status &= FileCommit(fileOld);
- fclose(fileOld);
- }
-
- fileOld = OpenUndoFile(posOld);
- if (fileOld) {
- if (fFinalize)
- status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize);
- status &= FileCommit(fileOld);
- fclose(fileOld);
- }
+ FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize);
+ FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize);
+ bool status = true;
+ status &= BlockFileSeq().Flush(block_pos_old, fFinalize);
+ status &= UndoFileSeq().Flush(undo_pos_old, fFinalize);
if (!status) {
AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");
}
}
-static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize);
+static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize);
static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams)
{
// Write undo information to disk
if (pindex->GetUndoPos().IsNull()) {
- CDiskBlockPos _pos;
+ FlatFilePos _pos;
if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40))
return error("ConnectBlock(): FindUndoPos failed");
if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart()))
@@ -2134,8 +2124,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
// Write blocks and block index to disk.
if (fDoFullFlush || fPeriodicWrite) {
// Depend on nMinDiskSpace to ensure we can write block index
- if (!CheckDiskSpace(0, true))
- return state.Error("out of disk space");
+ if (!CheckDiskSpace(GetBlocksDir())) {
+ return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
+ }
// First make sure all block and undo data is flushed to disk.
FlushBlockFile();
// Then update all block file information (which may refer to block and undo files).
@@ -2168,8 +2159,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
// twice (once in the log, and once in the tables). This is already
// an overestimation, as most will delete an existing entry or
// overwrite one. Still, use a conservative safety factor of 2.
- if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize()))
- return state.Error("out of disk space");
+ if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * pcoinsTip->GetCacheSize())) {
+ return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
+ }
// Flush the chainstate (which may refer to block index entries).
if (!pcoinsTip->Flush())
return AbortNode(state, "Failed to write to coin database");
@@ -2922,7 +2914,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block)
}
/** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */
-void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams)
+void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams)
{
pindexNew->nTx = block.vtx.size();
pindexNew->nChainTx = 0;
@@ -2968,7 +2960,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi
}
}
-static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false)
+static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false)
{
LOCK(cs_LastBlockFile);
@@ -3003,21 +2995,13 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
vinfoBlockFile[nFile].nSize += nAddSize;
if (!fKnown) {
- unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
- unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
- if (nNewChunks > nOldChunks) {
- if (fPruneMode)
- fCheckForPruning = true;
- if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) {
- FILE *file = OpenBlockFile(pos);
- if (file) {
- LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile);
- AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos);
- fclose(file);
- }
- }
- else
- return error("out of disk space");
+ bool out_of_space;
+ size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
+ if (out_of_space) {
+ return AbortNode("Disk space is low!", _("Error: Disk space is low!"));
+ }
+ if (bytes_allocated != 0 && fPruneMode) {
+ fCheckForPruning = true;
}
}
@@ -3025,32 +3009,23 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
return true;
}
-static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize)
+static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize)
{
pos.nFile = nFile;
LOCK(cs_LastBlockFile);
- unsigned int nNewSize;
pos.nPos = vinfoBlockFile[nFile].nUndoSize;
- nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize;
+ vinfoBlockFile[nFile].nUndoSize += nAddSize;
setDirtyFileInfo.insert(nFile);
- unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
- unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
- if (nNewChunks > nOldChunks) {
- if (fPruneMode)
- fCheckForPruning = true;
- if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) {
- FILE *file = OpenUndoFile(pos);
- if (file) {
- LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile);
- AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos);
- fclose(file);
- }
- }
- else
- return state.Error("out of disk space");
+ bool out_of_space;
+ size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
+ if (out_of_space) {
+ return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
+ }
+ if (bytes_allocated != 0 && fPruneMode) {
+ fCheckForPruning = true;
}
return true;
@@ -3435,26 +3410,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio
}
/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */
-static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) {
+static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) {
unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION);
- CDiskBlockPos blockPos;
+ FlatFilePos blockPos;
if (dbp != nullptr)
blockPos = *dbp;
if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) {
error("%s: FindBlockPos failed", __func__);
- return CDiskBlockPos();
+ return FlatFilePos();
}
if (dbp == nullptr) {
if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) {
AbortNode("Failed to write block");
- return CDiskBlockPos();
+ return FlatFilePos();
}
}
return blockPos;
}
/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */
-bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock)
+bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock)
{
const CBlock& block = *pblock;
@@ -3516,7 +3491,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali
// Write block to history file
if (fNewBlock) *fNewBlock = true;
try {
- CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp);
+ FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp);
if (blockPos.IsNull()) {
state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__));
return false;
@@ -3647,9 +3622,9 @@ void PruneOneBlockFile(const int fileNumber)
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
{
for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
- CDiskBlockPos pos(*it, 0);
- fs::remove(GetBlockPosFilename(pos, "blk"));
- fs::remove(GetBlockPosFilename(pos, "rev"));
+ FlatFilePos pos(*it, 0);
+ fs::remove(BlockFileSeq().FileName(pos));
+ fs::remove(UndoFileSeq().FileName(pos));
LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
}
}
@@ -3757,52 +3732,28 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte
nLastBlockWeCanPrune, count);
}
-bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir)
+static FlatFileSeq BlockFileSeq()
{
- uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available;
-
- // Check for nMinDiskSpace bytes (currently 50MB)
- if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes)
- return AbortNode("Disk space is low!", _("Error: Disk space is low!"));
-
- return true;
+ return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE);
}
-static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly)
+static FlatFileSeq UndoFileSeq()
{
- if (pos.IsNull())
- return nullptr;
- fs::path path = GetBlockPosFilename(pos, prefix);
- fs::create_directories(path.parent_path());
- FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+");
- if (!file && !fReadOnly)
- file = fsbridge::fopen(path, "wb+");
- if (!file) {
- LogPrintf("Unable to open file %s\n", path.string());
- return nullptr;
- }
- if (pos.nPos) {
- if (fseek(file, pos.nPos, SEEK_SET)) {
- LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string());
- fclose(file);
- return nullptr;
- }
- }
- return file;
+ return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE);
}
-FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) {
- return OpenDiskFile(pos, "blk", fReadOnly);
+FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) {
+ return BlockFileSeq().Open(pos, fReadOnly);
}
/** Open an undo file (rev?????.dat) */
-static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) {
- return OpenDiskFile(pos, "rev", fReadOnly);
+static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) {
+ return UndoFileSeq().Open(pos, fReadOnly);
}
-fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix)
+fs::path GetBlockPosFilename(const FlatFilePos &pos)
{
- return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile);
+ return BlockFileSeq().FileName(pos);
}
CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash)
@@ -3909,7 +3860,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE
}
for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++)
{
- CDiskBlockPos pos(*it, 0);
+ FlatFilePos pos(*it, 0);
if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) {
return false;
}
@@ -4342,7 +4293,7 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams)
try {
const CBlock& block = chainparams.GenesisBlock();
- CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr);
+ FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr);
if (blockPos.IsNull())
return error("%s: writing genesis block to disk failed", __func__);
CBlockIndex *pindex = AddToBlockIndex(block);
@@ -4359,10 +4310,10 @@ bool LoadGenesisBlock(const CChainParams& chainparams)
return g_chainstate.LoadGenesisBlock(chainparams);
}
-bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp)
+bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp)
{
// Map of disk positions for blocks with unknown parent (only used for reindex)
- static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent;
+ static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent;
int64_t nStart = GetTimeMillis();
int nLoaded = 0;
@@ -4448,9 +4399,9 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB
while (!queue.empty()) {
uint256 head = queue.front();
queue.pop_front();
- std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head);
+ std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head);
while (range.first != range.second) {
- std::multimap<uint256, CDiskBlockPos>::iterator it = range.first;
+ std::multimap<uint256, FlatFilePos>::iterator it = range.first;
std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>();
if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus()))
{
diff --git a/src/validation.h b/src/validation.h
index 1975846b69..4ef82ee2c7 100644
--- a/src/validation.h
+++ b/src/validation.h
@@ -181,9 +181,6 @@ extern arith_uint256 nMinimumChainWork;
/** Best header we've seen so far (used for getheaders queries' starting points). */
extern CBlockIndex *pindexBestHeader;
-/** Minimum disk space required - used in CheckDiskSpace() */
-static const uint64_t nMinDiskSpace = 52428800;
-
/** Pruning-related variables and constants */
/** True if any block files have ever been pruned. */
extern bool fHavePruned;
@@ -245,14 +242,12 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons
*/
bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main);
-/** Check whether enough disk space is available for an incoming block */
-bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false);
/** Open a block file (blk?????.dat) */
-FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false);
+FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false);
/** Translation to a filesystem path */
-fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix);
+fs::path GetBlockPosFilename(const FlatFilePos &pos);
/** Import blocks from an external file */
-bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = nullptr);
+bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr);
/** Ensures we have a genesis block in the block tree, possibly writing one to disk. */
bool LoadGenesisBlock(const CChainParams& chainparams);
/** Load the block tree and coins database from disk,
@@ -391,9 +386,9 @@ void InitScriptExecutionCache();
/** Functions for disk access for blocks */
-bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams);
+bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams);
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams);
-bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start);
+bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start);
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start);
/** Functions for validating blocks and updating the block tree */