Merge bitcoin/bitcoin#28186: kernel: Prune leveldb headers

d8f1222ac50f089a0af29eaf8ce0555bad8366ef refactor: Correct dbwrapper key naming (TheCharlatan)
be8f159ac59b9e700cbd3314ed71ebf39bd5b67a build: Remove leveldb from BITCOIN_INCLUDES (TheCharlatan)
c95b37d641b1eed4a62d55ca5342a6ed8c7a1ce7 refactor: Move CDBWrapper leveldb members to their own context struct (TheCharlatan)
c534a615e93452a5f509aaf5f68c600391a98d6a refactor: Split dbwrapper CDBWrapper::EstimateSize implementation (TheCharlatan)
586448888b72f7c87db4dcd30fc4e4044afae13b refactor: Move HandleError to dbwrapper implementation (TheCharlatan)
dede0eef7adb7413f62f5abd68cac8e01635ba4a refactor: Split dbwrapper CDBWrapper::Exists implementation (TheCharlatan)
a5c2eb57484314b04ec94523d14e0ef0c6c46d4f refactor: Fix logging.h includes (TheCharlatan)
84058e0eed9c05bc30984b39131e88ad1425628f refactor: Split dbwrapper CDBWrapper::Read implementation (TheCharlatan)
e4af2408f2ac59788567b6fc8cb3a68fc43da9fe refactor: Pimpl leveldb::Iterator for CDBIterator (TheCharlatan)
ef941ff1281e76308c3e746e592375bec023e9e4 refactor: Split dbwrapper CDBIterator::GetValue implementation (TheCharlatan)
b7a1ab5cb4e60230f62c94efb3a10d07c9af4883 refactor: Split dbwrapper CDBIterator::GetKey implementation (TheCharlatan)
d7437908cdf242626263ba9d5541addcddadc594 refactor: Split dbwrapper CDBIterator::Seek implementation (TheCharlatan)
ea8135de7e617259cda3fc7b1c8e7569d454fd57 refactor: Pimpl leveldb::batch for CDBBatch (TheCharlatan)
b9870c920dc475ec759eaf7339ea42aecba92138 refactor: Split dbwrapper CDBatch::Erase implementation (TheCharlatan)
532ee812a499e13b123af6b8415d8de1f3804f0f refactor: Split dbwrapper CDBBatch::Write implementation (TheCharlatan)
afc534df9adbf5599b286b5dc3531a4b9ac2d056 refactor: Wrap DestroyDB in dbwrapper helper (TheCharlatan)

Pull request description:

  Leveldb headers are currently included in the `dbwrapper.h` file and thus available to many of Bitcoin Core's source files. However, leveldb-specific functionality should be abstracted by the `dbwrapper` and does not need to be available to the rest of the code. Having leveldb included in a widely-used header such as `dbwrapper.h` bloats the entire project's header tree.

  The `dbwrapper` is a key component of the libbitcoinkernel library. Future users of this library would not want to contend with  having the leveldb headers exposed and potentially polluting their project's namespace.

  For these reasons, the leveldb headers are removed from the `dbwrapper` by moving leveldb-specific code to the implementation file and creating a [pimpl](https://en.cppreference.com/w/cpp/language/pimpl) where leveldb member variables are indispensable. As a final step, the leveldb include flags are removed from the `BITCOIN_INCLUDES` and moved to places where the dbwrapper is compiled.

  ---

  This pull request is part of the [libbitcoinkernel project](https://github.com/bitcoin/bitcoin/issues/27587), and more specifically its stage 1 step 3 "Decouple most non-consensus headers from libbitcoinkernel".

ACKs for top commit:
  stickies-v:
    re-ACK https://github.com/bitcoin/bitcoin/commit/d8f1222ac50f089a0af29eaf8ce0555bad8366ef
  MarcoFalke:
    ACK d8f1222ac50f089a0af29eaf8ce0555bad8366ef  🔠

Tree-SHA512: 0f58309be165af0162e648233451cd80fda88726fc10c0da7bfe4ec2ffa9afe63fbf7ffae9493698d3f39653b4ad870c372eee652ecc90ab1c29d86c387070f3

1 files changed, 192 insertions, 41 deletions

diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp
index 7cf6bef90f..c95937ba75 100644
--- a/src/dbwrapper.cpp
+++ b/src/dbwrapper.cpp
@@ -4,9 +4,12 @@
 
 #include <dbwrapper.h>
 
+#include <clientversion.h>
 #include <logging.h>
 #include <random.h>
-#include <tinyformat.h>
+#include <serialize.h>
+#include <span.h>
+#include <streams.h>
 #include <util/fs.h>
 #include <util/fs_helpers.h>
 #include <util/strencodings.h>
@@ -23,9 +26,31 @@
 #include <leveldb/helpers/memenv/memenv.h>
 #include <leveldb/iterator.h>
 #include <leveldb/options.h>
+#include <leveldb/slice.h>
 #include <leveldb/status.h>
+#include <leveldb/write_batch.h>
 #include <memory>
 #include <optional>
+#include <utility>
+
+static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); }
+
+bool DestroyDB(const std::string& path_str)
+{
+    return leveldb::DestroyDB(path_str, {}).ok();
+}
+
+/** Handle database error by throwing dbwrapper_error exception.
+ */
+static void HandleError(const leveldb::Status& status)
+{
+    if (status.ok())
+        return;
+    const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
+    LogPrintf("%s\n", errmsg);
+    LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
+    throw dbwrapper_error(errmsg);
+}
 
 class CBitcoinLevelDBLogger : public leveldb::Logger {
 public:
@@ -127,24 +152,91 @@ static leveldb::Options GetOptions(size_t nCacheSize)
     return options;
 }
 
+struct CDBBatch::WriteBatchImpl {
+    leveldb::WriteBatch batch;
+};
+
+CDBBatch::CDBBatch(const CDBWrapper& _parent) : parent(_parent),
+                                                m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()},
+                                                ssValue(SER_DISK, CLIENT_VERSION){};
+
+CDBBatch::~CDBBatch() = default;
+
+void CDBBatch::Clear()
+{
+    m_impl_batch->batch.Clear();
+    size_estimate = 0;
+}
+
+void CDBBatch::WriteImpl(Span<const std::byte> key, CDataStream& ssValue)
+{
+    leveldb::Slice slKey(CharCast(key.data()), key.size());
+    ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
+    leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
+    m_impl_batch->batch.Put(slKey, slValue);
+    // LevelDB serializes writes as:
+    // - byte: header
+    // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
+    // - byte[]: key
+    // - varint: value length
+    // - byte[]: value
+    // The formula below assumes the key and value are both less than 16k.
+    size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
+}
+
+void CDBBatch::EraseImpl(Span<const std::byte> key)
+{
+    leveldb::Slice slKey(CharCast(key.data()), key.size());
+    m_impl_batch->batch.Delete(slKey);
+    // LevelDB serializes erases as:
+    // - byte: header
+    // - varint: key length
+    // - byte[]: key
+    // The formula below assumes the key is less than 16kB.
+    size_estimate += 2 + (slKey.size() > 127) + slKey.size();
+}
+
+struct LevelDBContext {
+    //! custom environment this database is using (may be nullptr in case of default environment)
+    leveldb::Env* penv;
+
+    //! database options used
+    leveldb::Options options;
+
+    //! options used when reading from the database
+    leveldb::ReadOptions readoptions;
+
+    //! options used when iterating over values of the database
+    leveldb::ReadOptions iteroptions;
+
+    //! options used when writing to the database
+    leveldb::WriteOptions writeoptions;
+
+    //! options used when sync writing to the database
+    leveldb::WriteOptions syncoptions;
+
+    //! the database itself
+    leveldb::DB* pdb;
+};
+
 CDBWrapper::CDBWrapper(const DBParams& params)
-    : m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
+    : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
 {
-    penv = nullptr;
-    readoptions.verify_checksums = true;
-    iteroptions.verify_checksums = true;
-    iteroptions.fill_cache = false;
-    syncoptions.sync = true;
-    options = GetOptions(params.cache_bytes);
-    options.create_if_missing = true;
+    DBContext().penv = nullptr;
+    DBContext().readoptions.verify_checksums = true;
+    DBContext().iteroptions.verify_checksums = true;
+    DBContext().iteroptions.fill_cache = false;
+    DBContext().syncoptions.sync = true;
+    DBContext().options = GetOptions(params.cache_bytes);
+    DBContext().options.create_if_missing = true;
     if (params.memory_only) {
-        penv = leveldb::NewMemEnv(leveldb::Env::Default());
-        options.env = penv;
+        DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default());
+        DBContext().options.env = DBContext().penv;
     } else {
         if (params.wipe_data) {
             LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(params.path));
-            leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), options);
-            dbwrapper_private::HandleError(result);
+            leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options);
+            HandleError(result);
         }
         TryCreateDirectories(params.path);
         LogPrintf("Opening LevelDB in %s\n", fs::PathToString(params.path));
@@ -153,13 +245,13 @@ CDBWrapper::CDBWrapper(const DBParams& params)
     // because on POSIX leveldb passes the byte string directly to ::open(), and
     // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
     // (see env_posix.cc and env_windows.cc).
-    leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(params.path), &pdb);
-    dbwrapper_private::HandleError(status);
+    leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb);
+    HandleError(status);
     LogPrintf("Opened LevelDB successfully\n");
 
     if (params.options.force_compact) {
         LogPrintf("Starting database compaction of %s\n", fs::PathToString(params.path));
-        pdb->CompactRange(nullptr, nullptr);
+        DBContext().pdb->CompactRange(nullptr, nullptr);
         LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path));
     }
 
@@ -185,16 +277,16 @@ CDBWrapper::CDBWrapper(const DBParams& params)
 
 CDBWrapper::~CDBWrapper()
 {
-    delete pdb;
-    pdb = nullptr;
-    delete options.filter_policy;
-    options.filter_policy = nullptr;
-    delete options.info_log;
-    options.info_log = nullptr;
-    delete options.block_cache;
-    options.block_cache = nullptr;
-    delete penv;
-    options.env = nullptr;
+    delete DBContext().pdb;
+    DBContext().pdb = nullptr;
+    delete DBContext().options.filter_policy;
+    DBContext().options.filter_policy = nullptr;
+    delete DBContext().options.info_log;
+    DBContext().options.info_log = nullptr;
+    delete DBContext().options.block_cache;
+    DBContext().options.block_cache = nullptr;
+    delete DBContext().penv;
+    DBContext().options.env = nullptr;
 }
 
 bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
@@ -204,8 +296,8 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
     if (log_memory) {
         mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
     }
-    leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
-    dbwrapper_private::HandleError(status);
+    leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch);
+    HandleError(status);
     if (log_memory) {
         double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
         LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
@@ -218,7 +310,7 @@ size_t CDBWrapper::DynamicMemoryUsage() const
 {
     std::string memory;
     std::optional<size_t> parsed;
-    if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
+    if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
         LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
         return 0;
     }
@@ -244,6 +336,45 @@ std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
     return ret;
 }
 
+std::optional<std::string> CDBWrapper::ReadImpl(Span<const std::byte> key) const
+{
+    leveldb::Slice slKey(CharCast(key.data()), key.size());
+    std::string strValue;
+    leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
+    if (!status.ok()) {
+        if (status.IsNotFound())
+            return std::nullopt;
+        LogPrintf("LevelDB read failure: %s\n", status.ToString());
+        HandleError(status);
+    }
+    return strValue;
+}
+
+bool CDBWrapper::ExistsImpl(Span<const std::byte> key) const
+{
+    leveldb::Slice slKey(CharCast(key.data()), key.size());
+
+    std::string strValue;
+    leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
+    if (!status.ok()) {
+        if (status.IsNotFound())
+            return false;
+        LogPrintf("LevelDB read failure: %s\n", status.ToString());
+        HandleError(status);
+    }
+    return true;
+}
+
+size_t CDBWrapper::EstimateSizeImpl(Span<const std::byte> key1, Span<const std::byte> key2) const
+{
+    leveldb::Slice slKey1(CharCast(key1.data()), key1.size());
+    leveldb::Slice slKey2(CharCast(key2.data()), key2.size());
+    uint64_t size = 0;
+    leveldb::Range range(slKey1, slKey2);
+    DBContext().pdb->GetApproximateSizes(&range, 1, &size);
+    return size;
+}
+
 bool CDBWrapper::IsEmpty()
 {
     std::unique_ptr<CDBIterator> it(NewIterator());
@@ -251,23 +382,43 @@ bool CDBWrapper::IsEmpty()
     return !(it->Valid());
 }
 
-CDBIterator::~CDBIterator() { delete piter; }
-bool CDBIterator::Valid() const { return piter->Valid(); }
-void CDBIterator::SeekToFirst() { piter->SeekToFirst(); }
-void CDBIterator::Next() { piter->Next(); }
+struct CDBIterator::IteratorImpl {
+    const std::unique_ptr<leveldb::Iterator> iter;
 
-namespace dbwrapper_private {
+    explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {}
+};
 
-void HandleError(const leveldb::Status& status)
+CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent),
+                                                                                            m_impl_iter(std::move(_piter)) {}
+
+CDBIterator* CDBWrapper::NewIterator()
 {
-    if (status.ok())
-        return;
-    const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
-    LogPrintf("%s\n", errmsg);
-    LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
-    throw dbwrapper_error(errmsg);
+    return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))};
 }
 
+void CDBIterator::SeekImpl(Span<const std::byte> key)
+{
+    leveldb::Slice slKey(CharCast(key.data()), key.size());
+    m_impl_iter->iter->Seek(slKey);
+}
+
+Span<const std::byte> CDBIterator::GetKeyImpl() const
+{
+    return MakeByteSpan(m_impl_iter->iter->key());
+}
+
+Span<const std::byte> CDBIterator::GetValueImpl() const
+{
+    return MakeByteSpan(m_impl_iter->iter->value());
+}
+
+CDBIterator::~CDBIterator() = default;
+bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); }
+void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
+void CDBIterator::Next() { m_impl_iter->iter->Next(); }
+
+namespace dbwrapper_private {
+
 const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
 {
     return w.obfuscate_key;