Merge bitcoin/bitcoin#26606: wallet: Implement independent BDB parser

d51fbab4b32d56765e8faab6ad01245fb259b0ca wallet, test: Be able to always swap BDB endianness (Ava Chow)
0b753156ce60c29efb2386954ba7555ad8f642f5 test: Test bdb_ro dump of wallet without reset LSNs (Ava Chow)
c1984f128284589423b7e0cc06c9a3b23a242d95 test: Test dumping dbs with overflow pages (Ava Chow)
fd7b16e391ed320e35255157a28be14c947ef30a test: Test dumps of other endian BDB files (Ava Chow)
6ace3e953f0864bd7818f040c59a1bc70aa47512 bdb: Be able to make byteswapped databases (Ava Chow)
d9878903fb34939dee8e1462f079acc68110253d Error if LSNs are not reset (Ava Chow)
4d7a3ae78e55f25868979f1bd920857a4aecb825 Berkeley RO Database fuzz test (TheCharlatan)
3568dce9e93295674cdf5458c5bdf93ff01fd0a2 tests: Add BerkeleyRO to db prefix tests (Ava Chow)
70cfbfdadf16d3b115309c6938f07ef5b96c7cc1 wallettool: Optionally use BERKELEY_RO as format when dumping BDB wallets (Ava Chow)
dd57713f6ede3d46e97ee7df87c10001b0bf4c3d Add MakeBerkeleyRODatabase (Ava Chow)
6e50bee67d1d58aecd8a0ce8b7c3f5a7979365f5 Implement handling of other endianness in BerkeleyRODatabase (Ava Chow)
cdd61c9cc108df8e13f4e3891ff2c96355b3ee38 wallet: implement independent BDB deserializer in BerkeleyRODatabase (Ava Chow)
ecba23097955dad7208baa687fc405c846aee794 wallet: implement BerkeleyRODatabase::Backup (Ava Chow)
0c8e72847603540bb29b8b8aeb80fa3f2e3a2c9a wallet: implement BerkeleyROBatch (Ava Chow)
756ff9b478484b17c4a6e65c171c2e4fecb21ad4 wallet: add dummy BerkeleyRODatabase and BerkeleyROBatch classes (Ava Chow)
ca18aea5c4975ace4e307be96c74641d203fa389 Add AutoFile::seek and tell (Ava Chow)

Pull request description:

  Split from #26596

  This PR adds `BerkeleyRODatabase` which is an independent implementation of a BDB file parser. It provides read only access to a BDB file, and can therefore be used as a read only database backend for wallets. This will be used for dumping legacy wallet records and migrating legacy wallets without the need for BDB itself.

  Wallettool's `dump` command is changed to use `BerkeleyRODatabase` instead of `BerkeleyDatabase` (and `CWallet` itself) to demonstrate that this parser works and to test it against the existing wallettool functional tests.

ACKs for top commit:
  josibake:
    reACK https://github.com/bitcoin/bitcoin/commit/d51fbab4b32d56765e8faab6ad01245fb259b0ca
  TheCharlatan:
    Re-ACK d51fbab4b32d56765e8faab6ad01245fb259b0ca
  furszy:
    reACK d51fbab4b32d56765e8faab6ad01245fb259b0ca
  laanwj:
    re-ACK d51fbab4b32d56765e8faab6ad01245fb259b0ca
  theStack:
    ACK d51fbab4b32d56765e8faab6ad01245fb259b0ca

Tree-SHA512: 1e7b97edf223b2974eed2e9eac1179fc82bb6359e0a66b7d2a0c8b9fa515eae9ea036f1edf7c76cdab2e75ad994962b134b41056ccfbc33b8d54f0859e86657b

1 files changed, 124 insertions, 0 deletions

diff --git a/src/wallet/migrate.h b/src/wallet/migrate.h
new file mode 100644
index 0000000000..e4826450af
--- /dev/null
+++ b/src/wallet/migrate.h
@@ -0,0 +1,124 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_WALLET_MIGRATE_H
+#define BITCOIN_WALLET_MIGRATE_H
+
+#include <wallet/db.h>
+
+#include <optional>
+
+namespace wallet {
+
+using BerkeleyROData = std::map<SerializeData, SerializeData, std::less<>>;
+
+/**
+ * A class representing a BerkeleyDB file from which we can only read records.
+ * This is used only for migration of legacy to descriptor wallets
+ */
+class BerkeleyRODatabase : public WalletDatabase
+{
+private:
+    const fs::path m_filepath;
+
+public:
+    /** Create DB handle */
+    BerkeleyRODatabase(const fs::path& filepath, bool open = true) : WalletDatabase(), m_filepath(filepath)
+    {
+        if (open) Open();
+    }
+    ~BerkeleyRODatabase(){};
+
+    BerkeleyROData m_records;
+
+    /** Open the database if it is not already opened. */
+    void Open() override;
+
+    /** Indicate the a new database user has began using the database. Increments m_refcount */
+    void AddRef() override {}
+    /** Indicate that database user has stopped using the database and that it could be flushed or closed. Decrement m_refcount */
+    void RemoveRef() override {}
+
+    /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero
+     */
+    bool Rewrite(const char* pszSkip = nullptr) override { return false; }
+
+    /** Back up the entire database to a file.
+     */
+    bool Backup(const std::string& strDest) const override;
+
+    /** Make sure all changes are flushed to database file.
+     */
+    void Flush() override {}
+    /** Flush to the database file and close the database.
+     *  Also close the environment if no other databases are open in it.
+     */
+    void Close() override {}
+    /* flush the wallet passively (TRY_LOCK)
+       ideal to be called periodically */
+    bool PeriodicFlush() override { return false; }
+
+    void IncrementUpdateCounter() override {}
+
+    void ReloadDbEnv() override {}
+
+    /** Return path to main database file for logs and error messages. */
+    std::string Filename() override { return fs::PathToString(m_filepath); }
+
+    std::string Format() override { return "bdb_ro"; }
+
+    /** Make a DatabaseBatch connected to this database */
+    std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override;
+};
+
+class BerkeleyROCursor : public DatabaseCursor
+{
+private:
+    const BerkeleyRODatabase& m_database;
+    BerkeleyROData::const_iterator m_cursor;
+    BerkeleyROData::const_iterator m_cursor_end;
+
+public:
+    explicit BerkeleyROCursor(const BerkeleyRODatabase& database, Span<const std::byte> prefix = {});
+    ~BerkeleyROCursor() {}
+
+    Status Next(DataStream& key, DataStream& value) override;
+};
+
+/** RAII class that provides access to a BerkeleyRODatabase */
+class BerkeleyROBatch : public DatabaseBatch
+{
+private:
+    const BerkeleyRODatabase& m_database;
+
+    bool ReadKey(DataStream&& key, DataStream& value) override;
+    // WriteKey returns true since various automatic upgrades for older wallets will expect writing to not fail.
+    // It is okay for this batch type to not actually write anything as those automatic upgrades will occur again after migration.
+    bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return true; }
+    bool EraseKey(DataStream&& key) override { return false; }
+    bool HasKey(DataStream&& key) override;
+    bool ErasePrefix(Span<const std::byte> prefix) override { return false; }
+
+public:
+    explicit BerkeleyROBatch(const BerkeleyRODatabase& database) : m_database(database) {}
+    ~BerkeleyROBatch() {}
+
+    BerkeleyROBatch(const BerkeleyROBatch&) = delete;
+    BerkeleyROBatch& operator=(const BerkeleyROBatch&) = delete;
+
+    void Flush() override {}
+    void Close() override {}
+
+    std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<BerkeleyROCursor>(m_database); }
+    std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override;
+    bool TxnBegin() override { return false; }
+    bool TxnCommit() override { return false; }
+    bool TxnAbort() override { return false; }
+};
+
+//! Return object giving access to Berkeley Read Only database at specified path.
+std::unique_ptr<BerkeleyRODatabase> MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error);
+} // namespace wallet
+
+#endif // BITCOIN_WALLET_MIGRATE_H